From ca123ec40e8500a230bc2eb65eac6439096c5204 Mon Sep 17 00:00:00 2001 From: Sydney Acksman Date: Sat, 22 Feb 2020 22:55:48 -0600 Subject: [PATCH 01/52] Swap registry from input when merging from existing input --- csharp/src/Google.Protobuf/MessageParser.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/csharp/src/Google.Protobuf/MessageParser.cs b/csharp/src/Google.Protobuf/MessageParser.cs index 06d0f1059c..bbdbdc16aa 100644 --- a/csharp/src/Google.Protobuf/MessageParser.cs +++ b/csharp/src/Google.Protobuf/MessageParser.cs @@ -159,14 +159,17 @@ namespace Google.Protobuf internal void MergeFrom(IMessage message, CodedInputStream codedInput) { bool originalDiscard = codedInput.DiscardUnknownFields; + ExtensionRegistry originalRegistry = codedInput.ExtensionRegistry; try { codedInput.DiscardUnknownFields = DiscardUnknownFields; + codedInput.ExtensionRegistry = Extensions; message.MergeFrom(codedInput); } finally { codedInput.DiscardUnknownFields = originalDiscard; + codedInput.ExtensionRegistry = originalRegistry; } } From d36d84c77b37d712eea5a7539c038d38ab0e87fb Mon Sep 17 00:00:00 2001 From: Sydney Acksman Date: Sun, 23 Feb 2020 16:08:13 -0600 Subject: [PATCH 02/52] Add test for parsing using coded input --- .../GeneratedMessageTest.Proto2.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/csharp/src/Google.Protobuf.Test/GeneratedMessageTest.Proto2.cs b/csharp/src/Google.Protobuf.Test/GeneratedMessageTest.Proto2.cs index 718c3edcb8..921ef4e3a6 100644 --- a/csharp/src/Google.Protobuf.Test/GeneratedMessageTest.Proto2.cs +++ b/csharp/src/Google.Protobuf.Test/GeneratedMessageTest.Proto2.cs @@ -376,5 +376,18 @@ namespace Google.Protobuf TestGroupExtension extendable_parsed = TestGroupExtension.Parser.WithExtensionRegistry(new ExtensionRegistry() { TestNestedExtension.Extensions.OptionalGroupExtension }).ParseFrom(bytes); Assert.AreEqual(message, extendable_parsed); } + + [Test] + public void RoundTrip_ParseUsingCodedInput() + { + var message = new TestAllExtensions(); + message.SetExtension(UnittestExtensions.OptionalBoolExtension, true); + byte[] bytes = message.ToByteArray(); + using (CodedInputStream input = new CodedInputStream(bytes)) + { + var parsed = TestAllExtensions.Parser.WithExtensionRegistry(new ExtensionRegistry() { UnittestExtensions.OptionalBoolExtension }).ParseFrom(input); + Assert.AreEqual(message, parsed); + } + } } } From 115af28e8fb580c60bef269802b8f07e8ecccabf Mon Sep 17 00:00:00 2001 From: Phil Felton Date: Mon, 22 Jun 2020 11:16:54 +0100 Subject: [PATCH 03/52] Make propertyName public --- .../Google.Protobuf/Reflection/FieldDescriptor.cs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs b/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs index 7324e3dfc6..0ba16cf96f 100644 --- a/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs +++ b/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs @@ -45,7 +45,6 @@ namespace Google.Protobuf.Reflection private MessageDescriptor extendeeType; private MessageDescriptor messageType; private FieldType fieldType; - private readonly string propertyName; // Annoyingly, needed in Crosslink. private IFieldAccessor accessor; /// @@ -70,6 +69,11 @@ namespace Google.Protobuf.Reflection /// public string JsonName { get; } + /// + /// The name of the property in the ContainingType.ClrType class. + /// + public string PropertyName { get; } + /// /// Indicates whether this field supports presence, either implicitly (e.g. due to it being a message /// type field) or explicitly via Has/Clear members. If this returns true, it is safe to call @@ -123,7 +127,7 @@ namespace Google.Protobuf.Reflection // for later. // We could trust the generated code and check whether the type of the property is // a MapField, but that feels a tad nasty. - this.propertyName = propertyName; + PropertyName = propertyName; Extension = extension; JsonName = Proto.JsonName == "" ? JsonFormatter.ToJsonName(Proto.Name) : Proto.JsonName; } @@ -436,15 +440,15 @@ namespace Google.Protobuf.Reflection // If we're given no property name, that's because we really don't want an accessor. // This could be because it's a map message, or it could be that we're loading a FileDescriptor dynamically. // TODO: Support dynamic messages. - if (propertyName == null) + if (PropertyName == null) { return null; } - var property = ContainingType.ClrType.GetProperty(propertyName); + var property = ContainingType.ClrType.GetProperty(PropertyName); if (property == null) { - throw new DescriptorValidationException(this, $"Property {propertyName} not found in {ContainingType.ClrType}"); + throw new DescriptorValidationException(this, $"Property {PropertyName} not found in {ContainingType.ClrType}"); } return IsMap ? new MapFieldAccessor(property, this) : IsRepeated ? new RepeatedFieldAccessor(property, this) From 37f34f0ad66f4f9fa8a2259318c478fd73ed9272 Mon Sep 17 00:00:00 2001 From: David Mollitor Date: Thu, 27 Aug 2020 19:58:37 -0400 Subject: [PATCH 04/52] Use ArrayList copy constructor --- java/core/src/main/java/com/google/protobuf/FieldSet.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/FieldSet.java b/java/core/src/main/java/com/google/protobuf/FieldSet.java index d52aede958..8306c471cc 100644 --- a/java/core/src/main/java/com/google/protobuf/FieldSet.java +++ b/java/core/src/main/java/com/google/protobuf/FieldSet.java @@ -1078,8 +1078,7 @@ final class FieldSet> { // Wrap the contents in a new list so that the caller cannot change // the list's contents after setting it. - final List newList = new ArrayList(); - newList.addAll((List) value); + final List newList = new ArrayList((List) value); for (final Object element : newList) { verifyType(descriptor.getLiteType(), element); hasNestedBuilders = hasNestedBuilders || element instanceof MessageLite.Builder; From 00a7ea4de12da348bfb55159b1d3469d249c9a45 Mon Sep 17 00:00:00 2001 From: alusco-scratch <60235755+alusco-scratch@users.noreply.github.com> Date: Thu, 1 Oct 2020 21:20:31 -0700 Subject: [PATCH 05/52] Override Map.clone to use Map's dup method Message and Repeated field override clone so that it uses the internal implementation of dup but Map is missing this and only implements dup. This can lead to unexpected behavior since two out of three complex types behave correctly. --- ruby/ext/google/protobuf_c/map.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ruby/ext/google/protobuf_c/map.c b/ruby/ext/google/protobuf_c/map.c index 00d23a76fa..2caffd0861 100644 --- a/ruby/ext/google/protobuf_c/map.c +++ b/ruby/ext/google/protobuf_c/map.c @@ -827,6 +827,8 @@ void Map_register(VALUE module) { rb_define_method(klass, "clear", Map_clear, 0); rb_define_method(klass, "length", Map_length, 0); rb_define_method(klass, "dup", Map_dup, 0); + // Also define #clone so that we don't inherit Object#clone. + rb_define_method(klass, "clone", Map_dup, 0); rb_define_method(klass, "==", Map_eq, 1); rb_define_method(klass, "hash", Map_hash, 0); rb_define_method(klass, "to_h", Map_to_h, 0); From 74056a0e2834577ad590d023ee73cc20cf103d38 Mon Sep 17 00:00:00 2001 From: panda Date: Fri, 4 Dec 2020 13:38:45 -0800 Subject: [PATCH 06/52] Make MessageToDict convert map keys to strings --- python/google/protobuf/json_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/google/protobuf/json_format.py b/python/google/protobuf/json_format.py index c8f5602f36..cb91b37a71 100644 --- a/python/google/protobuf/json_format.py +++ b/python/google/protobuf/json_format.py @@ -236,7 +236,7 @@ class _Printer(object): else: recorded_key = 'false' else: - recorded_key = key + recorded_key = str(key) js_map[recorded_key] = self._FieldToJsonObject( v_field, value[key]) js[name] = js_map From 0a7a9a98c5e4a4d64890984e335a3d248fda3e42 Mon Sep 17 00:00:00 2001 From: Masaki Hara Date: Mon, 25 Jan 2021 15:04:50 +0900 Subject: [PATCH 07/52] Ruby: build extensions for arm64-darwin --- kokoro/release/ruby/macos/ruby/ruby_build_environment.sh | 9 ++++----- ruby/Rakefile | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh index 046b604b40..460af2c7fc 100755 --- a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh +++ b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh @@ -60,7 +60,8 @@ set -x ruby --version | grep 'ruby 2.7.0' for v in 3.0.0 2.7.0 ; do ccache -c - rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin11 MAKE="$MAKE" + rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin MAKE="$MAKE" + rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=arm64-darwin MAKE="$MAKE" done set +x rvm use 2.5.0 @@ -68,11 +69,9 @@ set -x ruby --version | grep 'ruby 2.5.0' for v in 2.6.0 2.5.1 2.4.0 2.3.0; do ccache -c - rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin11 MAKE="$MAKE" + rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin MAKE="$MAKE" + rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=arm64-darwin MAKE="$MAKE" done set +x rvm use 2.7.0 set -x - -sed 's/x86_64-darwin-11/universal-darwin/' ~/.rake-compiler/config.yml > "$CROSS_RUBY" -mv "$CROSS_RUBY" ~/.rake-compiler/config.yml diff --git a/ruby/Rakefile b/ruby/Rakefile index 3e3da055d3..2fd5288059 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -70,7 +70,7 @@ else ext.cross_platform = [ 'x86-mingw32', 'x64-mingw32', 'x86_64-linux', 'x86-linux', - 'universal-darwin' + 'x86_64-darwin', 'arm64-darwin', ] end From 6d847adda7b79f1e4668e848600caca09227459d Mon Sep 17 00:00:00 2001 From: Chris McClymont Date: Wed, 5 May 2021 13:26:14 +1000 Subject: [PATCH 08/52] Add class method from_time to ruby well known types - Timestamp --- ruby/lib/google/protobuf/well_known_types.rb | 5 +++++ ruby/tests/well_known_types_test.rb | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ruby/lib/google/protobuf/well_known_types.rb b/ruby/lib/google/protobuf/well_known_types.rb index 37f8d5b675..c9a9ab1455 100755 --- a/ruby/lib/google/protobuf/well_known_types.rb +++ b/ruby/lib/google/protobuf/well_known_types.rb @@ -85,6 +85,11 @@ module Google def from_time(time) self.seconds = time.to_i self.nanos = time.nsec + self + end + + def self.from_time(time) + new.from_time(time) end def to_i diff --git a/ruby/tests/well_known_types_test.rb b/ruby/tests/well_known_types_test.rb index ea042eb024..c069764167 100755 --- a/ruby/tests/well_known_types_test.rb +++ b/ruby/tests/well_known_types_test.rb @@ -15,16 +15,20 @@ class TestWellKnownTypes < Test::Unit::TestCase # millisecond accuracy time = Time.at(123456, 654321) - ts.from_time(time) + ts = Google::Protobuf::Timestamp.from_time(time) assert_equal 123456, ts.seconds assert_equal 654321000, ts.nanos assert_equal time, ts.to_time # nanosecond accuracy time = Time.at(123456, Rational(654321321, 1000)) - ts.from_time(time) + ts = Google::Protobuf::Timestamp.from_time(time) assert_equal 654321321, ts.nanos assert_equal time, ts.to_time + + # Instance method returns the same value as class method + assert_equal Google::Protobuf::Timestamp.new.from_time(time), + Google::Protobuf::Timestamp.from_time(time) end def test_duration From f2b5c33baff633f0dec7a6364e6c04efa16bc69a Mon Sep 17 00:00:00 2001 From: Masaki Hara Date: Wed, 29 Sep 2021 11:51:11 +0900 Subject: [PATCH 09/52] Specify aarch64 for older autotools --- kokoro/release/ruby/macos/ruby/ruby_build_environment.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh index 460af2c7fc..9954fb0c2f 100755 --- a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh +++ b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh @@ -61,7 +61,7 @@ ruby --version | grep 'ruby 2.7.0' for v in 3.0.0 2.7.0 ; do ccache -c rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin MAKE="$MAKE" - rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=arm64-darwin MAKE="$MAKE" + rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=aarch64-darwin MAKE="$MAKE" done set +x rvm use 2.5.0 @@ -70,7 +70,7 @@ ruby --version | grep 'ruby 2.5.0' for v in 2.6.0 2.5.1 2.4.0 2.3.0; do ccache -c rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin MAKE="$MAKE" - rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=arm64-darwin MAKE="$MAKE" + rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=aarch64-darwin MAKE="$MAKE" done set +x rvm use 2.7.0 From 72fa7269d9b07f89346d6d734178da255a977b28 Mon Sep 17 00:00:00 2001 From: Elliotte Rusty Harold Date: Wed, 6 Oct 2021 11:25:25 +0000 Subject: [PATCH 10/52] EasyMock should have scope test --- java/util/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/java/util/pom.xml b/java/util/pom.xml index 06f6c1b737..31ccf5d567 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -49,6 +49,7 @@ org.easymock easymock + test com.google.truth From eee60cec13e0656577f7d6baeecbbbe673de4e12 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Thu, 7 Oct 2021 20:28:20 +0000 Subject: [PATCH 11/52] Update the base image from jessie to stretch (for parity with the Java). Add Maven and OpenJDK 8 and add JRuby 9.2.x and 9.3.x. Replace keys.gnupg.net with keyserver.ubuntu.com as the GPG keyserver since the former no longer exists. --- kokoro/linux/dockerfile/test/ruby/Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/kokoro/linux/dockerfile/test/ruby/Dockerfile b/kokoro/linux/dockerfile/test/ruby/Dockerfile index b73bf84095..2affd14196 100644 --- a/kokoro/linux/dockerfile/test/ruby/Dockerfile +++ b/kokoro/linux/dockerfile/test/ruby/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:jessie +FROM debian:stretch # Install dependencies. We start with the basic ones require to build protoc # and the C++ build @@ -20,10 +20,13 @@ RUN apt-get update && apt-get install -y \ parallel \ time \ wget \ + # Java dependencies + maven \ + openjdk-8-jdk \ && apt-get clean # Install rvm -RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys \ +RUN gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys \ 409B6B1796C275462A1703113804BB82D39DC0E3 \ 7D2BAF1CF37B13E2069D6956105BD0E739499BDB RUN \curl -sSL https://get.rvm.io | bash -s master @@ -34,6 +37,8 @@ RUN /bin/bash -l -c "rvm install 2.5.1" RUN /bin/bash -l -c "rvm install 2.6.0" RUN /bin/bash -l -c "rvm install 2.7.0" RUN /bin/bash -l -c "rvm install 3.0.0" +RUN /bin/bash -l -c "rvm install jruby-9.2.19.0" +RUN /bin/bash -l -c "rvm install jruby-9.3.0.0" RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" From 728878e98f460fe8383a954d19d44e81ef76b2a2 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Thu, 7 Oct 2021 18:45:38 -0400 Subject: [PATCH 12/52] Add JRuby test targets for JRuby 9.2.x and 9.3.x. Standardize on JRuby 9.2.19.0 for building and for testing the 9.2.x branch. --- kokoro/linux/jruby92/build.sh | 18 ++++++++++++++++++ kokoro/linux/jruby92/continuous.cfg | 11 +++++++++++ kokoro/linux/jruby92/presubmit.cfg | 11 +++++++++++ kokoro/linux/jruby93/build.sh | 18 ++++++++++++++++++ kokoro/linux/jruby93/continuous.cfg | 11 +++++++++++ kokoro/linux/jruby93/presubmit.cfg | 11 +++++++++++ ruby/pom.xml | 2 +- ruby/travis-test.sh | 2 +- tests.sh | 13 ++++++++++--- 9 files changed, 92 insertions(+), 5 deletions(-) create mode 100755 kokoro/linux/jruby92/build.sh create mode 100644 kokoro/linux/jruby92/continuous.cfg create mode 100644 kokoro/linux/jruby92/presubmit.cfg create mode 100755 kokoro/linux/jruby93/build.sh create mode 100644 kokoro/linux/jruby93/continuous.cfg create mode 100644 kokoro/linux/jruby93/presubmit.cfg diff --git a/kokoro/linux/jruby92/build.sh b/kokoro/linux/jruby92/build.sh new file mode 100755 index 0000000000..5820115759 --- /dev/null +++ b/kokoro/linux/jruby92/build.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERHUB_ORGANIZATION=protobuftesting +export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/ruby +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="jruby92" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/jruby92/continuous.cfg b/kokoro/linux/jruby92/continuous.cfg new file mode 100644 index 0000000000..3339584149 --- /dev/null +++ b/kokoro/linux/jruby92/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/jruby92/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/jruby92/presubmit.cfg b/kokoro/linux/jruby92/presubmit.cfg new file mode 100644 index 0000000000..3339584149 --- /dev/null +++ b/kokoro/linux/jruby92/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/jruby92/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/jruby93/build.sh b/kokoro/linux/jruby93/build.sh new file mode 100755 index 0000000000..24c54d0b20 --- /dev/null +++ b/kokoro/linux/jruby93/build.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERHUB_ORGANIZATION=protobuftesting +export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/ruby +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="jruby93" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/jruby93/continuous.cfg b/kokoro/linux/jruby93/continuous.cfg new file mode 100644 index 0000000000..706d8488f7 --- /dev/null +++ b/kokoro/linux/jruby93/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/jruby93/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/jruby93/presubmit.cfg b/kokoro/linux/jruby93/presubmit.cfg new file mode 100644 index 0000000000..706d8488f7 --- /dev/null +++ b/kokoro/linux/jruby93/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/jruby93/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/ruby/pom.xml b/ruby/pom.xml index 5ba45156d8..0b13996d82 100644 --- a/ruby/pom.xml +++ b/ruby/pom.xml @@ -81,7 +81,7 @@ org.jruby jruby-complete - 9.2.11.1 + 9.2.19.0 provided diff --git a/ruby/travis-test.sh b/ruby/travis-test.sh index edca986fff..438d612da4 100755 --- a/ruby/travis-test.sh +++ b/ruby/travis-test.sh @@ -8,7 +8,7 @@ test_version() { RUBY_CONFORMANCE=test_ruby - if [ "$version" == "jruby-9.2.11.1" ] ; then + if [[ $version == jruby-9* ] ; then bash --login -c \ "rvm install $version && rvm use $version && rvm get head && \ which ruby && \ diff --git a/tests.sh b/tests.sh index f7397c1a17..06cbc6ce6d 100755 --- a/tests.sh +++ b/tests.sh @@ -452,10 +452,16 @@ build_ruby30() { cd ruby && bash travis-test.sh ruby-3.0.2 && cd .. } -build_jruby() { +build_jruby92() { internal_build_cpp # For conformance tests. internal_build_java jdk8 && cd .. # For Maven protobuf jar with local changes - cd ruby && bash travis-test.sh jruby-9.2.11.1 && cd .. + cd ruby && bash travis-test.sh jruby-9.2.19.0 && cd .. +} + +build_jruby93() { + internal_build_cpp # For conformance tests. + internal_build_java jdk8 && cd .. # For Maven protobuf jar with local changes + cd ruby && bash travis-test.sh jruby-9.3.0.0 && cd .. } build_javascript() { @@ -610,7 +616,8 @@ Usage: $0 { cpp | ruby26 | ruby27 | ruby30 | - jruby | + jruby92 | + jruby93 | ruby_all | php_all | php_all_32 | From b79a2f242c79981f647e2e1ac980cadb581c0596 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Thu, 7 Oct 2021 18:45:38 -0400 Subject: [PATCH 13/52] Add JRuby test targets for JRuby 9.2.x and 9.3.x. Standardize on JRuby 9.2.19.0 for building and for testing the 9.2.x branch. --- ruby/travis-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruby/travis-test.sh b/ruby/travis-test.sh index 438d612da4..f7fa8155ec 100755 --- a/ruby/travis-test.sh +++ b/ruby/travis-test.sh @@ -8,7 +8,7 @@ test_version() { RUBY_CONFORMANCE=test_ruby - if [[ $version == jruby-9* ] ; then + if [[ $version == jruby-9* ]] ; then bash --login -c \ "rvm install $version && rvm use $version && rvm get head && \ which ruby && \ From fd5202b6cd60d8336348bcb5ccdf8284c6430416 Mon Sep 17 00:00:00 2001 From: Alexey Solodkiy Date: Sat, 9 Oct 2021 21:39:18 +0300 Subject: [PATCH 14/52] Add twirp to RPC Implementations list (#9085) --- docs/third_party.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/third_party.md b/docs/third_party.md index ca2ac63fd3..06b31b1d25 100644 --- a/docs/third_party.md +++ b/docs/third_party.md @@ -129,6 +129,7 @@ GRPC (http://www.grpc.io/) is Google's RPC implementation for Protocol Buffers. * https://github.com/Yeolar/raster (C++) * https://github.com/jnordberg/wsrpc (JavaScript Node.js/Browser) * https://github.com/ppissias/xsrpcj (Java) +* https://github.com/twitchtv/twirp (Multiple languages) Inactive: From 5142e362bc82012c67ebaec96f193ba3310713fe Mon Sep 17 00:00:00 2001 From: NexusNull Date: Sat, 9 Oct 2021 20:42:59 +0200 Subject: [PATCH 15/52] Update README.md (#9078) remove dollar sign for easier copying --- src/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/README.md b/src/README.md index 51d9e2fe38..9db40fdde4 100644 --- a/src/README.md +++ b/src/README.md @@ -19,7 +19,7 @@ To build protobuf from source, the following tools are needed: On Ubuntu/Debian, you can install them with: - $ sudo apt-get install autoconf automake libtool curl make g++ unzip + sudo apt-get install autoconf automake libtool curl make g++ unzip On other platforms, please use the corresponding package managing tool to install them before proceeding. From 15c63ad850c0b60a728414fd107c5ed3b3eb2772 Mon Sep 17 00:00:00 2001 From: johanmoe <90901545+johanmoe@users.noreply.github.com> Date: Sat, 9 Oct 2021 20:45:56 +0200 Subject: [PATCH 16/52] Fix build error with MinGW (#9077) Fixes: * zero_copy_stream_impl.cc uses F_GETFL when build with MinGW, which isn't defined under Windows. --- src/google/protobuf/io/zero_copy_stream_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/google/protobuf/io/zero_copy_stream_impl.cc b/src/google/protobuf/io/zero_copy_stream_impl.cc index fc75e3ef56..c66bc862a7 100644 --- a/src/google/protobuf/io/zero_copy_stream_impl.cc +++ b/src/google/protobuf/io/zero_copy_stream_impl.cc @@ -104,7 +104,7 @@ FileInputStream::CopyingFileInputStream::CopyingFileInputStream( is_closed_(false), errno_(0), previous_seek_failed_(false) { -#ifndef _MSC_VER +#ifndef _WIN32 int flags = fcntl(file_, F_GETFL); flags &= ~O_NONBLOCK; fcntl(file_, F_SETFL, flags); From ae4fd1e24b15f25a8db890262faab60c4f669f66 Mon Sep 17 00:00:00 2001 From: Jimmy Yuen Ho Wong Date: Sat, 9 Oct 2021 19:54:15 +0100 Subject: [PATCH 17/52] Fix cl deprecation warning (#9046) --- editors/protobuf-mode.el | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/editors/protobuf-mode.el b/editors/protobuf-mode.el index 8102771665..aa31bd0c44 100644 --- a/editors/protobuf-mode.el +++ b/editors/protobuf-mode.el @@ -68,7 +68,7 @@ (eval-when-compile (and (= emacs-major-version 24) (>= emacs-minor-version 4) - (require 'cl)) + (require 'cl-lib)) (require 'cc-langs) (require 'cc-fonts)) From 9650e9fe8f737efcad485c2a8e6e696186ae3862 Mon Sep 17 00:00:00 2001 From: Elliotte Rusty Harold Date: Sun, 10 Oct 2021 11:11:21 +0000 Subject: [PATCH 18/52] update to 3.18.1 (#9057) * update to 3.18.1 --- java/lite.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/lite.md b/java/lite.md index f248ff6c3e..603609f676 100644 --- a/java/lite.md +++ b/java/lite.md @@ -30,7 +30,7 @@ protobuf Java runtime. If you are using Maven, use the following: com.google.protobuf protobuf-javalite - 3.9.1 + 3.18.1 ``` From 2ccc5df358fdefd026c5a7d92f99ef8cd4e9ed2d Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 11 Oct 2021 19:10:09 +0000 Subject: [PATCH 19/52] Install core rather than util --- tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests.sh b/tests.sh index 06cbc6ce6d..fcfb75f865 100755 --- a/tests.sh +++ b/tests.sh @@ -222,7 +222,7 @@ internal_build_java() { cp -r java $dir cd $dir && $MVN clean # Skip tests here - callers will decide what tests they want to run - $MVN install -pl util -Dmaven.test.skip=true + $MVN install -pl core -Dmaven.test.skip=true } build_java() { From 1b22582bc049270026ce399bd1390572bdabf203 Mon Sep 17 00:00:00 2001 From: mgabris Date: Mon, 11 Oct 2021 22:39:14 +0200 Subject: [PATCH 20/52] Fix unused variable warnings in extension_set.h (#9073) --- src/google/protobuf/extension_set.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/google/protobuf/extension_set.h b/src/google/protobuf/extension_set.h index 2c52fe7d1d..beb4202ae9 100644 --- a/src/google/protobuf/extension_set.h +++ b/src/google/protobuf/extension_set.h @@ -1359,8 +1359,8 @@ class MessageTypeTraits { ConstType default_value) { return static_cast(set.GetMessage(number, default_value)); } - static inline std::nullptr_t GetPtr(int number, const ExtensionSet& set, - ConstType default_value) { + static inline std::nullptr_t GetPtr(int /* number */, const ExtensionSet& /* set */, + ConstType /* default_value */) { // Cannot be implemented because of forward declared messages? return nullptr; } @@ -1412,13 +1412,13 @@ class RepeatedMessageTypeTraits { static inline ConstType Get(int number, const ExtensionSet& set, int index) { return static_cast(set.GetRepeatedMessage(number, index)); } - static inline std::nullptr_t GetPtr(int number, const ExtensionSet& set, - int index) { + static inline std::nullptr_t GetPtr(int /* number */, const ExtensionSet& /* set */, + int /* index */) { // Cannot be implemented because of forward declared messages? return nullptr; } - static inline std::nullptr_t GetRepeatedPtr(int number, - const ExtensionSet& set) { + static inline std::nullptr_t GetRepeatedPtr(int /* number */, + const ExtensionSet& /* set */) { // Cannot be implemented because of forward declared messages? return nullptr; } From 3bdcc1266b65e10a61d398ba15bfbec612cbdb5d Mon Sep 17 00:00:00 2001 From: Dietmar Scheidl Date: Mon, 11 Oct 2021 22:46:54 +0200 Subject: [PATCH 21/52] Fix build on AIX and SunOS (#8373) (#9065) * fix includes for AIX and SunOS --- src/google/protobuf/io/coded_stream.h | 4 ++++ src/google/protobuf/port_def.inc | 2 +- src/google/protobuf/stubs/port.h | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/google/protobuf/io/coded_stream.h b/src/google/protobuf/io/coded_stream.h index 67d6362b58..0a39c220f4 100644 --- a/src/google/protobuf/io/coded_stream.h +++ b/src/google/protobuf/io/coded_stream.h @@ -136,6 +136,10 @@ #include // __BYTE_ORDER #elif defined(__FreeBSD__) #include // __BYTE_ORDER +#elif (defined(sun) || defined(__sun)) && (defined(__SVR4) || defined(__svr4__)) +#include // __BYTE_ORDER +#elif defined(_AIX) || defined(__TOS_AIX__) +#include // BYTE_ORDER #else #if !defined(__QNX__) #include // __BYTE_ORDER diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index b5838c41b3..e8790ad53b 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -607,7 +607,7 @@ #ifdef PROTOBUF_ATTRIBUTE_INIT_PRIORITY #error PROTOBUF_ATTRIBUTE_INIT_PRIORITY was previously defined #endif -#if PROTOBUF_GNUC_MIN(3, 0) && (!defined(__APPLE__) || defined(__clang__)) +#if PROTOBUF_GNUC_MIN(3, 0) && (!defined(__APPLE__) || defined(__clang__)) && !((defined(sun) || defined(__sun)) && (defined(__SVR4) || defined(__svr4__))) #define PROTOBUF_ATTRIBUTE_INIT_PRIORITY __attribute__((init_priority((102)))) #else #define PROTOBUF_ATTRIBUTE_INIT_PRIORITY diff --git a/src/google/protobuf/stubs/port.h b/src/google/protobuf/stubs/port.h index fa0ec47150..045e25d892 100644 --- a/src/google/protobuf/stubs/port.h +++ b/src/google/protobuf/stubs/port.h @@ -61,6 +61,10 @@ #include // __BYTE_ORDER #elif defined(__FreeBSD__) #include // __BYTE_ORDER +#elif (defined(sun) || defined(__sun)) && (defined(__SVR4) || defined(__svr4__)) +#include // __BYTE_ORDER +#elif defined(_AIX) || defined(__TOS_AIX__) +#include // BYTE_ORDER #else #if !defined(__QNX__) #include // __BYTE_ORDER From 9488e2f8eb48f04fef2724e5ae3ee7113ec380df Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 12 Oct 2021 10:08:05 -0700 Subject: [PATCH 22/52] Update cmake file lists (#9038) I ran ./update_file_lists.sh to update these files. --- cmake/extract_includes.bat.in | 1 + cmake/libprotobuf.cmake | 1 - cmake/libprotoc.cmake | 16 +++------------- cmake/tests.cmake | 4 ++-- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/cmake/extract_includes.bat.in b/cmake/extract_includes.bat.in index 8e26aedd18..dd96c6ae31 100644 --- a/cmake/extract_includes.bat.in +++ b/cmake/extract_includes.bat.in @@ -48,6 +48,7 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\descriptor_database.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\duration.pb.h" include\google\protobuf\duration.pb.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\dynamic_message.h" include\google\protobuf\dynamic_message.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\empty.pb.h" include\google\protobuf\empty.pb.h +copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\explicitly_constructed.h" include\google\protobuf\explicitly_constructed.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\extension_set.h" include\google\protobuf\extension_set.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\extension_set_inl.h" include\google\protobuf\extension_set_inl.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\field_access_listener.h" include\google\protobuf\field_access_listener.h diff --git a/cmake/libprotobuf.cmake b/cmake/libprotobuf.cmake index e4b810c66a..668b8c2784 100644 --- a/cmake/libprotobuf.cmake +++ b/cmake/libprotobuf.cmake @@ -21,7 +21,6 @@ set(libprotobuf_files ${protobuf_source_dir}/src/google/protobuf/io/tokenizer.cc ${protobuf_source_dir}/src/google/protobuf/map_field.cc ${protobuf_source_dir}/src/google/protobuf/message.cc - ${protobuf_source_dir}/src/google/protobuf/reflection_internal.h ${protobuf_source_dir}/src/google/protobuf/reflection_ops.cc ${protobuf_source_dir}/src/google/protobuf/service.cc ${protobuf_source_dir}/src/google/protobuf/source_context.pb.cc diff --git a/cmake/libprotoc.cmake b/cmake/libprotoc.cmake index 505e469d07..6f043316b9 100644 --- a/cmake/libprotoc.cmake +++ b/cmake/libprotoc.cmake @@ -36,7 +36,6 @@ set(libprotoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_doc_comment.cc ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_enum.cc ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_enum_field.cc - ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_enum_field.h ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_enum_field_lite.cc ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_enum_lite.cc ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_extension.cc @@ -65,28 +64,17 @@ set(libprotoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/js/js_generator.cc ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types_embed.cc ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_enum.cc - ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_enum.h ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_enum_field.cc - ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_enum_field.h ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_extension.cc - ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_extension.h ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_field.cc - ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_field.h ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_file.cc - ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_file.h ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_generator.cc ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_helpers.cc ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_map_field.cc - ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_map_field.h ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_message.cc - ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_message.h ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_message_field.cc - ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_message_field.h - ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_nsobject_methods.h ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_oneof.cc - ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_oneof.h ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_primitive_field.cc - ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_primitive_field.h ${protobuf_source_dir}/src/google/protobuf/compiler/php/php_generator.cc ${protobuf_source_dir}/src/google/protobuf/compiler/plugin.cc ${protobuf_source_dir}/src/google/protobuf/compiler/plugin.pb.cc @@ -94,7 +82,6 @@ set(libprotoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/ruby/ruby_generator.cc ${protobuf_source_dir}/src/google/protobuf/compiler/subprocess.cc ${protobuf_source_dir}/src/google/protobuf/compiler/zip_writer.cc - ${protobuf_source_dir}/src/google/protobuf/compiler/zip_writer.h ) set(libprotoc_headers @@ -108,14 +95,17 @@ set(libprotoc_headers ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_generator.h ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_names.h ${protobuf_source_dir}/src/google/protobuf/compiler/csharp/csharp_options.h + ${protobuf_source_dir}/src/google/protobuf/compiler/importer.h ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_generator.h ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_kotlin_generator.h ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_names.h ${protobuf_source_dir}/src/google/protobuf/compiler/js/js_generator.h ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_generator.h ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_helpers.h + ${protobuf_source_dir}/src/google/protobuf/compiler/parser.h ${protobuf_source_dir}/src/google/protobuf/compiler/php/php_generator.h ${protobuf_source_dir}/src/google/protobuf/compiler/plugin.h + ${protobuf_source_dir}/src/google/protobuf/compiler/plugin.pb.h ${protobuf_source_dir}/src/google/protobuf/compiler/python/python_generator.h ${protobuf_source_dir}/src/google/protobuf/compiler/ruby/ruby_generator.h ) diff --git a/cmake/tests.cmake b/cmake/tests.cmake index 9ede4f328c..529685856e 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -119,10 +119,10 @@ set(common_lite_test_files set(common_test_files ${common_lite_test_files} - ${protobuf_source_dir}/src/google/protobuf/compiler/mock_code_generator.cc ${protobuf_source_dir}/src/google/protobuf/map_test_util.inc ${protobuf_source_dir}/src/google/protobuf/reflection_tester.cc ${protobuf_source_dir}/src/google/protobuf/test_util.cc + ${protobuf_source_dir}/src/google/protobuf/test_util.inc ${protobuf_source_dir}/src/google/protobuf/testing/file.cc ${protobuf_source_dir}/src/google/protobuf/testing/googletest.cc ) @@ -132,7 +132,6 @@ set(tests_files ${protobuf_source_dir}/src/google/protobuf/arena_unittest.cc ${protobuf_source_dir}/src/google/protobuf/arenastring_unittest.cc ${protobuf_source_dir}/src/google/protobuf/compiler/annotation_test_util.cc - ${protobuf_source_dir}/src/google/protobuf/compiler/annotation_test_util.h ${protobuf_source_dir}/src/google/protobuf/compiler/command_line_interface_unittest.cc ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_bootstrap_unittest.cc ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/cpp_move_unittest.cc @@ -145,6 +144,7 @@ set(tests_files ${protobuf_source_dir}/src/google/protobuf/compiler/importer_unittest.cc ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_doc_comment_unittest.cc ${protobuf_source_dir}/src/google/protobuf/compiler/java/java_plugin_unittest.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/mock_code_generator.cc ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_helpers_unittest.cc ${protobuf_source_dir}/src/google/protobuf/compiler/parser_unittest.cc ${protobuf_source_dir}/src/google/protobuf/compiler/python/python_plugin_unittest.cc From 99612d0885f2fe76400803ccdb425af46f29f481 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 12 Oct 2021 10:21:22 -0700 Subject: [PATCH 23/52] Update CHANGES.txt --- CHANGES.txt | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 7682fbdd19..66f1bc0f4c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,19 @@ Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) - Protocol Compiler + + Python + * Proto2 DecodeError now includes message name in error message + + C++ * Make proto2::Message::DiscardUnknownFields() non-virtual + * Separate RepeatedPtrField into its own header file + * For default floating point values of 0, consider all bits significant + + Java + * For default floating point values of 0, consider all bits significant + * Annotate `//java/com/google/protobuf/util/...` with nullness annotations + + Kotlin + * Switch Kotlin proto DSLs to be implemented with inline value classes 2021-10-04 version 3.18.1 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) From 68c17dcde81a7dfe2633eef7e94a3916f851765f Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 12 Oct 2021 11:12:39 -0700 Subject: [PATCH 24/52] Tweak syntax of casting function to void GCC 4.9 seems to be unable to handle (void) syntax with a function, but it is OK with a static_cast to void. --- src/google/protobuf/repeated_ptr_field.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/google/protobuf/repeated_ptr_field.h b/src/google/protobuf/repeated_ptr_field.h index c96b17517c..7d635e3bb5 100644 --- a/src/google/protobuf/repeated_ptr_field.h +++ b/src/google/protobuf/repeated_ptr_field.h @@ -1631,7 +1631,7 @@ class RepeatedPtrIterator { : it_(other.it_) { // Force a compiler error if the other type is not convertible to ours. if (false) { - (void)[](OtherElement* from) -> Element* { return from; }; + static_cast([](OtherElement* from) -> Element* { return from; }); } } From 7f79a416ff4950ed255423812e29004e9d1db182 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 12 Oct 2021 11:49:50 -0700 Subject: [PATCH 25/52] Update the file lists with ./update_file_lists.sh --- BUILD | 1 + cmake/extract_includes.bat.in | 1 + 2 files changed, 2 insertions(+) diff --git a/BUILD b/BUILD index 9fbf69dfca..a9ed429f10 100644 --- a/BUILD +++ b/BUILD @@ -147,6 +147,7 @@ cc_library( "src/google/protobuf/message_lite.cc", "src/google/protobuf/parse_context.cc", "src/google/protobuf/repeated_field.cc", + "src/google/protobuf/repeated_ptr_field.cc", "src/google/protobuf/stubs/bytestream.cc", "src/google/protobuf/stubs/common.cc", "src/google/protobuf/stubs/int128.cc", diff --git a/cmake/extract_includes.bat.in b/cmake/extract_includes.bat.in index dd96c6ae31..605c5f966b 100644 --- a/cmake/extract_includes.bat.in +++ b/cmake/extract_includes.bat.in @@ -93,6 +93,7 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\port_undef.inc" inclu copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\reflection.h" include\google\protobuf\reflection.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\reflection_ops.h" include\google\protobuf\reflection_ops.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\repeated_field.h" include\google\protobuf\repeated_field.h +copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\repeated_ptr_field.h" include\google\protobuf\repeated_ptr_field.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\service.h" include\google\protobuf\service.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\source_context.pb.h" include\google\protobuf\source_context.pb.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\struct.pb.h" include\google\protobuf\struct.pb.h From 0da2ca5a1742421d7b9a04b997ec9961fe0fc25a Mon Sep 17 00:00:00 2001 From: Chris Povirk Date: Tue, 12 Oct 2021 16:11:17 -0400 Subject: [PATCH 26/52] Add jsr305 to protobuf-util deps. (#9059) This prepares for a change (internal CL 399474184, to be mirrored out to GitHub) to add a few such annotations in FieldMaskUtil and JsonFormat. (Technically, this PR is probably not "necessary" because protobuf-util already depends transitively on jsr305. But it's better hygiene to depend on it directly, and the direct dependency could protect against problems if protobuf-util drops some of its other deps -- or if those deps drop their own deps on jsr305.) --- java/util/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/java/util/pom.xml b/java/util/pom.xml index 31ccf5d567..3c955789ed 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -32,6 +32,11 @@ j2objc-annotations 1.3 + + com.google.code.findbugs + jsr305 + 3.0.2 + com.google.guava guava-testlib From 454f0cccaa22e3c4958e25b76cbd25a677294cc1 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 12 Oct 2021 15:37:40 -0700 Subject: [PATCH 27/52] Add jsr305 dependency for Bazel --- WORKSPACE | 9 ++++++++- java/util/BUILD | 1 + maven_install.json | 22 +++------------------- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 2174cabf8c..cc2120ec93 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -38,13 +38,15 @@ bind( load("@rules_jvm_external//:defs.bzl", "maven_install") maven_install( artifacts = [ + "com.google.code.findbugs:jsr305:3.0.2", "com.google.code.gson:gson:2.8.6", "com.google.errorprone:error_prone_annotations:2.3.2", - "com.google.j2objc:j2obj_annotations:1.3", + "com.google.j2objc:j2objc-annotations:1.3", "com.google.guava:guava:30.1.1-jre", "com.google.truth:truth:1.1.2", "junit:junit:4.12", "org.easymock:easymock:3.2", + ], repositories = [ "https://repo1.maven.org/maven2", @@ -78,6 +80,11 @@ bind( actual = "@maven//:com_google_j2objc_j2objc_annotations", ) +bind( + name = "jsr305", + actual = "@maven//:com_google_code_findbugs_jsr305", +) + bind( name = "junit", actual = "@maven//:junit_junit", diff --git a/java/util/BUILD b/java/util/BUILD index 02e5549689..3855da96f0 100644 --- a/java/util/BUILD +++ b/java/util/BUILD @@ -14,6 +14,7 @@ java_library( "//external:error_prone_annotations", "//external:j2objc_annotations", "//external:gson", + "//external:jsr305", "//external:guava", "//java/core", "//java/lite", diff --git a/maven_install.json b/maven_install.json index f9342185c4..6168aa4af2 100644 --- a/maven_install.json +++ b/maven_install.json @@ -1,6 +1,8 @@ { "dependency_tree": { - "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": 1033791982, + "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL", + "__INPUT_ARTIFACTS_HASH": 1907885757, + "__RESOLVED_ARTIFACTS_HASH": 375457873, "conflict_resolution": { "com.google.errorprone:error_prone_annotations:2.3.2": "com.google.errorprone:error_prone_annotations:2.5.1", "junit:junit:4.12": "junit:junit:4.13.1" @@ -201,24 +203,6 @@ "sha256": "b3dd1cf5019f942d8cc2afad0aa6aef4b21532446fe90a6b68d567e3389763dd", "url": "https://repo1.maven.org/maven2/org/easymock/easymock/3.2/easymock-3.2.jar" }, - { - "coord": "org.easymock:easymockclassextension:3.2", - "dependencies": [ - "org.easymock:easymock:3.2", - "cglib:cglib-nodep:2.2.2", - "org.objenesis:objenesis:1.3" - ], - "directDependencies": [ - "org.easymock:easymock:3.2" - ], - "file": "v1/https/repo1.maven.org/maven2/org/easymock/easymockclassextension/3.2/easymockclassextension-3.2.jar", - "mirror_urls": [ - "https://repo1.maven.org/maven2/org/easymock/easymockclassextension/3.2/easymockclassextension-3.2.jar", - "https://repo.maven.apache.org/maven2/org/easymock/easymockclassextension/3.2/easymockclassextension-3.2.jar" - ], - "sha256": "e2aeb3ecec87d859b2f3072985d4b15873558bcf6410f422db0c0c5194c76c87", - "url": "https://repo1.maven.org/maven2/org/easymock/easymockclassextension/3.2/easymockclassextension-3.2.jar" - }, { "coord": "org.hamcrest:hamcrest-core:1.3", "dependencies": [], From 9aa1adc60c52c8682c21ad1fc98afa1d3b805563 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 12 Oct 2021 15:43:34 -0700 Subject: [PATCH 28/52] Removed unused references to easymock_classextension --- WORKSPACE | 5 ----- java/core/BUILD | 1 - 2 files changed, 6 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index cc2120ec93..c88d242db2 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -95,11 +95,6 @@ bind( actual = "@maven//:org_easymock_easymock", ) -bind( - name = "easymock_classextension", - actual = "@maven//:org_easymock_easymockclassextension", -) - bind( name = "truth", actual = "@maven//:com_google_truth_truth", diff --git a/java/core/BUILD b/java/core/BUILD index 42124bb4f0..c65f10a4e1 100644 --- a/java/core/BUILD +++ b/java/core/BUILD @@ -255,7 +255,6 @@ junit_tests( ":java_test_protos_java_proto", ":test_util", "//external:easymock", - "//external:easymock_classextension", "//external:guava", "//external:junit", "//external:truth", From 8171716ae9ffd87f26a98a77d9811f4654646f73 Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 13 Oct 2021 00:14:05 +0100 Subject: [PATCH 29/52] Added "object" as a reserved name for PHP (#8962) * Added "object" as a reserved name for PHP * Fixed spacing --- php/ext/google/protobuf/names.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/php/ext/google/protobuf/names.c b/php/ext/google/protobuf/names.c index 6108ff4a0d..a99188800d 100644 --- a/php/ext/google/protobuf/names.c +++ b/php/ext/google/protobuf/names.c @@ -71,22 +71,22 @@ static void stringsink_uninit(stringsink *sink) { free(sink->ptr); } /* def name -> classname ******************************************************/ const char *const kReservedNames[] = { - "abstract", "and", "array", "as", "break", - "callable", "case", "catch", "class", "clone", - "const", "continue", "declare", "default", "die", - "do", "echo", "else", "elseif", "empty", - "enddeclare", "endfor", "endforeach", "endif", "endswitch", - "endwhile", "eval", "exit", "extends", "final", - "finally", "fn", "for", "foreach", "function", - "if", "implements", "include", "include_once", "instanceof", - "global", "goto", "insteadof", "interface", "isset", - "list", "match", "namespace", "new", "or", - "print", "private", "protected", "public", "require", - "require_once", "return", "static", "switch", "throw", - "trait", "try", "unset", "use", "var", - "while", "xor", "yield", "int", "float", - "bool", "string", "true", "false", "null", - "void", "iterable", NULL}; + "abstract", "and", "array", "as", "break", + "callable", "case", "catch", "class", "clone", + "const", "continue", "declare", "default", "die", + "do", "echo", "else", "elseif", "empty", + "enddeclare", "endfor", "endforeach", "endif", "endswitch", + "endwhile", "eval", "exit", "extends", "final", + "finally", "fn", "for", "foreach", "function", + "if", "implements", "include", "include_once", "instanceof", + "global", "goto", "insteadof", "interface", "isset", + "list", "match", "namespace", "new", "object", + "or", "print", "private", "protected", "public", + "require", "require_once", "return", "static", "switch", + "throw", "trait", "try", "unset", "use", + "var", "while", "xor", "yield", "int", + "float", "bool", "string", "true", "false", + "null", "void", "iterable", NULL}; bool is_reserved_name(const char* name) { int i; From 39013ab238a152b1794080b369f8c6648ab8104b Mon Sep 17 00:00:00 2001 From: Postmodern Date: Tue, 12 Oct 2021 17:03:24 -0700 Subject: [PATCH 30/52] Remove unused rubygems-tasks development dependency (#8824) * Cannot find any occurrence of `require 'rubygems/tasks` or `Gem::Tasks` in the source code, so I am confident that this development dependency can safely be removed. --- ruby/google-protobuf.gemspec | 1 - 1 file changed, 1 deletion(-) diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index 7a9e3f38b7..b4ae37bb3c 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -25,5 +25,4 @@ Gem::Specification.new do |s| s.required_ruby_version = '>= 2.3' s.add_development_dependency "rake-compiler", "~> 1.1.0" s.add_development_dependency "test-unit", '~> 3.0', '>= 3.0.9' - s.add_development_dependency "rubygems-tasks", "~> 0.2.4" end From c69b90cb066d5ee1869107aa18448820176d8eeb Mon Sep 17 00:00:00 2001 From: Elliotte Rusty Harold Date: Wed, 13 Oct 2021 11:32:48 +0000 Subject: [PATCH 31/52] deps: update maven-antrun-plugin fixes #8704 --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index 9864f73cc3..e39536f920 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -158,7 +158,7 @@ maven-antrun-plugin - 1.8 + 3.0.0 org.codehaus.mojo From e47ed057a587f401889cabec7780129869d6091d Mon Sep 17 00:00:00 2001 From: Raul Bocanegra Algarra Date: Wed, 13 Oct 2021 18:13:18 +0200 Subject: [PATCH 32/52] Make glob recursive if option is enabled (#8783) --- .../protobuf_distutils/generate_py_protobufs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/protobuf_distutils/protobuf_distutils/generate_py_protobufs.py b/python/protobuf_distutils/protobuf_distutils/generate_py_protobufs.py index 515ded2334..88b8d45e62 100644 --- a/python/protobuf_distutils/protobuf_distutils/generate_py_protobufs.py +++ b/python/protobuf_distutils/protobuf_distutils/generate_py_protobufs.py @@ -120,7 +120,7 @@ class generate_py_protobufs(Command): if self.proto_files is None: files = glob.glob(os.path.join(self.source_dir, '*.proto')) if self.recurse: - files.extend(glob.glob(os.path.join(self.source_dir, '**', '*.proto'))) + files.extend(glob.glob(os.path.join(self.source_dir, '**', '*.proto'), recursive=True)) self.proto_files = [f.partition(self.proto_root_path + os.path.sep)[-1] for f in files] if not self.proto_files: raise DistutilsOptionError('no .proto files were found under ' + self.source_dir) From ab993cf6ca5fecc743b2f3d0b96b69708977982b Mon Sep 17 00:00:00 2001 From: Elliotte Rusty Harold Date: Wed, 13 Oct 2021 16:36:21 +0000 Subject: [PATCH 33/52] document the kinds of patches the repo is open to (#8900) * document the kinds of patches the repo is open to --- CONTRIBUTING.md | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index db1ff31cac..8ef5dd29c8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,9 +1,31 @@ # Contributing to Protocol Buffers -We welcome your contributions to protocol buffers. This doc describes the +We welcome some types of contributions to protocol buffers. This doc describes the process to contribute patches to protobuf and the general guidelines we expect contributors to follow. +## What We Accept + +* Bug fixes with unit tests demonstrating the problem are very welcome. + We also appreciate bug reports, even when they don't come with a patch. + Bug fixes without tests are usually not accepted. +* New APIs and features with adequate test coverage and documentation + may be accepted if they do not compromise backwards + compatibility. However there's a fairly high bar of usefulness a new public + method must clear before it will be accepted. Features that are fine in + isolation are often rejected because they don't have enough impact to justify the + conceptual burden and ongoing maintenance cost. It's best to file an issue + and get agreement from maintainers on the value of a new feature before + working on a PR. +* Performance optimizations may be accepted if they have convincing benchmarks that demonstrate + an improvement and they do not significantly increase complexity. +* Changes to existing APIs are almost never accepted. Stability and + backwards compatibility are paramount. In the unlikely event a breaking change + is required, it must usually be implemented in google3 first. +* Changes to the wire and text formats are never accepted. Any breaking change + to these formats would have to be implemented as a completely new format. + We cannot begin generating protos that cannot be parsed by existing code. + ## Before You Start We accept patches in the form of github pull requests. If you are new to @@ -58,7 +80,7 @@ the final release. * Create small PRs that are narrowly focused on addressing a single concern. We often receive PRs that are trying to fix several things at a time, but if only one fix is considered acceptable, nothing gets merged and both author's - & review's time is wasted. Create more PRs to address different concerns and + & reviewer's time is wasted. Create more PRs to address different concerns and everyone will be happy. * For speculative changes, consider opening an issue and discussing it first. If you are suggesting a behavioral or API change, make sure you get explicit From 1ab7789f384472b84872a885d998a7a7ef411acc Mon Sep 17 00:00:00 2001 From: Hong Xu Date: Wed, 13 Oct 2021 10:26:40 -0700 Subject: [PATCH 34/52] Emacs: Protobuf mode should be derived from prog-mode (#9076) Prog mode is a basic major mode for buffers containing programming language source code: https://www.gnu.org/software/emacs/manual/html_node/elisp/Basic-Major-Modes.html A lot of programming mode setup is based on whether the major mode is derived from `prog-mode`. --- editors/protobuf-mode.el | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/editors/protobuf-mode.el b/editors/protobuf-mode.el index aa31bd0c44..bbb82b7f4b 100644 --- a/editors/protobuf-mode.el +++ b/editors/protobuf-mode.el @@ -193,7 +193,7 @@ ;;;###autoload (add-to-list 'auto-mode-alist '("\\.proto\\'" . protobuf-mode)) ;;;###autoload -(defun protobuf-mode () +(define-derived-mode protobuf-mode prog-mode "Protobuf" "Major mode for editing Protocol Buffers description language. The hook `c-mode-common-hook' is run with no argument at mode From dd3a6486ccfe0bd1d5d8d589fe05a3d985a79a02 Mon Sep 17 00:00:00 2001 From: Elliotte Rusty Harold Date: Wed, 13 Oct 2021 17:34:49 +0000 Subject: [PATCH 35/52] Update versions in README files (#9093) * update version numbers in java/README.md * update lite too --- update_version.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/update_version.py b/update_version.py index b96f715f91..2c2b4890eb 100755 --- a/update_version.py +++ b/update_version.py @@ -241,6 +241,24 @@ def UpdateJava(): RewriteXml('protoc-artifacts/pom.xml', lambda document : ReplaceText( Find(document.documentElement, 'version'), GetFullVersion())) + + RewriteTextFile('java/README.md', + lambda line : re.sub( + r'.*', + '%s' % GetFullVersion(), + line)) + + RewriteTextFile('java/README.md', + lambda line : re.sub( + r'implementation \'com.google.protobuf:protobuf-java:.*\'', + 'implementation \'com.google.protobuf:protobuf-java:%s\'' % GetFullVersion(), + line)) + + RewriteTextFile('java/lite.md', + lambda line : re.sub( + r'.*', + '%s' % GetFullVersion(), + line)) def UpdateJavaScript(): From 3881f49ce9b3c736aaeca95be3e2018361091a98 Mon Sep 17 00:00:00 2001 From: miyucy Date: Thu, 14 Oct 2021 04:54:49 +0900 Subject: [PATCH 36/52] Add size to Map class (#8068) --- ruby/ext/google/protobuf_c/map.c | 1 + .../java/com/google/protobuf/jruby/RubyMap.java | 2 +- ruby/tests/basic.rb | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/ruby/ext/google/protobuf_c/map.c b/ruby/ext/google/protobuf_c/map.c index d5b47e4e18..d1a648daec 100644 --- a/ruby/ext/google/protobuf_c/map.c +++ b/ruby/ext/google/protobuf_c/map.c @@ -680,6 +680,7 @@ void Map_register(VALUE module) { rb_define_method(klass, "delete", Map_delete, 1); rb_define_method(klass, "clear", Map_clear, 0); rb_define_method(klass, "length", Map_length, 0); + rb_define_method(klass, "size", Map_length, 0); rb_define_method(klass, "dup", Map_dup, 0); rb_define_method(klass, "==", Map_eq, 1); rb_define_method(klass, "freeze", Map_freeze, 0); diff --git a/ruby/src/main/java/com/google/protobuf/jruby/RubyMap.java b/ruby/src/main/java/com/google/protobuf/jruby/RubyMap.java index 8140ec5b64..f7379b148e 100644 --- a/ruby/src/main/java/com/google/protobuf/jruby/RubyMap.java +++ b/ruby/src/main/java/com/google/protobuf/jruby/RubyMap.java @@ -332,7 +332,7 @@ public class RubyMap extends RubyObject { * * Returns the number of entries (key-value pairs) in the map. */ - @JRubyMethod + @JRubyMethod(name = {"length", "size"}) public IRubyObject length(ThreadContext context) { return context.runtime.newFixnum(this.table.size()); } diff --git a/ruby/tests/basic.rb b/ruby/tests/basic.rb index 2a7a251f2e..841b8b502f 100755 --- a/ruby/tests/basic.rb +++ b/ruby/tests/basic.rb @@ -620,5 +620,21 @@ module BasicTest assert_raise(FrozenErrorType) { m.map_string_int32.delete('a') } assert_raise(FrozenErrorType) { m.map_string_int32.clear } end + + def test_map_length + m = proto_module::MapMessage.new + assert_equal 0, m.map_string_int32.length + assert_equal 0, m.map_string_msg.length + assert_equal 0, m.map_string_int32.size + assert_equal 0, m.map_string_msg.size + + m.map_string_int32['a'] = 1 + m.map_string_int32['b'] = 2 + m.map_string_msg['a'] = proto_module::TestMessage2.new + assert_equal 2, m.map_string_int32.length + assert_equal 1, m.map_string_msg.length + assert_equal 2, m.map_string_int32.size + assert_equal 1, m.map_string_msg.size + end end end From f8a22b45c04162b117204bb7c1211e1e4e13f19d Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Wed, 13 Oct 2021 13:51:09 -0700 Subject: [PATCH 37/52] Remove references to six (#9096) We no longer support Python 2 and therefore no longer need to depend on six. I'm hoping this will make it possible to merge #8214. --- BUILD | 1 - Makefile.am | 1 - .../python_run_tests_with_qemu_aarch64.sh | 2 +- .../python/windows/build_artifacts.bat | 1 - protobuf_deps.bzl | 8 -------- third_party/BUILD | 2 +- third_party/six.BUILD | 19 ------------------- 7 files changed, 2 insertions(+), 32 deletions(-) delete mode 100644 third_party/six.BUILD diff --git a/BUILD b/BUILD index a9ed429f10..39ea104566 100644 --- a/BUILD +++ b/BUILD @@ -954,7 +954,6 @@ py_proto_library( protoc = ":protoc", py_libs = [ ":python_srcs", - "@six//:six", ], srcs_version = "PY2AND3", visibility = ["//visibility:public"], diff --git a/Makefile.am b/Makefile.am index 6fb1c84fc5..f37ae9fdae 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1426,7 +1426,6 @@ EXTRA_DIST = $(@DIST_LANG@_EXTRA_DIST) \ examples/pubspec.yaml \ protobuf.bzl \ protobuf_deps.bzl \ - third_party/six.BUILD \ third_party/zlib.BUILD \ util/python/BUILD \ internal.bzl diff --git a/kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh b/kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh index 527fc4849d..5026d0448f 100755 --- a/kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh +++ b/kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh @@ -7,7 +7,7 @@ cd $(dirname $0)/../../.. cd python PYTHON="/opt/python/cp38-cp38/bin/python" -${PYTHON} -m pip install --user six pytest auditwheel +${PYTHON} -m pip install --user pytest auditwheel # check that we are really using aarch64 python (${PYTHON} -c 'import sysconfig; print(sysconfig.get_platform())' | grep -q "linux-aarch64") || (echo "Wrong python platform, needs to be aarch64 python."; exit 1) diff --git a/kokoro/release/python/windows/build_artifacts.bat b/kokoro/release/python/windows/build_artifacts.bat index 5c5df7c21c..340bda8dd9 100644 --- a/kokoro/release/python/windows/build_artifacts.bat +++ b/kokoro/release/python/windows/build_artifacts.bat @@ -9,7 +9,6 @@ set PACKAGE_NAME=protobuf set REPO_DIR=protobuf set BUILD_DLL=OFF set UNICODE=ON -set PB_TEST_DEP="six==1.9" set OTHER_TEST_DEP="setuptools==38.5.1" set OLD_PATH=C:\Program Files (x86)\MSBuild\14.0\bin\;%PATH% diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index ec9d8e9362..422ee0629e 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -24,14 +24,6 @@ def protobuf_deps(): urls = ["https://github.com/madler/zlib/archive/v1.2.11.tar.gz"], ) - if not native.existing_rule("six"): - http_archive( - name = "six", - build_file = "@com_google_protobuf//:third_party/six.BUILD", - sha256 = "d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73", - urls = ["https://pypi.python.org/packages/source/s/six/six-1.12.0.tar.gz"], - ) - if not native.existing_rule("rules_cc"): http_archive( name = "rules_cc", diff --git a/third_party/BUILD b/third_party/BUILD index b66101a39a..a8b35efceb 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -1 +1 @@ -exports_files(["six.BUILD", "zlib.BUILD"]) +exports_files(["zlib.BUILD"]) diff --git a/third_party/six.BUILD b/third_party/six.BUILD deleted file mode 100644 index 041c72c66e..0000000000 --- a/third_party/six.BUILD +++ /dev/null @@ -1,19 +0,0 @@ -load("@rules_python//python:defs.bzl", "py_library") - -# Consume `six.py` as `__init__.py` for compatibility -# with `--incompatible_default_to_explicit_init_py`. -# https://github.com/protocolbuffers/protobuf/pull/6795#issuecomment-546060749 -# https://github.com/bazelbuild/bazel/issues/10076 -genrule( - name = "copy_six", - srcs = ["six-1.12.0/six.py"], - outs = ["__init__.py"], - cmd = "cp $< $(@)", -) - -py_library( - name = "six", - srcs = ["__init__.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], -) From 9eba6eddcebaea2c806e362748a68ce3b8e624a3 Mon Sep 17 00:00:00 2001 From: Dirk Boye Date: Wed, 13 Oct 2021 22:54:54 +0200 Subject: [PATCH 38/52] update rules_python dependency to version 0.1.0 (#8214) other bazel libraries (e.g. rules_docker 0.15.0) require rules_python 0.1.0 or above. running protobuf_deps() before importing rules_docker will lead to errors. upgrading rules_python fixes this problem. --- protobuf_deps.bzl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 422ee0629e..7420bc1fd7 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -51,9 +51,8 @@ def protobuf_deps(): if not native.existing_rule("rules_python"): http_archive( name = "rules_python", - sha256 = "e5470e92a18aa51830db99a4d9c492cc613761d5bdb7131c04bd92b9834380f6", - strip_prefix = "rules_python-4b84ad270387a7c439ebdccfd530e2339601ef27", - urls = ["https://github.com/bazelbuild/rules_python/archive/4b84ad270387a7c439ebdccfd530e2339601ef27.tar.gz"], + sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0", + urls = ["https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz"], ) if not native.existing_rule("rules_jvm_external"): From 40e9cedf7a738afa16557a5e7d401d6217e5134b Mon Sep 17 00:00:00 2001 From: Elliotte Rusty Harold Date: Wed, 13 Oct 2021 21:12:10 +0000 Subject: [PATCH 39/52] JDK 6 is too old to care about (#9097) @cpovirk --- kokoro/linux/64-bit/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/kokoro/linux/64-bit/Dockerfile b/kokoro/linux/64-bit/Dockerfile index 3a279e6602..189d0cf941 100644 --- a/kokoro/linux/64-bit/Dockerfile +++ b/kokoro/linux/64-bit/Dockerfile @@ -60,8 +60,6 @@ RUN apt-get clean && apt-get update && apt-get install -y --force-yes \ nunit \ # -- For all Java builds -- \ maven \ - # -- For java_jdk6 -- \ - # oops! not in jessie. too old? openjdk-6-jdk \ # -- For java_jdk7 -- \ openjdk-7-jdk \ # -- For java_oracle7 -- \ From 255dec16d66854b1d77d37672b043840f40f07c5 Mon Sep 17 00:00:00 2001 From: Justin Paupore Date: Wed, 13 Oct 2021 14:19:42 -0700 Subject: [PATCH 40/52] Add Android stlport and default toolchains to BUILD. (#8290) These are additional possibilities for --crosstool_top for Android NDK compilation. Since the NDK doesn't have -lpthread, getting these flags wrong leads to a linker error. Fixes: 180084220 --- BUILD | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/BUILD b/BUILD index 39ea104566..389882b06d 100644 --- a/BUILD +++ b/BUILD @@ -65,6 +65,13 @@ create_compiler_config_setting( ], ) +# Android NDK builds can specify different crosstool_top flags to choose which +# STL they use for C++. We need these multiple variants to catch all of those +# versions of crosstool_top and reliably detect Android. +# +# For more info on the various crosstool_tops used by NDK Bazel builds, see: +# https://docs.bazel.build/versions/master/android-ndk.html#configuring-the-stl + config_setting( name = "android", values = { @@ -76,6 +83,17 @@ config_setting( ], ) +config_setting( + name = "android-stlport", + values = { + "crosstool_top": "@androidndk//:toolchain-stlport", + }, + visibility = [ + # Public, but Protobuf only visibility. + "//:__subpackages__", + ], +) + config_setting( name = "android-libcpp", values = { @@ -98,11 +116,24 @@ config_setting( ], ) +config_setting( + name = "android-default", + values = { + "crosstool_top": "@androidndk//:default_crosstool", + }, + visibility = [ + # Public, but Protobuf only visibility. + "//:__subpackages__", + ], +) + # Android and MSVC builds do not need to link in a separate pthread library. LINK_OPTS = select({ ":android": [], + ":android-stlport": [], ":android-libcpp": [], ":android-gnu-libstdcpp": [], + ":android-default": [], ":msvc": [ # Suppress linker warnings about files with no symbols defined. "-ignore:4221", From 25180ac9b6e7412aa3a4e6a87113d11b61934ae2 Mon Sep 17 00:00:00 2001 From: Shigeo Hashimoto Date: Fri, 15 Oct 2021 01:45:40 +0900 Subject: [PATCH 41/52] Fix build failed for visual studio in multi-byte windows environments (#7235) * Set source and executable charset to utf-8 when Visual Studio * Remove unnecessary version check for visual studio --- cmake/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index caa92150c7..94dc810cce 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -209,6 +209,8 @@ if (MSVC) # Build with multiple processes add_definitions(/MP) endif() + # Set source file and execution character sets to UTF-8 + add_definitions(/utf-8) # MSVC warning suppressions add_definitions( /wd4018 # 'expression' : signed/unsigned mismatch From 4e176f29b4151ea1763cff395c7536a681b6bdad Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Tue, 12 May 2020 09:13:33 -0400 Subject: [PATCH 42/52] Added GetDebugString to Python FileDescriptor interface. --- .../protobuf/internal/descriptor_test.py | 25 +++++++++++++++++++ python/google/protobuf/pyext/descriptor.cc | 5 ++++ 2 files changed, 30 insertions(+) diff --git a/python/google/protobuf/internal/descriptor_test.py b/python/google/protobuf/internal/descriptor_test.py index 88d7136bc8..169f4f777d 100644 --- a/python/google/protobuf/internal/descriptor_test.py +++ b/python/google/protobuf/internal/descriptor_test.py @@ -51,6 +51,28 @@ TEST_EMPTY_MESSAGE_DESCRIPTOR_ASCII = """ name: 'TestEmptyMessage' """ +TEST_FILE_DESCRIPTOR_DEBUG = """syntax = "proto2"; + +package protobuf_unittest; + +message NestedMessage { + enum ForeignEnum { + FOREIGN_FOO = 4; + FOREIGN_BAR = 5; + FOREIGN_BAZ = 6; + } + optional int32 bb = 1; +} + +message ResponseMessage { +} + +service Service { + rpc CallMethod(.protobuf_unittest.NestedMessage) returns (.protobuf_unittest.ResponseMessage); +} + +""" + warnings.simplefilter('error', DeprecationWarning) @@ -121,6 +143,9 @@ class DescriptorTest(unittest.TestCase): def testContainingServiceFixups(self): self.assertEqual(self.my_service, self.my_method.containing_service) + def testGetDebugString(self): + self.assertEqual(self.my_file.GetDebugString(), TEST_FILE_DESCRIPTOR_DEBUG) + def testGetOptions(self): self.assertEqual(self.my_enum.GetOptions(), descriptor_pb2.EnumOptions()) diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc index 9708b84013..bc49e91b92 100644 --- a/python/google/protobuf/pyext/descriptor.cc +++ b/python/google/protobuf/pyext/descriptor.cc @@ -1393,6 +1393,10 @@ static int SetHasOptions(PyFileDescriptor *self, PyObject *value, return CheckCalledFromGeneratedFile("has_options"); } +static PyObject* GetDebugString(PyFileDescriptor *self) { + return PyString_FromCppString(_GetDescriptor(self)->DebugString()); +} + static PyObject* GetOptions(PyFileDescriptor *self) { return GetOrBuildOptions(_GetDescriptor(self)); } @@ -1439,6 +1443,7 @@ static PyGetSetDef Getters[] = { }; static PyMethodDef Methods[] = { + { "GetDebugString", (PyCFunction)GetDebugString, METH_NOARGS, }, { "GetOptions", (PyCFunction)GetOptions, METH_NOARGS, }, { "CopyToProto", (PyCFunction)CopyToProto, METH_O, }, {NULL} From 5100be2b7746391c2724e2793e1428c36b63c98b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Mon, 16 Dec 2019 16:21:41 +0100 Subject: [PATCH 43/52] Prevent integer overflow for unknown fields The PyInt_FromLong() conversion function will truncate 64 bit integer values on 32 bit architectures. We will now use the PyLong_* functions with the appropriate minimum size for each field. Note that this commit also switches to the unsigned versions, since the unknown integer fields have been declared unsigned anyway. Fixes #6205 --- python/google/protobuf/pyext/unknown_fields.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/google/protobuf/pyext/unknown_fields.cc b/python/google/protobuf/pyext/unknown_fields.cc index 7f4fb23edf..5dbcd88568 100644 --- a/python/google/protobuf/pyext/unknown_fields.cc +++ b/python/google/protobuf/pyext/unknown_fields.cc @@ -274,13 +274,13 @@ static PyObject* GetData(PyUnknownFieldRef* self, void *closure) { PyObject* data = NULL; switch (field->type()) { case UnknownField::TYPE_VARINT: - data = PyLong_FromLong(field->varint()); + data = PyLong_FromUnsignedLongLong(field->varint()); break; case UnknownField::TYPE_FIXED32: - data = PyLong_FromLong(field->fixed32()); + data = PyLong_FromUnsignedLong(field->fixed32()); break; case UnknownField::TYPE_FIXED64: - data = PyLong_FromLong(field->fixed64()); + data = PyLong_FromUnsignedLongLong(field->fixed64()); break; case UnknownField::TYPE_LENGTH_DELIMITED: data = PyBytes_FromStringAndSize(field->length_delimited().data(), From 3e02f65f5cc6049e1593b7e99f8abe5fc227c994 Mon Sep 17 00:00:00 2001 From: Daniel Kuschny Date: Fri, 15 Oct 2021 00:02:21 +0200 Subject: [PATCH 44/52] Skip exports if not available by CommonJS (#8856) --- js/commonjs/export.js | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/js/commonjs/export.js b/js/commonjs/export.js index a9932e95ae..2025d9a3d2 100644 --- a/js/commonjs/export.js +++ b/js/commonjs/export.js @@ -15,15 +15,17 @@ goog.require('jspb.ExtensionFieldInfo'); goog.require('jspb.Message'); goog.require('jspb.Map'); -exports.Map = jspb.Map; -exports.Message = jspb.Message; -exports.BinaryReader = jspb.BinaryReader; -exports.BinaryWriter = jspb.BinaryWriter; -exports.ExtensionFieldInfo = jspb.ExtensionFieldInfo; -exports.ExtensionFieldBinaryInfo = jspb.ExtensionFieldBinaryInfo; +if ( typeof exports === 'object' ) { + exports.Map = jspb.Map; + exports.Message = jspb.Message; + exports.BinaryReader = jspb.BinaryReader; + exports.BinaryWriter = jspb.BinaryWriter; + exports.ExtensionFieldInfo = jspb.ExtensionFieldInfo; + exports.ExtensionFieldBinaryInfo = jspb.ExtensionFieldBinaryInfo; -// These are used by generated code but should not be used directly by clients. -exports.exportSymbol = goog.exportSymbol; -exports.inherits = goog.inherits; -exports.object = {extend: goog.object.extend}; -exports.typeOf = goog.typeOf; + // These are used by generated code but should not be used directly by clients. + exports.exportSymbol = goog.exportSymbol; + exports.inherits = goog.inherits; + exports.object = {extend: goog.object.extend}; + exports.typeOf = goog.typeOf; +} \ No newline at end of file From 6bc21b531e6d4d5166d0be04acff37a2849e4a34 Mon Sep 17 00:00:00 2001 From: Marnix Bouhuis Date: Fri, 15 Oct 2021 00:15:20 +0200 Subject: [PATCH 45/52] Update the way we get the global object, to comply with CSP no-unsafe-eval (#8864) --- src/google/protobuf/compiler/js/js_generator.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/google/protobuf/compiler/js/js_generator.cc b/src/google/protobuf/compiler/js/js_generator.cc index cfd0e037c9..5542256f38 100644 --- a/src/google/protobuf/compiler/js/js_generator.cc +++ b/src/google/protobuf/compiler/js/js_generator.cc @@ -3625,7 +3625,16 @@ void Generator::GenerateFile(const GeneratorOptions& options, if (options.import_style == GeneratorOptions::kImportCommonJsStrict) { printer->Print("var proto = {};\n\n"); } else { - printer->Print("var global = Function('return this')();\n\n"); + // To get the global object we call a function with .call(null), this will set "this" inside the + // function to the global object. + // This does not work if we are running in strict mode ("use strict"), + // so we fallback to the following things (in order from first to last): + // - window: defined in browsers + // - global: defined in most server side environments like NodeJS + // - self: defined inside Web Workers (WorkerGlobalScope) + // - Function('return this')(): this will work on most platforms, but it may be blocked by things like CSP. + // Function('') is almost the same as eval('') + printer->Print("var global = (function() { return this || window || global || self || Function('return this')(); }).call(null);\n\n"); } for (int i = 0; i < file->dependency_count(); i++) { From 72f085747e157ba048ef5ebd11f49e54e6c2ab6e Mon Sep 17 00:00:00 2001 From: Jorg Brown Date: Thu, 14 Oct 2021 19:44:18 -0700 Subject: [PATCH 46/52] Use int32_t rather than int32. --- src/google/protobuf/compiler/objectivec/objectivec_helpers.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/google/protobuf/compiler/objectivec/objectivec_helpers.h b/src/google/protobuf/compiler/objectivec/objectivec_helpers.h index c5f948c84e..13a1052406 100644 --- a/src/google/protobuf/compiler/objectivec/objectivec_helpers.h +++ b/src/google/protobuf/compiler/objectivec/objectivec_helpers.h @@ -263,7 +263,7 @@ class PROTOC_EXPORT TextFormatDecodeData { TextFormatDecodeData(const TextFormatDecodeData&) = delete; TextFormatDecodeData& operator=(const TextFormatDecodeData&) = delete; - void AddString(int32 key, const std::string& input_for_decode, + void AddString(int32_t key, const std::string& input_for_decode, const std::string& desired_output); size_t num_entries() const { return entries_.size(); } std::string Data() const; @@ -272,7 +272,7 @@ class PROTOC_EXPORT TextFormatDecodeData { const std::string& desired_output); private: - typedef std::pair DataEntry; + typedef std::pair DataEntry; std::vector entries_; }; From ef0bd1343be03df69e976ce44853a3aebcc6d81f Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Fri, 15 Oct 2021 11:39:21 -0400 Subject: [PATCH 47/52] Only run GetDebugString test with cpp impl. --- python/google/protobuf/internal/descriptor_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/google/protobuf/internal/descriptor_test.py b/python/google/protobuf/internal/descriptor_test.py index 169f4f777d..057b6b01a7 100644 --- a/python/google/protobuf/internal/descriptor_test.py +++ b/python/google/protobuf/internal/descriptor_test.py @@ -143,6 +143,10 @@ class DescriptorTest(unittest.TestCase): def testContainingServiceFixups(self): self.assertEqual(self.my_service, self.my_method.containing_service) + @unittest.skipIf( + api_implementation.Type() != 'cpp', + 'GetDebugString is only available with the cpp implementation', + ) def testGetDebugString(self): self.assertEqual(self.my_file.GetDebugString(), TEST_FILE_DESCRIPTOR_DEBUG) From c01cd6ec794fd91b316cb236663c81eefa1efa4e Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Fri, 15 Oct 2021 11:24:49 -0600 Subject: [PATCH 48/52] Add python 3.10 (#9034) * Add python 3.10 * Update setup.py and tox.ini * fix: fix 3.9 -> 3.10 * fix: py310-cpp --- .../dockerfile/test/python310/Dockerfile | 31 +++++++++++++++++++ kokoro/linux/python310/build.sh | 18 +++++++++++ kokoro/linux/python310/continuous.cfg | 11 +++++++ kokoro/linux/python310/presubmit.cfg | 11 +++++++ kokoro/linux/python310_cpp/build.sh | 18 +++++++++++ kokoro/linux/python310_cpp/continuous.cfg | 11 +++++++ kokoro/linux/python310_cpp/presubmit.cfg | 11 +++++++ .../release/python/linux/build_artifacts.sh | 2 ++ .../release/python/macos/build_artifacts.sh | 1 + .../python/windows/build_artifacts.bat | 10 ++++++ .../python/windows/build_single_artifact.bat | 6 ++++ .../windows/install_python_interpreters.ps1 | 17 ++++++++++ python/setup.py | 5 +-- python/tox.ini | 4 +-- tests.sh | 9 ++++++ 15 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 kokoro/linux/dockerfile/test/python310/Dockerfile create mode 100755 kokoro/linux/python310/build.sh create mode 100644 kokoro/linux/python310/continuous.cfg create mode 100644 kokoro/linux/python310/presubmit.cfg create mode 100755 kokoro/linux/python310_cpp/build.sh create mode 100644 kokoro/linux/python310_cpp/continuous.cfg create mode 100644 kokoro/linux/python310_cpp/presubmit.cfg diff --git a/kokoro/linux/dockerfile/test/python310/Dockerfile b/kokoro/linux/dockerfile/test/python310/Dockerfile new file mode 100644 index 0000000000..e16e93b3b2 --- /dev/null +++ b/kokoro/linux/dockerfile/test/python310/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.10-buster + +# Install dependencies. We start with the basic ones require to build protoc +# and the C++ build +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + git \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + parallel \ + time \ + wget \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Install Python libraries. +RUN python -m pip install --no-cache-dir --upgrade \ + pip \ + setuptools \ + tox \ + wheel diff --git a/kokoro/linux/python310/build.sh b/kokoro/linux/python310/build.sh new file mode 100755 index 0000000000..0d8a2c9c6d --- /dev/null +++ b/kokoro/linux/python310/build.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERHUB_ORGANIZATION=protobuftesting +export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python310 +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="python310" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/python310/continuous.cfg b/kokoro/linux/python310/continuous.cfg new file mode 100644 index 0000000000..6ec74d8597 --- /dev/null +++ b/kokoro/linux/python310/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/python310/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/python310/presubmit.cfg b/kokoro/linux/python310/presubmit.cfg new file mode 100644 index 0000000000..6ec74d8597 --- /dev/null +++ b/kokoro/linux/python310/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/python310/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/python310_cpp/build.sh b/kokoro/linux/python310_cpp/build.sh new file mode 100755 index 0000000000..2903a2d9c2 --- /dev/null +++ b/kokoro/linux/python310_cpp/build.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERHUB_ORGANIZATION=protobuftesting +export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python310 +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="python310_cpp" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/python310_cpp/continuous.cfg b/kokoro/linux/python310_cpp/continuous.cfg new file mode 100644 index 0000000000..7ec844196e --- /dev/null +++ b/kokoro/linux/python310_cpp/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/python310_cpp/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/python310_cpp/presubmit.cfg b/kokoro/linux/python310_cpp/presubmit.cfg new file mode 100644 index 0000000000..7ec844196e --- /dev/null +++ b/kokoro/linux/python310_cpp/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/python310_cpp/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/release/python/linux/build_artifacts.sh b/kokoro/release/python/linux/build_artifacts.sh index 2407a302c7..31da562814 100755 --- a/kokoro/release/python/linux/build_artifacts.sh +++ b/kokoro/release/python/linux/build_artifacts.sh @@ -61,7 +61,9 @@ build_artifact_version 3.6 build_artifact_version 3.7 build_artifact_version 3.8 build_artifact_version 3.9 +build_artifact_version 3.10 build_crosscompiled_aarch64_artifact_version 3.7 build_crosscompiled_aarch64_artifact_version 3.8 build_crosscompiled_aarch64_artifact_version 3.9 +build_crosscompiled_aarch64_artifact_version 3.10 diff --git a/kokoro/release/python/macos/build_artifacts.sh b/kokoro/release/python/macos/build_artifacts.sh index fbbd650b71..75644539fa 100755 --- a/kokoro/release/python/macos/build_artifacts.sh +++ b/kokoro/release/python/macos/build_artifacts.sh @@ -55,3 +55,4 @@ build_artifact_version 3.6 build_artifact_version 3.7 build_artifact_version 3.8 build_artifact_version 3.9 +build_artifact_version 3.10 diff --git a/kokoro/release/python/windows/build_artifacts.bat b/kokoro/release/python/windows/build_artifacts.bat index 340bda8dd9..ac466a901d 100644 --- a/kokoro/release/python/windows/build_artifacts.bat +++ b/kokoro/release/python/windows/build_artifacts.bat @@ -73,6 +73,16 @@ SET PYTHON_VERSION=3.9 SET PYTHON_ARCH=64 CALL build_single_artifact.bat || goto :error +SET PYTHON=C:\python310_32bit +SET PYTHON_VERSION=3.10 +SET PYTHON_ARCH=32 +CALL build_single_artifact.bat || goto :error + +SET PYTHON=C:\python310 +SET PYTHON_VERSION=3.10 +SET PYTHON_ARCH=64 +CALL build_single_artifact.bat || goto :error + goto :EOF :error diff --git a/kokoro/release/python/windows/build_single_artifact.bat b/kokoro/release/python/windows/build_single_artifact.bat index 920f9c83a8..8d3cd0c9d8 100644 --- a/kokoro/release/python/windows/build_single_artifact.bat +++ b/kokoro/release/python/windows/build_single_artifact.bat @@ -24,6 +24,12 @@ if %PYTHON%==C:\python39_32bit set vcplatform=Win32 if %PYTHON%==C:\python39 set generator=Visual Studio 14 Win64 if %PYTHON%==C:\python39 set vcplatform=x64 +if %PYTHON%==C:\python310_32bit set generator=Visual Studio 14 +if %PYTHON%==C:\python310_32bit set vcplatform=Win32 + +if %PYTHON%==C:\python310 set generator=Visual Studio 14 Win64 +if %PYTHON%==C:\python310 set vcplatform=x64 + REM Prepend newly installed Python to the PATH of this build (this cannot be REM done from inside the powershell script as it would require to restart REM the parent CMD process). diff --git a/kokoro/release/python/windows/install_python_interpreters.ps1 b/kokoro/release/python/windows/install_python_interpreters.ps1 index b63259a829..f193eedeb0 100644 --- a/kokoro/release/python/windows/install_python_interpreters.ps1 +++ b/kokoro/release/python/windows/install_python_interpreters.ps1 @@ -95,3 +95,20 @@ $Python39x64Config = @{ PythonInstallerHash = "b61a33dc28f13b561452f3089c87eb63" } Install-Python @Python39x64Config + +# Python 3.10 +$Python39x86Config = @{ + PythonVersion = "3.10.0" + PythonInstaller = "python-3.10.0" + PythonInstallPath = "C:\python310_32bit" + PythonInstallerHash = "6de353f2f7422aa030d4ccc788ffa75e" +} +Install-Python @Python310x86Config + +$Python39x64Config = @{ + PythonVersion = "3.10.0" + PythonInstaller = "python-3.10.0-amd64" + PythonInstallPath = "C:\python310" + PythonInstallerHash = "39135519b044757f0a3b09d63612b0da" +} +Install-Python @Python310x64Config diff --git a/python/setup.py b/python/setup.py index ab42b6fe17..77d9cd1f6c 100755 --- a/python/setup.py +++ b/python/setup.py @@ -287,11 +287,12 @@ if __name__ == '__main__': classifiers=[ "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], namespace_packages=['google'], packages=find_packages( diff --git a/python/tox.ini b/python/tox.ini index 88dd842e82..7142b86f0f 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{35,36,37,38,39}-{cpp,python} + py{35,36,37,38,39,310}-{cpp,python} [testenv] usedevelop=true @@ -14,7 +14,7 @@ setenv = commands = python setup.py -q build_py python: python setup.py -q build - py{35,36,37,38,39}-cpp: python setup.py -q build --cpp_implementation --warnings_as_errors --compile_static_extension + py{35,36,37,38,39,310}-cpp: python setup.py -q build --cpp_implementation --warnings_as_errors --compile_static_extension python: python setup.py -q test -q cpp: python setup.py -q test -q --cpp_implementation python: python setup.py -q test_conformance diff --git a/tests.sh b/tests.sh index fcfb75f865..176bc5f2bc 100755 --- a/tests.sh +++ b/tests.sh @@ -375,6 +375,10 @@ build_python39() { build_python_version py39-python } +build_python310() { + build_python_version py310-python +} + build_python_cpp() { internal_build_cpp export LD_LIBRARY_PATH=../src/.libs # for Linux @@ -427,6 +431,11 @@ build_python39_cpp() { build_python_cpp_version py39-cpp } +build_python310_cpp() { + build_python_cpp_version py310-cpp +} + + build_ruby23() { internal_build_cpp # For conformance tests. cd ruby && bash travis-test.sh ruby-2.3.8 && cd .. From d049bce84405a26996979de1e9333c7d0fc7378a Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Fri, 15 Oct 2021 13:10:38 -0700 Subject: [PATCH 49/52] Remove references to HAVE_PTHREAD (#9100) This is based on @haberman's changes in #8257. Now that we're using std::mutex we no longer need to check whether pthreads are available, so this commit removes references to HAVE_PTHREAD. I left the autotools build alone, though, since we are likely to drop support for it soon anyway. --- BUILD | 2 -- Protobuf-C++.podspec | 3 --- cmake/CMakeLists.txt | 3 --- cmake/protobuf-module.cmake.in | 14 -------------- protobuf-lite.pc.in | 4 ++-- protobuf.pc.in | 4 ++-- src/google/protobuf/stubs/common.cc | 4 ---- 7 files changed, 4 insertions(+), 30 deletions(-) diff --git a/BUILD b/BUILD index 389882b06d..1690d42198 100644 --- a/BUILD +++ b/BUILD @@ -26,7 +26,6 @@ ZLIB_DEPS = ["@zlib//:zlib"] ################################################################################ MSVC_COPTS = [ - "/DHAVE_PTHREAD", "/wd4018", # -Wno-sign-compare "/wd4065", # switch statement contains 'default' but no 'case' labels "/wd4146", # unary minus operator applied to unsigned type, result still unsigned @@ -46,7 +45,6 @@ MSVC_COPTS = [ COPTS = select({ ":msvc": MSVC_COPTS, "//conditions:default": [ - "-DHAVE_PTHREAD", "-DHAVE_ZLIB", "-Wmissing-field-initializers", "-Woverloaded-virtual", diff --git a/Protobuf-C++.podspec b/Protobuf-C++.podspec index f285e58af8..9b8fae51a7 100644 --- a/Protobuf-C++.podspec +++ b/Protobuf-C++.podspec @@ -35,9 +35,6 @@ Pod::Spec.new do |s| # Do not let src/google/protobuf/stubs/time.h override system API 'USE_HEADERMAP' => 'NO', 'ALWAYS_SEARCH_USER_PATHS' => 'NO', - - # Configure tool is not being used for Xcode. When building, assume pthread is supported. - 'GCC_PREPROCESSOR_DEFINITIONS' => '"$(inherited)" "HAVE_PTHREAD=1"', } end diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 94dc810cce..51e8478f6e 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -127,9 +127,6 @@ if (protobuf_DISABLE_RTTI) endif() find_package(Threads REQUIRED) -if (CMAKE_USE_PTHREADS_INIT) - add_definitions(-DHAVE_PTHREAD) -endif (CMAKE_USE_PTHREADS_INIT) set(_protobuf_FIND_ZLIB) if (protobuf_WITH_ZLIB) diff --git a/cmake/protobuf-module.cmake.in b/cmake/protobuf-module.cmake.in index 810256e54c..09b9d29c22 100644 --- a/cmake/protobuf-module.cmake.in +++ b/cmake/protobuf-module.cmake.in @@ -110,16 +110,6 @@ function(_protobuf_find_libraries name filename) endif() endfunction() -# Internal function: find threads library -function(_protobuf_find_threads) - set(CMAKE_THREAD_PREFER_PTHREAD TRUE) - find_package(Threads) - if(Threads_FOUND) - list(APPEND PROTOBUF_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) - set(PROTOBUF_LIBRARIES "${PROTOBUF_LIBRARIES}" PARENT_SCOPE) - endif() -endfunction() - # # Main. # @@ -139,10 +129,6 @@ _protobuf_find_libraries(Protobuf_LITE protobuf-lite) # The Protobuf Protoc Library _protobuf_find_libraries(Protobuf_PROTOC protoc) -if(UNIX) - _protobuf_find_threads() -endif() - # Set the include directory get_target_property(Protobuf_INCLUDE_DIRS protobuf::libprotobuf INTERFACE_INCLUDE_DIRECTORIES) diff --git a/protobuf-lite.pc.in b/protobuf-lite.pc.in index 68a2bb455b..f92e4ad9d7 100644 --- a/protobuf-lite.pc.in +++ b/protobuf-lite.pc.in @@ -6,6 +6,6 @@ includedir=@includedir@ Name: Protocol Buffers Description: Google's Data Interchange Format Version: @VERSION@ -Libs: -L${libdir} -lprotobuf-lite @PTHREAD_LIBS@ -Cflags: -I${includedir} @PTHREAD_CFLAGS@ +Libs: -L${libdir} -lprotobuf-lite +Cflags: -I${includedir} Conflicts: protobuf diff --git a/protobuf.pc.in b/protobuf.pc.in index 055a9d0563..e9bef5d0fe 100644 --- a/protobuf.pc.in +++ b/protobuf.pc.in @@ -6,8 +6,8 @@ includedir=@includedir@ Name: Protocol Buffers Description: Google's Data Interchange Format Version: @VERSION@ -Libs: -L${libdir} -lprotobuf @PTHREAD_LIBS@ +Libs: -L${libdir} -lprotobuf Libs.private: @LIBS@ -Cflags: -I${includedir} @PTHREAD_CFLAGS@ +Cflags: -I${includedir} Conflicts: protobuf-lite diff --git a/src/google/protobuf/stubs/common.cc b/src/google/protobuf/stubs/common.cc index 9067818a5c..82d46531f2 100644 --- a/src/google/protobuf/stubs/common.cc +++ b/src/google/protobuf/stubs/common.cc @@ -44,10 +44,6 @@ #endif #include #define snprintf _snprintf // see comment in strutil.cc -#elif defined(HAVE_PTHREAD) -#include -#else -#error "No suitable threading library available." #endif #if defined(__ANDROID__) #include From 6b0a1c2407b1d1de6e701784dd9c27e4e58697bb Mon Sep 17 00:00:00 2001 From: Nils Date: Fri, 15 Oct 2021 21:12:01 +0100 Subject: [PATCH 50/52] Add prost and quick-protobuf to third_party.md (#9104) --- docs/third_party.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/third_party.md b/docs/third_party.md index 06b31b1d25..645ada81f4 100644 --- a/docs/third_party.md +++ b/docs/third_party.md @@ -97,7 +97,9 @@ These are projects we know about implementing Protocol Buffers for other program * Ruby: http://github.com/mozy/ruby-protocol-buffers * Ruby: https://github.com/bmizerany/beefcake/tree/master/lib/beefcake * Ruby: https://github.com/localshred/protobuf +* Rust: https://github.com/tokio-rs/prost * Rust: https://github.com/stepancheg/rust-protobuf/ +* Rust: https://github.com/tafia/quick-protobuf * Scala: http://github.com/jeffplaisance/scala-protobuf * Scala: http://code.google.com/p/protobuf-scala * Scala: https://github.com/SandroGrzicic/ScalaBuff From f367bfb13d9ff5f097666bcdf7389aa8cef902b7 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Fri, 15 Oct 2021 14:26:54 -0700 Subject: [PATCH 51/52] Use dedicated helper functions for map descriptors (#9099) The Descriptor class now has map_key() and map_value() methods for accessing the special key and value fields in generated map entry messages. This commit updates all the relevant code to use these accessors instead of the clunkier FindFieldByName("key") or FindFieldByName("value") approach. --- python/google/protobuf/pyext/message.cc | 4 +- .../protobuf/compiler/cpp/cpp_map_field.cc | 8 ++-- .../protobuf/compiler/cpp/cpp_message.cc | 4 +- .../cpp/cpp_parse_function_generator.cc | 2 +- .../compiler/csharp/csharp_map_field.cc | 4 +- .../protobuf/compiler/java/java_helpers.cc | 2 +- .../protobuf/compiler/java/java_map_field.cc | 4 +- .../compiler/java/java_map_field_lite.cc | 4 +- .../protobuf/compiler/java/java_message.cc | 2 +- .../compiler/java/java_message_builder.cc | 2 +- .../objectivec/objectivec_map_field.cc | 6 +-- .../protobuf/compiler/php/php_generator.cc | 8 ++-- .../protobuf/generated_message_reflection.cc | 4 +- src/google/protobuf/map_field.h | 4 +- src/google/protobuf/map_test.inc | 48 +++++++++---------- .../util/message_differencer_unittest.cc | 2 +- 16 files changed, 54 insertions(+), 54 deletions(-) diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index cb48faa440..097d6bf9e8 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -1074,7 +1074,7 @@ int InitAttributes(CMessage* self, PyObject* args, PyObject* kwargs) { if (descriptor->is_map()) { ScopedPyObjectPtr map(GetFieldValue(self, descriptor)); const FieldDescriptor* value_descriptor = - descriptor->message_type()->FindFieldByName("value"); + descriptor->message_type()->map_value(); if (value_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { ScopedPyObjectPtr iter(PyObject_GetIter(value)); if (iter == NULL) { @@ -2582,7 +2582,7 @@ PyObject* GetFieldValue(CMessage* self, ContainerBase* py_container = nullptr; if (field_descriptor->is_map()) { const Descriptor* entry_type = field_descriptor->message_type(); - const FieldDescriptor* value_type = entry_type->FindFieldByName("value"); + const FieldDescriptor* value_type = entry_type->map_value(); if (value_type->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { CMessageClass* value_class = message_factory::GetMessageClass( GetFactoryForMessage(self), value_type->message_type()); diff --git a/src/google/protobuf/compiler/cpp/cpp_map_field.cc b/src/google/protobuf/compiler/cpp/cpp_map_field.cc index 130e90ebbe..96f512d6c7 100644 --- a/src/google/protobuf/compiler/cpp/cpp_map_field.cc +++ b/src/google/protobuf/compiler/cpp/cpp_map_field.cc @@ -54,9 +54,9 @@ void SetMessageVariables(const FieldDescriptor* descriptor, (*variables)["full_name"] = descriptor->full_name(); const FieldDescriptor* key = - descriptor->message_type()->FindFieldByName("key"); + descriptor->message_type()->map_key(); const FieldDescriptor* val = - descriptor->message_type()->FindFieldByName("value"); + descriptor->message_type()->map_value(); (*variables)["key_cpp"] = PrimitiveTypeName(options, key->cpp_type()); switch (val->cpp_type()) { case FieldDescriptor::CPPTYPE_MESSAGE: @@ -207,9 +207,9 @@ void MapFieldGenerator::GenerateSerializeWithCachedSizesToArray( format("if (!this->_internal_$name$().empty()) {\n"); format.Indent(); const FieldDescriptor* key_field = - descriptor_->message_type()->FindFieldByName("key"); + descriptor_->message_type()->map_key(); const FieldDescriptor* value_field = - descriptor_->message_type()->FindFieldByName("value"); + descriptor_->message_type()->map_value(); const bool string_key = key_field->type() == FieldDescriptor::TYPE_STRING; const bool string_value = value_field->type() == FieldDescriptor::TYPE_STRING; diff --git a/src/google/protobuf/compiler/cpp/cpp_message.cc b/src/google/protobuf/compiler/cpp/cpp_message.cc index 70d8a57e3a..13850460f0 100644 --- a/src/google/protobuf/compiler/cpp/cpp_message.cc +++ b/src/google/protobuf/compiler/cpp/cpp_message.cc @@ -276,8 +276,8 @@ void CollectMapInfo(const Options& options, const Descriptor* descriptor, std::map* variables) { GOOGLE_CHECK(IsMapEntryMessage(descriptor)); std::map& vars = *variables; - const FieldDescriptor* key = descriptor->FindFieldByName("key"); - const FieldDescriptor* val = descriptor->FindFieldByName("value"); + const FieldDescriptor* key = descriptor->map_key(); + const FieldDescriptor* val = descriptor->map_value(); vars["key_cpp"] = PrimitiveTypeName(options, key->cpp_type()); switch (val->cpp_type()) { case FieldDescriptor::CPPTYPE_MESSAGE: diff --git a/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.cc b/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.cc index 810f240a89..027bf82d88 100644 --- a/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.cc +++ b/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.cc @@ -752,7 +752,7 @@ void ParseFunctionGenerator::GenerateLengthDelim(Formatter& format, case FieldDescriptor::TYPE_MESSAGE: { if (field->is_map()) { const FieldDescriptor* val = - field->message_type()->FindFieldByName("value"); + field->message_type()->map_value(); GOOGLE_CHECK(val); if (val->type() == FieldDescriptor::TYPE_ENUM && !HasPreservingUnknownEnumSemantics(field)) { diff --git a/src/google/protobuf/compiler/csharp/csharp_map_field.cc b/src/google/protobuf/compiler/csharp/csharp_map_field.cc index 44c13e2f63..a13b995da8 100644 --- a/src/google/protobuf/compiler/csharp/csharp_map_field.cc +++ b/src/google/protobuf/compiler/csharp/csharp_map_field.cc @@ -57,9 +57,9 @@ MapFieldGenerator::~MapFieldGenerator() { void MapFieldGenerator::GenerateMembers(io::Printer* printer) { const FieldDescriptor* key_descriptor = - descriptor_->message_type()->FindFieldByName("key"); + descriptor_->message_type()->map_key(); const FieldDescriptor* value_descriptor = - descriptor_->message_type()->FindFieldByName("value"); + descriptor_->message_type()->map_value(); variables_["key_type_name"] = type_name(key_descriptor); variables_["value_type_name"] = type_name(value_descriptor); std::unique_ptr key_generator( diff --git a/src/google/protobuf/compiler/java/java_helpers.cc b/src/google/protobuf/compiler/java/java_helpers.cc index f37ecde784..1b1aebff34 100644 --- a/src/google/protobuf/compiler/java/java_helpers.cc +++ b/src/google/protobuf/compiler/java/java_helpers.cc @@ -1050,7 +1050,7 @@ int GetExperimentalJavaFieldType(const FieldDescriptor* field) { if (field->is_map()) { if (!SupportUnknownEnumValue(field)) { const FieldDescriptor* value = - field->message_type()->FindFieldByName("value"); + field->message_type()->map_value(); if (GetJavaType(value) == JAVATYPE_ENUM) { extra_bits |= kMapWithProto2EnumValue; } diff --git a/src/google/protobuf/compiler/java/java_map_field.cc b/src/google/protobuf/compiler/java/java_map_field.cc index 8a89100714..74d43bbc34 100644 --- a/src/google/protobuf/compiler/java/java_map_field.cc +++ b/src/google/protobuf/compiler/java/java_map_field.cc @@ -47,14 +47,14 @@ const FieldDescriptor* KeyField(const FieldDescriptor* descriptor) { GOOGLE_CHECK_EQ(FieldDescriptor::TYPE_MESSAGE, descriptor->type()); const Descriptor* message = descriptor->message_type(); GOOGLE_CHECK(message->options().map_entry()); - return message->FindFieldByName("key"); + return message->map_key(); } const FieldDescriptor* ValueField(const FieldDescriptor* descriptor) { GOOGLE_CHECK_EQ(FieldDescriptor::TYPE_MESSAGE, descriptor->type()); const Descriptor* message = descriptor->message_type(); GOOGLE_CHECK(message->options().map_entry()); - return message->FindFieldByName("value"); + return message->map_value(); } std::string TypeName(const FieldDescriptor* field, diff --git a/src/google/protobuf/compiler/java/java_map_field_lite.cc b/src/google/protobuf/compiler/java/java_map_field_lite.cc index e71116866e..dac3ae93fb 100644 --- a/src/google/protobuf/compiler/java/java_map_field_lite.cc +++ b/src/google/protobuf/compiler/java/java_map_field_lite.cc @@ -49,14 +49,14 @@ const FieldDescriptor* KeyField(const FieldDescriptor* descriptor) { GOOGLE_CHECK_EQ(FieldDescriptor::TYPE_MESSAGE, descriptor->type()); const Descriptor* message = descriptor->message_type(); GOOGLE_CHECK(message->options().map_entry()); - return message->FindFieldByName("key"); + return message->map_key(); } const FieldDescriptor* ValueField(const FieldDescriptor* descriptor) { GOOGLE_CHECK_EQ(FieldDescriptor::TYPE_MESSAGE, descriptor->type()); const Descriptor* message = descriptor->message_type(); GOOGLE_CHECK(message->options().map_entry()); - return message->FindFieldByName("value"); + return message->map_value(); } std::string TypeName(const FieldDescriptor* field, diff --git a/src/google/protobuf/compiler/java/java_message.cc b/src/google/protobuf/compiler/java/java_message.cc index 27d1014f69..8fdd115c7c 100644 --- a/src/google/protobuf/compiler/java/java_message.cc +++ b/src/google/protobuf/compiler/java/java_message.cc @@ -67,7 +67,7 @@ using internal::WireFormatLite; namespace { std::string MapValueImmutableClassdName(const Descriptor* descriptor, ClassNameResolver* name_resolver) { - const FieldDescriptor* value_field = descriptor->FindFieldByName("value"); + const FieldDescriptor* value_field = descriptor->map_value(); GOOGLE_CHECK_EQ(FieldDescriptor::TYPE_MESSAGE, value_field->type()); return name_resolver->GetImmutableClassName(value_field->message_type()); } diff --git a/src/google/protobuf/compiler/java/java_message_builder.cc b/src/google/protobuf/compiler/java/java_message_builder.cc index 320852b1be..b44015cbd9 100644 --- a/src/google/protobuf/compiler/java/java_message_builder.cc +++ b/src/google/protobuf/compiler/java/java_message_builder.cc @@ -61,7 +61,7 @@ namespace java { namespace { std::string MapValueImmutableClassdName(const Descriptor* descriptor, ClassNameResolver* name_resolver) { - const FieldDescriptor* value_field = descriptor->FindFieldByName("value"); + const FieldDescriptor* value_field = descriptor->map_value(); GOOGLE_CHECK_EQ(FieldDescriptor::TYPE_MESSAGE, value_field->type()); return name_resolver->GetImmutableClassName(value_field->message_type()); } diff --git a/src/google/protobuf/compiler/objectivec/objectivec_map_field.cc b/src/google/protobuf/compiler/objectivec/objectivec_map_field.cc index 746224ff86..c1b1f53b14 100644 --- a/src/google/protobuf/compiler/objectivec/objectivec_map_field.cc +++ b/src/google/protobuf/compiler/objectivec/objectivec_map_field.cc @@ -153,7 +153,7 @@ void MapFieldGenerator::FinishInitialization(void) { // Use the array_comment support in RepeatedFieldGenerator to output what the // values in the map are. const FieldDescriptor* value_descriptor = - descriptor_->message_type()->FindFieldByName("value"); + descriptor_->message_type()->map_value(); if (GetObjectiveCType(value_descriptor) == OBJECTIVECTYPE_ENUM) { variables_["array_comment"] = "// |" + variables_["name"] + "| values are |" + value_field_generator_->variable("storage_type") + "|\n"; @@ -164,7 +164,7 @@ void MapFieldGenerator::DetermineForwardDeclarations( std::set* fwd_decls) const { RepeatedFieldGenerator::DetermineForwardDeclarations(fwd_decls); const FieldDescriptor* value_descriptor = - descriptor_->message_type()->FindFieldByName("value"); + descriptor_->message_type()->map_value(); if (GetObjectiveCType(value_descriptor) == OBJECTIVECTYPE_MESSAGE) { const std::string& value_storage_type = value_field_generator_->variable("storage_type"); @@ -176,7 +176,7 @@ void MapFieldGenerator::DetermineObjectiveCClassDefinitions( std::set* fwd_decls) const { // Class name is already in "storage_type". const FieldDescriptor* value_descriptor = - descriptor_->message_type()->FindFieldByName("value"); + descriptor_->message_type()->map_value(); if (GetObjectiveCType(value_descriptor) == OBJECTIVECTYPE_MESSAGE) { fwd_decls->insert(ObjCClassDeclaration( value_field_generator_->variable("storage_type"))); diff --git a/src/google/protobuf/compiler/php/php_generator.cc b/src/google/protobuf/compiler/php/php_generator.cc index 05f8acad87..4b7fda91d4 100644 --- a/src/google/protobuf/compiler/php/php_generator.cc +++ b/src/google/protobuf/compiler/php/php_generator.cc @@ -741,8 +741,8 @@ void GenerateFieldAccessor(const FieldDescriptor* field, const Options& options, // Type check. if (field->is_map()) { const Descriptor* map_entry = field->message_type(); - const FieldDescriptor* key = map_entry->FindFieldByName("key"); - const FieldDescriptor* value = map_entry->FindFieldByName("value"); + const FieldDescriptor* key = map_entry->map_key(); + const FieldDescriptor* value = map_entry->map_value(); printer->Print( "$arr = GPBUtil::checkMapField($var, " "\\Google\\Protobuf\\Internal\\GPBType::^key_type^, " @@ -889,9 +889,9 @@ void GenerateMessageToPool(const std::string& name_prefix, const FieldDescriptor* field = message->field(i); if (field->is_map()) { const FieldDescriptor* key = - field->message_type()->FindFieldByName("key"); + field->message_type()->map_key(); const FieldDescriptor* val = - field->message_type()->FindFieldByName("value"); + field->message_type()->map_value(); printer->Print( "->map('^field^', \\Google\\Protobuf\\Internal\\GPBType::^key^, " "\\Google\\Protobuf\\Internal\\GPBType::^value^, ^number^^other^)\n", diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index cf7cd852cd..f6de1b5609 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -2325,7 +2325,7 @@ bool Reflection::InsertOrLookupMapValue(Message* message, MapValueRef* val) const { USAGE_CHECK(IsMapFieldInApi(field), "InsertOrLookupMapValue", "Field is not a map field."); - val->SetType(field->message_type()->FindFieldByName("value")->cpp_type()); + val->SetType(field->message_type()->map_value()->cpp_type()); return MutableRaw(message, field) ->InsertOrLookupMapValue(key, val); } @@ -2335,7 +2335,7 @@ bool Reflection::LookupMapValue(const Message& message, MapValueConstRef* val) const { USAGE_CHECK(IsMapFieldInApi(field), "LookupMapValue", "Field is not a map field."); - val->SetType(field->message_type()->FindFieldByName("value")->cpp_type()); + val->SetType(field->message_type()->map_value()->cpp_type()); return GetRaw(message, field).LookupMapValue(key, val); } diff --git a/src/google/protobuf/map_field.h b/src/google/protobuf/map_field.h index c87d44e5eb..19b73ca2b7 100644 --- a/src/google/protobuf/map_field.h +++ b/src/google/protobuf/map_field.h @@ -855,8 +855,8 @@ class PROTOBUF_EXPORT MapIterator { MapIterator(Message* message, const FieldDescriptor* field) { const Reflection* reflection = message->GetReflection(); map_ = reflection->MutableMapData(message, field); - key_.SetType(field->message_type()->FindFieldByName("key")->cpp_type()); - value_.SetType(field->message_type()->FindFieldByName("value")->cpp_type()); + key_.SetType(field->message_type()->map_key()->cpp_type()); + value_.SetType(field->message_type()->map_value()->cpp_type()); map_->InitializeIterator(this); } MapIterator(const MapIterator& other) { diff --git a/src/google/protobuf/map_test.inc b/src/google/protobuf/map_test.inc index 733f13beff..ed75c6fec6 100644 --- a/src/google/protobuf/map_test.inc +++ b/src/google/protobuf/map_test.inc @@ -1259,21 +1259,21 @@ TEST_F(MapFieldReflectionTest, RegularFields) { desc->FindFieldByName("map_int32_foreign_message"); const FieldDescriptor* fd_map_int32_in32_key = - fd_map_int32_int32->message_type()->FindFieldByName("key"); + fd_map_int32_int32->message_type()->map_key(); const FieldDescriptor* fd_map_int32_in32_value = - fd_map_int32_int32->message_type()->FindFieldByName("value"); + fd_map_int32_int32->message_type()->map_value(); const FieldDescriptor* fd_map_int32_double_key = - fd_map_int32_double->message_type()->FindFieldByName("key"); + fd_map_int32_double->message_type()->map_key(); const FieldDescriptor* fd_map_int32_double_value = - fd_map_int32_double->message_type()->FindFieldByName("value"); + fd_map_int32_double->message_type()->map_value(); const FieldDescriptor* fd_map_string_string_key = - fd_map_string_string->message_type()->FindFieldByName("key"); + fd_map_string_string->message_type()->map_key(); const FieldDescriptor* fd_map_string_string_value = - fd_map_string_string->message_type()->FindFieldByName("value"); + fd_map_string_string->message_type()->map_value(); const FieldDescriptor* fd_map_int32_foreign_message_key = - fd_map_int32_foreign_message->message_type()->FindFieldByName("key"); + fd_map_int32_foreign_message->message_type()->map_key(); const FieldDescriptor* fd_map_int32_foreign_message_value = - fd_map_int32_foreign_message->message_type()->FindFieldByName("value"); + fd_map_int32_foreign_message->message_type()->map_value(); // Get RepeatedPtrField objects for all fields of interest. const RepeatedPtrField& mf_int32_int32 = @@ -1446,21 +1446,21 @@ TEST_F(MapFieldReflectionTest, RepeatedFieldRefForRegularFields) { desc->FindFieldByName("map_int32_foreign_message"); const FieldDescriptor* fd_map_int32_in32_key = - fd_map_int32_int32->message_type()->FindFieldByName("key"); + fd_map_int32_int32->message_type()->map_key(); const FieldDescriptor* fd_map_int32_in32_value = - fd_map_int32_int32->message_type()->FindFieldByName("value"); + fd_map_int32_int32->message_type()->map_value(); const FieldDescriptor* fd_map_int32_double_key = - fd_map_int32_double->message_type()->FindFieldByName("key"); + fd_map_int32_double->message_type()->map_key(); const FieldDescriptor* fd_map_int32_double_value = - fd_map_int32_double->message_type()->FindFieldByName("value"); + fd_map_int32_double->message_type()->map_value(); const FieldDescriptor* fd_map_string_string_key = - fd_map_string_string->message_type()->FindFieldByName("key"); + fd_map_string_string->message_type()->map_key(); const FieldDescriptor* fd_map_string_string_value = - fd_map_string_string->message_type()->FindFieldByName("value"); + fd_map_string_string->message_type()->map_value(); const FieldDescriptor* fd_map_int32_foreign_message_key = - fd_map_int32_foreign_message->message_type()->FindFieldByName("key"); + fd_map_int32_foreign_message->message_type()->map_key(); const FieldDescriptor* fd_map_int32_foreign_message_value = - fd_map_int32_foreign_message->message_type()->FindFieldByName("value"); + fd_map_int32_foreign_message->message_type()->map_value(); // Get RepeatedFieldRef objects for all fields of interest. const RepeatedFieldRef mf_int32_int32 = @@ -2038,7 +2038,7 @@ TEST_F(MapFieldReflectionTest, MapSizeWithDuplicatedKey) { const Reflection* entry_reflection = entry1->GetReflection(); const FieldDescriptor* key_field = - entry1->GetDescriptor()->FindFieldByName("key"); + entry1->GetDescriptor()->map_key(); entry_reflection->SetInt32(entry1, key_field, 1); entry_reflection->SetInt32(entry2, key_field, 1); @@ -2059,7 +2059,7 @@ TEST_F(MapFieldReflectionTest, MapSizeWithDuplicatedKey) { const Reflection* entry_reflection = entry1->GetReflection(); const FieldDescriptor* key_field = - entry1->GetDescriptor()->FindFieldByName("key"); + entry1->GetDescriptor()->map_key(); entry_reflection->SetInt32(entry1, key_field, 1); entry_reflection->SetInt32(entry2, key_field, 1); @@ -2951,7 +2951,7 @@ TEST(GeneratedMapFieldReflectionTest, EmbedProto2Message) { UNITTEST::TestMessageMap::descriptor()->FindFieldByName( "map_int32_message"); const FieldDescriptor* value = - map_field->message_type()->FindFieldByName("value"); + map_field->message_type()->map_value(); Message* entry_message = message.GetReflection()->AddMessage(&message, map_field); @@ -2971,9 +2971,9 @@ TEST(GeneratedMapFieldReflectionTest, MergeFromClearMapEntry) { const FieldDescriptor* map_field = UNITTEST::TestMap::descriptor()->FindFieldByName("map_int32_int32"); const FieldDescriptor* key = - map_field->message_type()->FindFieldByName("key"); + map_field->message_type()->map_key(); const FieldDescriptor* value = - map_field->message_type()->FindFieldByName("value"); + map_field->message_type()->map_value(); Message* entry_message1 = message.GetReflection()->AddMessage(&message, map_field); @@ -3005,7 +3005,7 @@ TEST(GeneratedMapFieldReflectionTest, Proto2MapEntryClear) { const FieldDescriptor* field_descriptor = descriptor->FindFieldByName("known_map_field"); const FieldDescriptor* value_descriptor = - field_descriptor->message_type()->FindFieldByName("value"); + field_descriptor->message_type()->map_value(); Message* sub_message = message.GetReflection()->AddMessage(&message, field_descriptor); EXPECT_EQ(0, sub_message->GetReflection()->GetEnumValue(*sub_message, @@ -3133,7 +3133,7 @@ TEST_F(MapFieldInDynamicMessageTest, MapEntryReferernceValidAfterSerialize) { message.get(), "map_int32_foreign_message", 0); const Reflection* map_entry_reflection = map_entry->GetReflection(); const Descriptor* map_entry_desc = map_entry->GetDescriptor(); - const FieldDescriptor* value_field = map_entry_desc->FindFieldByName("value"); + const FieldDescriptor* value_field = map_entry_desc->map_value(); Message* submsg = map_entry_reflection->MutableMessage(map_entry, value_field); @@ -3696,7 +3696,7 @@ TEST(TextFormatMapTest, NoDisableReflectionIterator) { // Modify map via the iterator (invalidated in previous implementation.). const Reflection* map_entry_reflection = iter->GetReflection(); const FieldDescriptor* value_field_desc = - iter->GetDescriptor()->FindFieldByName("value"); + iter->GetDescriptor()->map_value(); map_entry_reflection->SetInt32(&(*iter), value_field_desc, 2); GOOGLE_LOG(INFO) << iter->DebugString(); diff --git a/src/google/protobuf/util/message_differencer_unittest.cc b/src/google/protobuf/util/message_differencer_unittest.cc index fbbcd3c8be..2304743346 100644 --- a/src/google/protobuf/util/message_differencer_unittest.cc +++ b/src/google/protobuf/util/message_differencer_unittest.cc @@ -3408,7 +3408,7 @@ class LengthMapKeyComparator const Reflection* reflection1 = message1.GetReflection(); const Reflection* reflection2 = message2.GetReflection(); const FieldDescriptor* key_field = - message1.GetDescriptor()->FindFieldByName("key"); + message1.GetDescriptor()->map_key(); return reflection1->GetString(message1, key_field).size() == reflection2->GetString(message2, key_field).size(); } From 8a0aa4b3721f89fdbfae57c1dcbc4ef7bed733a8 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Fri, 15 Oct 2021 14:41:59 -0700 Subject: [PATCH 52/52] Update Mac test runs to download RVM directly from GitHub (#9107) Our Mac test runs recently started failing to download RVM. The issue appears to be a combination of an SSL certificate expiring and old OpenSSL versions having a bug preventing them from validating the replacement certificate: https://github.com/rvm/rvm/issues/5133 This commit works around the problem by downloading RVM from GitHub as suggested in one of the comments on the issue above. --- kokoro/macos/prepare_build_macos_rc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kokoro/macos/prepare_build_macos_rc b/kokoro/macos/prepare_build_macos_rc index ef428fcda3..c0017b64ac 100755 --- a/kokoro/macos/prepare_build_macos_rc +++ b/kokoro/macos/prepare_build_macos_rc @@ -33,5 +33,8 @@ if [[ "${KOKORO_INSTALL_RVM:-}" == "yes" ]] ; then curl -sSL https://rvm.io/mpapis.asc | gpg --import - curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - - curl -sSL https://get.rvm.io | bash -s stable --ruby + # Old OpenSSL versions cannot handle the SSL certificate used by + # https://get.rvm.io, so as a workaround we download RVM directly from + # GitHub. See this issue for details: https://github.com/rvm/rvm/issues/5133 + curl -sSL https://raw.githubusercontent.com/rvm/rvm/master/binscripts/rvm-installer | bash -s stable --ruby fi