mirror of
https://github.com/protocolbuffers/protobuf
synced 2026-08-26 02:23:14 -04:00
24fef03259
There's a test run in test_python.yml that is non-trivial to get working with Python 3.12 due to some refactoring of our Docker images that would be needed. But this change updates everything else to add coverage for Python 3.12. The main changes necessary to get the builds working were to upgrade some Pip packages via requirements.txt, including in a patch to `rules_fuzzing` that I plan to upstream soon. I also had to take an explicit dependency on `setuptools`. I removed tox.ini, since it was outdated and we have not been actively maintaining it. PiperOrigin-RevId: 580548224
435 lines
16 KiB
Python
Executable file
435 lines
16 KiB
Python
Executable file
#! /usr/bin/env python
|
|
# Protocol Buffers - Google's data interchange format
|
|
# Copyright 2008 Google Inc. All rights reserved.
|
|
#
|
|
# Use of this source code is governed by a BSD-style
|
|
# license that can be found in the LICENSE file or at
|
|
# https://developers.google.com/open-source/licenses/bsd
|
|
#
|
|
# See README for usage instructions.
|
|
|
|
# pylint:disable=missing-module-docstring
|
|
# pylint:disable=g-bad-import-order
|
|
import fnmatch
|
|
import glob
|
|
import os
|
|
import pkg_resources
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import sysconfig
|
|
|
|
# pylint:disable=g-importing-member
|
|
# pylint:disable=g-multiple-import
|
|
|
|
from setuptools import setup, Extension, find_packages
|
|
|
|
from setuptools.command.build_ext import build_ext as _build_ext
|
|
from setuptools.command.build_py import build_py as _build_py
|
|
|
|
# Find the Protocol Compiler.
|
|
if 'PROTOC' in os.environ and os.path.exists(os.environ['PROTOC']):
|
|
protoc = os.environ['PROTOC']
|
|
elif os.path.exists('../bazel-bin/protoc'):
|
|
protoc = '../bazel-bin/protoc'
|
|
elif os.path.exists('../bazel-bin/protoc.exe'):
|
|
protoc = '../bazel-bin/protoc.exe'
|
|
elif os.path.exists('../protoc'):
|
|
protoc = '../protoc'
|
|
elif os.path.exists('../protoc.exe'):
|
|
protoc = '../protoc.exe'
|
|
elif os.path.exists('../vsprojects/Debug/protoc.exe'):
|
|
protoc = '../vsprojects/Debug/protoc.exe'
|
|
elif os.path.exists('../vsprojects/Release/protoc.exe'):
|
|
protoc = '../vsprojects/Release/protoc.exe'
|
|
else:
|
|
protoc = shutil.which('protoc')
|
|
|
|
|
|
def GetVersion():
|
|
"""Reads and returns the version from google/protobuf/__init__.py.
|
|
|
|
Do not import google.protobuf.__init__ directly, because an installed
|
|
protobuf library may be loaded instead.
|
|
|
|
Returns:
|
|
The version.
|
|
"""
|
|
|
|
with open(os.path.join('google', 'protobuf', '__init__.py')) as version_file:
|
|
exec(version_file.read(), globals()) # pylint:disable=exec-used
|
|
return __version__ # pylint:disable=undefined-variable
|
|
|
|
|
|
def GenProto(source, require=True):
|
|
"""Generates a _pb2.py from the given .proto file.
|
|
|
|
Does nothing if the output already exists and is newer than the input.
|
|
|
|
Args:
|
|
source: the .proto file path.
|
|
require: if True, exit immediately when a path is not found.
|
|
"""
|
|
|
|
if not require and not os.path.exists(source):
|
|
return
|
|
|
|
output = source.replace('.proto', '_pb2.py').replace('../src/', '')
|
|
|
|
if (not os.path.exists(output) or
|
|
(os.path.exists(source) and
|
|
os.path.getmtime(source) > os.path.getmtime(output))):
|
|
print('Generating %s...' % output)
|
|
|
|
if not os.path.exists(source):
|
|
sys.stderr.write("Can't find required file: %s\n" % source)
|
|
sys.exit(-1)
|
|
|
|
if protoc is None:
|
|
sys.stderr.write(
|
|
'protoc is not installed nor found in ../src. Please compile it '
|
|
'or install the binary package.\n')
|
|
sys.exit(-1)
|
|
|
|
protoc_command = [protoc, '-I../src', '-I.', '--python_out=.', source]
|
|
if subprocess.call(protoc_command) != 0:
|
|
sys.exit(-1)
|
|
|
|
|
|
def GenerateUnittestProtos():
|
|
"""Generates protobuf code for unittests."""
|
|
GenProto('../src/google/protobuf/any_test.proto', False)
|
|
GenProto('../src/google/protobuf/map_proto2_unittest.proto', False)
|
|
GenProto('../src/google/protobuf/map_unittest.proto', False)
|
|
GenProto('../src/google/protobuf/test_messages_proto3.proto', False)
|
|
GenProto('../src/google/protobuf/test_messages_proto2.proto', False)
|
|
GenProto('../src/google/protobuf/unittest.proto', False)
|
|
GenProto('../src/google/protobuf/unittest_custom_options.proto', False)
|
|
GenProto('../src/google/protobuf/unittest_import.proto', False)
|
|
GenProto('../src/google/protobuf/unittest_import_public.proto', False)
|
|
GenProto('../src/google/protobuf/unittest_mset.proto', False)
|
|
GenProto('../src/google/protobuf/unittest_mset_wire_format.proto', False)
|
|
GenProto('../src/google/protobuf/unittest_no_generic_services.proto', False)
|
|
GenProto('../src/google/protobuf/unittest_proto3_arena.proto', False)
|
|
GenProto('../src/google/protobuf/unittest_retention.proto', False)
|
|
GenProto('../src/google/protobuf/util/json_format.proto', False)
|
|
GenProto('../src/google/protobuf/util/json_format_proto3.proto', False)
|
|
GenProto('google/protobuf/internal/any_test.proto', False)
|
|
GenProto('google/protobuf/internal/descriptor_pool_test1.proto', False)
|
|
GenProto('google/protobuf/internal/descriptor_pool_test2.proto', False)
|
|
GenProto('google/protobuf/internal/factory_test1.proto', False)
|
|
GenProto('google/protobuf/internal/factory_test2.proto', False)
|
|
GenProto('google/protobuf/internal/file_options_test.proto', False)
|
|
GenProto('google/protobuf/internal/import_test_package/import_public.proto',
|
|
False)
|
|
GenProto(
|
|
'google/protobuf/internal/import_test_package/import_public_nested.proto',
|
|
False)
|
|
GenProto('google/protobuf/internal/import_test_package/inner.proto', False)
|
|
GenProto('google/protobuf/internal/import_test_package/outer.proto', False)
|
|
GenProto('google/protobuf/internal/missing_enum_values.proto', False)
|
|
GenProto('google/protobuf/internal/message_set_extensions.proto', False)
|
|
GenProto('google/protobuf/internal/more_extensions.proto', False)
|
|
GenProto('google/protobuf/internal/more_extensions_dynamic.proto', False)
|
|
GenProto('google/protobuf/internal/more_messages.proto', False)
|
|
GenProto('google/protobuf/internal/no_package.proto', False)
|
|
GenProto('google/protobuf/internal/packed_field_test.proto', False)
|
|
GenProto('google/protobuf/internal/test_bad_identifiers.proto', False)
|
|
GenProto('google/protobuf/internal/test_proto3_optional.proto', False)
|
|
GenProto('google/protobuf/pyext/python.proto', False)
|
|
|
|
|
|
class BuildPyCmd(_build_py):
|
|
"""Custom build_py command for building the protobuf runtime."""
|
|
|
|
def run(self):
|
|
# Generate necessary .proto file if it doesn't exist.
|
|
GenProto('../src/google/protobuf/descriptor.proto')
|
|
GenProto('../src/google/protobuf/compiler/plugin.proto')
|
|
GenProto('../src/google/protobuf/any.proto')
|
|
GenProto('../src/google/protobuf/api.proto')
|
|
GenProto('../src/google/protobuf/duration.proto')
|
|
GenProto('../src/google/protobuf/empty.proto')
|
|
GenProto('../src/google/protobuf/field_mask.proto')
|
|
GenProto('../src/google/protobuf/source_context.proto')
|
|
GenProto('../src/google/protobuf/struct.proto')
|
|
GenProto('../src/google/protobuf/timestamp.proto')
|
|
GenProto('../src/google/protobuf/type.proto')
|
|
GenProto('../src/google/protobuf/wrappers.proto')
|
|
GenerateUnittestProtos()
|
|
|
|
# _build_py is an old-style class, so super() doesn't work.
|
|
_build_py.run(self)
|
|
|
|
def find_package_modules(self, package, package_dir):
|
|
exclude = (
|
|
'*test*',
|
|
'google/protobuf/internal/*_pb2.py',
|
|
'google/protobuf/internal/_parameterized.py',
|
|
'google/protobuf/pyext/python_pb2.py',
|
|
)
|
|
modules = _build_py.find_package_modules(self, package, package_dir)
|
|
return [(pkg, mod, fil) for (pkg, mod, fil) in modules
|
|
if not any(fnmatch.fnmatchcase(fil, pat=pat) for pat in exclude)]
|
|
|
|
|
|
class BuildExtCmd(_build_ext):
|
|
"""Command class for building the protobuf Python extension."""
|
|
|
|
def get_ext_filename(self, ext_name):
|
|
# since python3.5, python extensions' shared libraries use a suffix that
|
|
# corresponds to the value of sysconfig.get_config_var('EXT_SUFFIX') and
|
|
# contains info about the architecture the library targets. E.g. on x64
|
|
# linux the suffix is ".cpython-XYZ-x86_64-linux-gnu.so" When
|
|
# crosscompiling python wheels, we need to be able to override this
|
|
# suffix so that the resulting file name matches the target architecture
|
|
# and we end up with a well-formed wheel.
|
|
filename = _build_ext.get_ext_filename(self, ext_name)
|
|
orig_ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
|
|
new_ext_suffix = os.getenv('PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX')
|
|
if new_ext_suffix and filename.endswith(orig_ext_suffix):
|
|
filename = filename[:-len(orig_ext_suffix)] + new_ext_suffix
|
|
return filename
|
|
|
|
|
|
class TestConformanceCmd(_build_py):
|
|
target = '//python:conformance_test'
|
|
|
|
def run(self):
|
|
# Python 2.6 dodges these extra failures.
|
|
os.environ['CONFORMANCE_PYTHON_EXTRA_FAILURES'] = (
|
|
'--failure_list failure_list_python-post26.txt')
|
|
cmd = 'bazel test %s' % (TestConformanceCmd.target,)
|
|
subprocess.check_call(cmd, shell=True)
|
|
|
|
|
|
def GetOptionFromArgv(option_str):
|
|
if option_str in sys.argv:
|
|
sys.argv.remove(option_str)
|
|
return True
|
|
return False
|
|
|
|
|
|
def _GetFlagValues(flag_long, flag_short):
|
|
"""Searches sys.argv for setuptools-style flags and yields values."""
|
|
|
|
expect_value = flag_long.endswith('=')
|
|
flag_res = [re.compile(r'--?%s(=(.*))?' %
|
|
(flag_long[:-1] if expect_value else flag_long))]
|
|
if flag_short:
|
|
flag_res.append(re.compile(r'-%s(.*)?' % (flag_short,)))
|
|
|
|
flag_match = None
|
|
for arg in sys.argv:
|
|
# If the last arg was like '-O', check if this is the library we want.
|
|
if flag_match is not None:
|
|
yield arg
|
|
flag_match = None
|
|
continue
|
|
|
|
for flag_re in flag_res:
|
|
m = flag_re.match(arg)
|
|
if m is None:
|
|
continue
|
|
if not expect_value:
|
|
yield arg
|
|
continue
|
|
groups = m.groups()
|
|
# Check for matches:
|
|
# --long-name=foo => ('=foo', 'foo')
|
|
# -Xfoo => ('foo')
|
|
# N.B.: if the flag is like '--long-name=', then there is a value
|
|
# (the empty string).
|
|
if groups[0] or groups[-1]:
|
|
yield groups[-1]
|
|
continue
|
|
flag_match = m
|
|
|
|
return False
|
|
|
|
|
|
def HasStaticLibprotobufOpt():
|
|
"""Returns true if there is a --link-objects arg for libprotobuf."""
|
|
|
|
lib_re = re.compile(r'(.*[/\\])?(lib)?protobuf([.]pic)?[.](a|lib)')
|
|
for value in _GetFlagValues('link-objects=', 'O'):
|
|
if lib_re.match(value):
|
|
return True
|
|
return False
|
|
|
|
|
|
def HasLibraryDirsOpt():
|
|
"""Returns true if there is a --library-dirs arg."""
|
|
return any(_GetFlagValues('library-dirs=', 'L'))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
ext_module_list = []
|
|
warnings_as_errors = '--warnings_as_errors'
|
|
if GetOptionFromArgv('--cpp_implementation'):
|
|
# Link libprotobuf.a and libprotobuf-lite.a statically with the
|
|
# extension. Note that those libraries have to be compiled with
|
|
# -fPIC for this to work.
|
|
compile_static_ext = HasStaticLibprotobufOpt()
|
|
if GetOptionFromArgv('--compile_static_extension'):
|
|
# FUTURE: add a warning and deprecate --compile_static_extension.
|
|
compile_static_ext = True
|
|
extra_objects = None
|
|
if compile_static_ext:
|
|
libraries = None
|
|
library_dirs = None
|
|
if not HasStaticLibprotobufOpt():
|
|
if os.path.exists('../bazel-bin/src/google/protobuf/libprotobuf.a'):
|
|
extra_objects = ['../bazel-bin/src/google/protobuf/libprotobuf.a']
|
|
else:
|
|
extra_objects = ['../libprotobuf.a']
|
|
extra_objects += list(
|
|
glob.iglob('../third_party/utf8_range/*.a'))
|
|
# Repeat all of these enough times to eliminate order-dependence.
|
|
extra_objects += list(
|
|
glob.iglob('../third_party/abseil-cpp/absl/**/*.a'))
|
|
extra_objects += list(
|
|
glob.iglob('../third_party/abseil-cpp/absl/**/*.a'))
|
|
extra_objects += list(
|
|
glob.iglob('../third_party/abseil-cpp/absl/**/*.a'))
|
|
else:
|
|
libraries = ['protobuf']
|
|
if HasLibraryDirsOpt():
|
|
library_dirs = None
|
|
elif os.path.exists('../bazel-bin/src/google/protobuf/libprotobuf.a'):
|
|
library_dirs = ['../bazel-bin/src/google/protobuf']
|
|
else:
|
|
library_dirs = ['..']
|
|
|
|
TestConformanceCmd.target = ('//python:conformance_test_cpp '
|
|
'--define=use_fast_cpp_protos=true')
|
|
|
|
extra_compile_args = []
|
|
|
|
message_extra_link_args = None
|
|
api_implementation_link_args = None
|
|
if 'darwin' in sys.platform:
|
|
if sys.version_info[0] == 2:
|
|
message_init_symbol = 'init_message'
|
|
api_implementation_init_symbol = 'init_api_implementation'
|
|
else:
|
|
message_init_symbol = 'PyInit__message'
|
|
api_implementation_init_symbol = 'PyInit__api_implementation'
|
|
message_extra_link_args = [
|
|
'-Wl,-exported_symbol,_%s' % message_init_symbol
|
|
]
|
|
api_implementation_link_args = [
|
|
'-Wl,-exported_symbol,_%s' % api_implementation_init_symbol
|
|
]
|
|
|
|
if sys.platform != 'win32':
|
|
extra_compile_args.append('-Wno-write-strings')
|
|
extra_compile_args.append('-Wno-invalid-offsetof')
|
|
extra_compile_args.append('-Wno-sign-compare')
|
|
extra_compile_args.append('-Wno-unused-variable')
|
|
extra_compile_args.append('-std=c++14')
|
|
|
|
if sys.platform == 'darwin':
|
|
extra_compile_args.append('-Wno-shorten-64-to-32')
|
|
extra_compile_args.append('-Wno-deprecated-register')
|
|
|
|
# https://developer.apple.com/documentation/xcode_release_notes/xcode_10_release_notes
|
|
# C++ projects must now migrate to libc++ and are recommended to set a
|
|
# deployment target of macOS 10.9 or later, or iOS 7 or later.
|
|
if sys.platform == 'darwin':
|
|
mac_target = str(sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET'))
|
|
if mac_target and (pkg_resources.parse_version(mac_target) <
|
|
pkg_resources.parse_version('10.9.0')):
|
|
os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.9'
|
|
os.environ['_PYTHON_HOST_PLATFORM'] = re.sub(
|
|
r'macosx-[0-9]+\.[0-9]+-(.+)',
|
|
r'macosx-10.9-\1',
|
|
sysconfig.get_platform(),
|
|
)
|
|
|
|
# https://github.com/Theano/Theano/issues/4926
|
|
if sys.platform == 'win32':
|
|
extra_compile_args.append('-D_hypot=hypot')
|
|
|
|
# https://github.com/tpaviot/pythonocc-core/issues/48
|
|
if sys.platform == 'win32' and '64 bit' in sys.version:
|
|
extra_compile_args.append('-DMS_WIN64')
|
|
|
|
# MSVS default is dymanic
|
|
if sys.platform == 'win32':
|
|
extra_compile_args.append('/MT')
|
|
|
|
if 'clang' in os.popen('$CC --version 2> /dev/null').read():
|
|
extra_compile_args.append('-Wno-shorten-64-to-32')
|
|
|
|
if warnings_as_errors in sys.argv:
|
|
extra_compile_args.append('-Werror')
|
|
sys.argv.remove(warnings_as_errors)
|
|
|
|
# C++ implementation extension
|
|
ext_module_list.extend([
|
|
Extension(
|
|
'google.protobuf.pyext._message',
|
|
glob.glob('google/protobuf/pyext/*.cc'),
|
|
include_dirs=['.', '../src', '../third_party/abseil-cpp'],
|
|
libraries=libraries,
|
|
extra_objects=extra_objects,
|
|
extra_link_args=message_extra_link_args,
|
|
library_dirs=library_dirs,
|
|
extra_compile_args=extra_compile_args,
|
|
),
|
|
Extension(
|
|
'google.protobuf.internal._api_implementation',
|
|
glob.glob('google/protobuf/internal/api_implementation.cc'),
|
|
extra_compile_args=(extra_compile_args +
|
|
['-DPYTHON_PROTO2_CPP_IMPL_V2']),
|
|
extra_link_args=api_implementation_link_args,
|
|
),
|
|
])
|
|
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
|
|
|
|
install_requires = []
|
|
|
|
setup(
|
|
name='protobuf',
|
|
version=GetVersion(),
|
|
description='Protocol Buffers',
|
|
download_url='https://github.com/protocolbuffers/protobuf/releases',
|
|
long_description="Protocol Buffers are Google's data interchange format",
|
|
url='https://developers.google.com/protocol-buffers/',
|
|
project_urls={
|
|
'Source': 'https://github.com/protocolbuffers/protobuf',
|
|
},
|
|
maintainer='protobuf@googlegroups.com',
|
|
maintainer_email='protobuf@googlegroups.com',
|
|
license='BSD-3-Clause',
|
|
classifiers=[
|
|
'Programming Language :: Python',
|
|
'Programming Language :: Python :: 3',
|
|
# LINT.IfChange
|
|
# Remove importlib fallback path when we drop Python 3.8 support.
|
|
'Programming Language :: Python :: 3.8',
|
|
# LINT.ThenChange(//depot/google3/google/protobuf/internal/test_util.py)
|
|
'Programming Language :: Python :: 3.9',
|
|
'Programming Language :: Python :: 3.10',
|
|
'Programming Language :: Python :: 3.11',
|
|
'Programming Language :: Python :: 3.12',
|
|
],
|
|
namespace_packages=['google'],
|
|
packages=find_packages(
|
|
exclude=[
|
|
'import_test_package',
|
|
'protobuf_distutils',
|
|
],
|
|
),
|
|
test_suite='google.protobuf.internal',
|
|
cmdclass={
|
|
'build_py': BuildPyCmd,
|
|
'build_ext': BuildExtCmd,
|
|
'test_conformance': TestConformanceCmd,
|
|
},
|
|
install_requires=install_requires,
|
|
ext_modules=ext_module_list,
|
|
python_requires='>=3.8',
|
|
)
|