From d99161e7d07f0177d802a44436b205cd9302c912 Mon Sep 17 00:00:00 2001 From: opacam Date: Wed, 12 Dec 2018 00:30:29 +0100 Subject: [PATCH 1/2] Update protobuf_cpp recipe and grant python3 compatibility The removed flags are already set in base class, so no need to set in here. --- .../recipes/protobuf_cpp/__init__.py | 45 ++++----- .../fix-python3-compatibility.patch | 91 +++++++++++++++++++ 2 files changed, 108 insertions(+), 28 deletions(-) create mode 100644 pythonforandroid/recipes/protobuf_cpp/fix-python3-compatibility.patch diff --git a/pythonforandroid/recipes/protobuf_cpp/__init__.py b/pythonforandroid/recipes/protobuf_cpp/__init__.py index 6cfa674b57..f41e51ab3f 100644 --- a/pythonforandroid/recipes/protobuf_cpp/__init__.py +++ b/pythonforandroid/recipes/protobuf_cpp/__init__.py @@ -1,7 +1,7 @@ from pythonforandroid.recipe import PythonRecipe from pythonforandroid.logger import shprint, info_notify from pythonforandroid.util import current_directory, shutil -from os.path import exists, join, dirname +from os.path import exists, join import sh from multiprocessing import cpu_count from pythonforandroid.toolchain import info @@ -11,7 +11,7 @@ class ProtobufCppRecipe(PythonRecipe): name = 'protobuf_cpp' - version = '3.5.1' + version = '3.6.1' url = 'https://github.com/google/protobuf/releases/download/v{version}/protobuf-python-{version}.tar.gz' call_hostpython_via_targetpython = False depends = ['cffi', 'setuptools'] @@ -20,6 +20,12 @@ class ProtobufCppRecipe(PythonRecipe): def prebuild_arch(self, arch): super(ProtobufCppRecipe, self).prebuild_arch(arch) + + patch_mark = join(self.get_build_dir(arch.arch), '.protobuf-patched') + if self.ctx.python_recipe.name == 'python3' and not exists(patch_mark): + self.apply_patch('fix-python3-compatibility.patch', arch.arch) + shprint(sh.touch, patch_mark) + # During building, host needs to transpile .proto files to .py # ideally with the same version as protobuf runtime, or with an older one. # Because protoc is compiled for target (i.e. Android), we need an other binary @@ -100,34 +106,18 @@ def install_python_package(self, arch): with current_directory(join(self.get_build_dir(arch.arch), 'python')): hostpython = sh.Command(self.hostpython_location) - if self.ctx.python_recipe.from_crystax: - hpenv = env.copy() - shprint(hostpython, 'setup.py', 'install', '-O2', - '--root={}'.format(self.ctx.get_python_install_dir()), - '--install-lib=.', - '--cpp_implementation', - _env=hpenv, *self.setup_extra_args) - else: - hppath = join(dirname(self.hostpython_location), 'Lib', - 'site-packages') - hpenv = env.copy() - if 'PYTHONPATH' in hpenv: - hpenv['PYTHONPATH'] = ':'.join([hppath] + - hpenv['PYTHONPATH'].split(':')) - else: - hpenv['PYTHONPATH'] = hppath - shprint(hostpython, 'setup.py', 'install', '-O2', - '--root={}'.format(self.ctx.get_python_install_dir()), - '--install-lib=lib/python2.7/site-packages', - '--cpp_implementation', - _env=hpenv, *self.setup_extra_args) + hpenv = env.copy() + shprint(hostpython, 'setup.py', 'install', '-O2', + '--root={}'.format(self.ctx.get_python_install_dir()), + '--install-lib=.', + '--cpp_implementation', + _env=hpenv, *self.setup_extra_args) def get_recipe_env(self, arch): env = super(ProtobufCppRecipe, self).get_recipe_env(arch) if self.protoc_dir is not None: # we need protoc with binary for host platform env['PROTOC'] = join(self.protoc_dir, 'bin', 'protoc') - env['PYTHON_ROOT'] = self.ctx.get_python_install_dir() env['TARGET_OS'] = 'OS_ANDROID_CROSSCOMPILE' env['CFLAGS'] += ( ' -I' + self.ctx.ndk_dir + '/platforms/android-' + @@ -136,17 +126,16 @@ def get_recipe_env(self, arch): ' -I' + self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + '/include' + ' -I' + self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + - self.ctx.toolchain_version + '/libs/' + arch.arch + '/include' + - ' -I' + env['PYTHON_ROOT'] + '/include/python2.7') + self.ctx.toolchain_version + '/libs/' + arch.arch + '/include') + env['CFLAGS'] += ' -std=gnu++11' env['CXXFLAGS'] = env['CFLAGS'] env['CXXFLAGS'] += ' -frtti' env['CXXFLAGS'] += ' -fexceptions' env['LDFLAGS'] += ( ' -L' + self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + - '/libs/' + arch.arch + ' -lgnustl_shared -lpython2.7 -landroid -llog') + '/libs/' + arch.arch + ' -lgnustl_shared -landroid -llog') - env['LDSHARED'] = env['CC'] + ' -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions' return env diff --git a/pythonforandroid/recipes/protobuf_cpp/fix-python3-compatibility.patch b/pythonforandroid/recipes/protobuf_cpp/fix-python3-compatibility.patch new file mode 100644 index 0000000000..e77debaa61 --- /dev/null +++ b/pythonforandroid/recipes/protobuf_cpp/fix-python3-compatibility.patch @@ -0,0 +1,91 @@ +From 539bc017a62f91bdf7c547b58948cb5a2f59d918 Mon Sep 17 00:00:00 2001 +From: Ben Webb +Date: Thu, 12 Jul 2018 10:58:10 -0700 +Subject: [PATCH] Add Python 3.7 compatibility (#4862) + +Compilation of Python wrappers fails with Python 3.7 because +the Python folks changed their C API such that +PyUnicode_AsUTF8AndSize() now returns a const char* rather +than a char*. Add a patch to work around. Relates #4086. +--- + python/google/protobuf/pyext/descriptor.cc | 2 +- + python/google/protobuf/pyext/descriptor_containers.cc | 2 +- + python/google/protobuf/pyext/descriptor_pool.cc | 2 +- + python/google/protobuf/pyext/extension_dict.cc | 2 +- + python/google/protobuf/pyext/message.cc | 4 ++-- + 5 files changed, 6 insertions(+), 6 deletions(-) + +diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc +index 8af0cb1289..19a1c38a62 100644 +--- a/python/google/protobuf/pyext/descriptor.cc ++++ b/python/google/protobuf/pyext/descriptor.cc +@@ -56,7 +56,7 @@ + #endif + #define PyString_AsStringAndSize(ob, charpp, sizep) \ + (PyUnicode_Check(ob)? \ +- ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ ++ ((*(charpp) = const_cast(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \ + PyBytes_AsStringAndSize(ob, (charpp), (sizep))) + #endif + +diff --git a/python/google/protobuf/pyext/descriptor_containers.cc b/python/google/protobuf/pyext/descriptor_containers.cc +index bc007f7efa..0153664f50 100644 +--- a/python/google/protobuf/pyext/descriptor_containers.cc ++++ b/python/google/protobuf/pyext/descriptor_containers.cc +@@ -66,7 +66,7 @@ + #endif + #define PyString_AsStringAndSize(ob, charpp, sizep) \ + (PyUnicode_Check(ob)? \ +- ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ ++ ((*(charpp) = const_cast(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \ + PyBytes_AsStringAndSize(ob, (charpp), (sizep))) + #endif + +diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc +index 95882aeb35..962accc6e9 100644 +--- a/python/google/protobuf/pyext/descriptor_pool.cc ++++ b/python/google/protobuf/pyext/descriptor_pool.cc +@@ -48,7 +48,7 @@ + #endif + #define PyString_AsStringAndSize(ob, charpp, sizep) \ + (PyUnicode_Check(ob)? \ +- ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ ++ ((*(charpp) = const_cast(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \ + PyBytes_AsStringAndSize(ob, (charpp), (sizep))) + #endif + +diff --git a/python/google/protobuf/pyext/extension_dict.cc b/python/google/protobuf/pyext/extension_dict.cc +index 018b5c2c49..174c5470c2 100644 +--- a/python/google/protobuf/pyext/extension_dict.cc ++++ b/python/google/protobuf/pyext/extension_dict.cc +@@ -53,7 +53,7 @@ + #endif + #define PyString_AsStringAndSize(ob, charpp, sizep) \ + (PyUnicode_Check(ob)? \ +- ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ ++ ((*(charpp) = const_cast(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \ + PyBytes_AsStringAndSize(ob, (charpp), (sizep))) + #endif + +diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc +index 5893533adf..31094b7e10 100644 +--- a/python/google/protobuf/pyext/message.cc ++++ b/python/google/protobuf/pyext/message.cc +@@ -79,7 +79,7 @@ + (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AsString(ob)) + #define PyString_AsStringAndSize(ob, charpp, sizep) \ + (PyUnicode_Check(ob)? \ +- ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ ++ ((*(charpp) = const_cast(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \ + PyBytes_AsStringAndSize(ob, (charpp), (sizep))) + #endif + #endif +@@ -1529,7 +1529,7 @@ PyObject* HasField(CMessage* self, PyObject* arg) { + return NULL; + } + #else +- field_name = PyUnicode_AsUTF8AndSize(arg, &size); ++ field_name = const_cast(PyUnicode_AsUTF8AndSize(arg, &size)); + if (!field_name) { + return NULL; + } From 786044d2dec111c7792b02bf0c7a2cf08bc16379 Mon Sep 17 00:00:00 2001 From: opacam Date: Tue, 15 Jan 2019 16:32:06 +0100 Subject: [PATCH 2/2] Move libraries from LDFLAGS to LIBS for protobuf_cpp recipe Because this is how you are supposed to do it, you must use LDFLAGS for linker flags and LDLIBS (or the equivalent LOADLIBES) for the libraries --- pythonforandroid/recipes/protobuf_cpp/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pythonforandroid/recipes/protobuf_cpp/__init__.py b/pythonforandroid/recipes/protobuf_cpp/__init__.py index f41e51ab3f..d82fcf7206 100644 --- a/pythonforandroid/recipes/protobuf_cpp/__init__.py +++ b/pythonforandroid/recipes/protobuf_cpp/__init__.py @@ -134,7 +134,8 @@ def get_recipe_env(self, arch): env['LDFLAGS'] += ( ' -L' + self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + - '/libs/' + arch.arch + ' -lgnustl_shared -landroid -llog') + '/libs/' + arch.arch) + env['LIBS'] = env.get('LIBS', '') + ' -lgnustl_shared -landroid -llog' return env