From 72aef93d6bec2d3ba2dd18297491aec12d0ddcd8 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 10:48:36 +0200 Subject: [PATCH 01/24] [recipe-lib] Add attribute Recipe.built_libraries so we can refactor some essential operations performed for library recipes (like copy the library into the right location, or decide if we should build the library recipe again) --- pythonforandroid/recipe.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index f165ce474c..0792adee10 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -106,6 +106,25 @@ class Recipe(with_metaclass(RecipeMeta)): archs = ['armeabi'] # Not currently implemented properly + built_libraries = {} + """Each recipe that builds a system library (e.g.:libffi, openssl, etc...) + should contain a dict holding the relevant information of the library. The + keys should be the generated libraries and the values the relative path of + the library inside his build folder. This dict will be used to perform + different operations: + - copy the library into the right location, depending on if it's shared + or static) + - check if we have to rebuild the library + + Here an example of how it would look like for `libffi` recipe: + + - `built_libraries = {'libffi.so': '.libs'}` + + .. note:: in case that the built library resides in recipe's build + directory, you can set the following values for the relative + path: `'.', None or ''` + """ + @property def version(self): key = 'VERSION_' + self.name From f7aeb75637d5bea5fa7778d6ae5e745b5a1e98a0 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 10:58:56 +0200 Subject: [PATCH 02/24] [recipe-lib] Add method Recipe.get_libraries This function will allows us to get a list of the built/context libraries. This will be useful to check if we have to built a library recipe or which library should be rebuild --- pythonforandroid/recipe.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 0792adee10..918a316794 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -573,6 +573,27 @@ def install_libs(self, arch, *libs): def has_libs(self, arch, *libs): return all(map(lambda l: self.ctx.has_lib(arch.arch, l), libs)) + def get_libraries(self, arch_name, in_context=False): + """Return the full path of the library depending on the architecture. + Per default, the build library path it will be returned, unless + `get_libraries` has been called with kwarg `in_context` set to + True. + + .. note:: this method should be used for library recipes only + """ + recipe_libs = set() + if not self.built_libraries: + return recipe_libs + for lib, rel_path in self.built_libraries.items(): + if not in_context: + abs_path = join(self.get_build_dir(arch_name), rel_path, lib) + if rel_path in {".", "", None}: + abs_path = join(self.get_build_dir(arch_name), lib) + else: + abs_path = join(self.ctx.get_libs_dir(arch_name), lib) + recipe_libs.add(abs_path) + return recipe_libs + @classmethod def recipe_dirs(cls, ctx): recipe_dirs = [] From c62813e0f395f4c17026afa06bc1f1a07baf8084 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 11:05:00 +0200 Subject: [PATCH 03/24] [recipe-lib] Make `Recipe.should_build` decide the library build In case that the attribute built_libraries has been set, we will check the need of the recipe build, otherwise we will act as usual, forcing the build unless overwrote in subclass --- pythonforandroid/recipe.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 918a316794..8080e6fd0a 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -498,9 +498,14 @@ def apply_patches(self, arch, build_dir=None): def should_build(self, arch): '''Should perform any necessary test and return True only if it needs - building again. + building again. Per default we implement a library test, in case that + we detect so. ''' + if self.built_libraries: + return not all( + exists(lib) for lib in self.get_libraries(arch.arch) + ) return True def build_arch(self, arch): From 47cba1b18c519182aee34eafd22f38b2e0b95b8a Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 11:07:45 +0200 Subject: [PATCH 04/24] [recipe-lib] Add method `Recipe.install_libraries` So we can: - control the copy of the library recipe - refactor the install of the lib for all library recipes --- pythonforandroid/build.py | 1 + pythonforandroid/recipe.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index 16cc459b4c..cc6cea1406 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -575,6 +575,7 @@ def build_recipes(build_order, python_modules, ctx, project_dir, info_main('Building {} for {}'.format(recipe.name, arch.arch)) if recipe.should_build(arch): recipe.build_arch(arch) + recipe.install_libraries(arch) else: info('{} said it is already built, skipping' .format(recipe.name)) diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 8080e6fd0a..0edca4a514 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -516,6 +516,19 @@ def build_arch(self, arch): if hasattr(self, build): getattr(self, build)() + def install_libraries(self, arch): + '''This method is always called after `build_arch`. In case that we + detect a library recipe, defined by the class attribute + `built_libraries`, we will copy all defined libraries into the + right location. + ''' + if not self.built_libraries: + return + shared_libs = [ + lib for lib in self.get_libraries(arch) if lib.endswith(".so") + ] + self.install_libs(arch, *shared_libs) + def postbuild_arch(self, arch): '''Run any post-build tasks for the Recipe. By default, this checks if any postbuild_archname methods exist for the archname of the From b7bf34af6c9cba3fe4c7f201bbbd046f4af663fd Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 11:42:08 +0200 Subject: [PATCH 05/24] [recipe-lib] Make libffi a library recipe --- pythonforandroid/recipes/libffi/__init__.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pythonforandroid/recipes/libffi/__init__.py b/pythonforandroid/recipes/libffi/__init__.py index 3608df6eb0..0d26ec2c43 100644 --- a/pythonforandroid/recipes/libffi/__init__.py +++ b/pythonforandroid/recipes/libffi/__init__.py @@ -2,7 +2,7 @@ from multiprocessing import cpu_count from pythonforandroid.recipe import Recipe from pythonforandroid.logger import shprint -from pythonforandroid.util import current_directory, ensure_dir +from pythonforandroid.util import current_directory import sh @@ -31,8 +31,7 @@ class LibffiRecipe(Recipe): patches = ['remove-version-info.patch'] - def should_build(self, arch): - return not exists(join(self.ctx.get_libs_dir(arch.arch), 'libffi.so')) + built_libraries = {'libffi.so': '.libs'} def build_arch(self, arch): env = self.get_recipe_env(arch) @@ -46,10 +45,6 @@ def build_arch(self, arch): '--disable-builddir', '--enable-shared', _env=env) shprint(sh.make, '-j', str(cpu_count()), 'libffi.la', _env=env) - ensure_dir(self.ctx.get_libs_dir(arch.arch)) - self.install_libs( - arch, join(self.get_build_dir(arch.arch), '.libs', 'libffi.so') - ) def get_include_dirs(self, arch): return [join(self.get_build_dir(arch.arch), 'include')] From bbb84ab8afbeafdc2895786a4bdb0f7258c86d7f Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 11:47:01 +0200 Subject: [PATCH 06/24] [recipe-lib] Make openssl a library recipe and ... also make the imports from the right module --- pythonforandroid/recipes/openssl/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pythonforandroid/recipes/openssl/__init__.py b/pythonforandroid/recipes/openssl/__init__.py index a4aad3df16..a2ecb2c549 100644 --- a/pythonforandroid/recipes/openssl/__init__.py +++ b/pythonforandroid/recipes/openssl/__init__.py @@ -1,6 +1,8 @@ from os.path import join -from pythonforandroid.toolchain import Recipe, shprint, current_directory +from pythonforandroid.recipe import Recipe +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint import sh @@ -50,6 +52,11 @@ class OpenSSLRecipe(Recipe): url = 'https://www.openssl.org/source/openssl-{url_version}.tar.gz' + built_libraries = { + 'libcrypto{version}.so'.format(version=version): '.', + 'libssl{version}.so'.format(version=version): '.', + } + @property def versioned_url(self): if self.url is None: @@ -85,10 +92,6 @@ def link_flags(self, arch): in the format: `-L -l`''' return self.link_dirs_flags(arch) + self.link_libs_flags() - def should_build(self, arch): - return not self.has_libs(arch, 'libssl' + self.version + '.so', - 'libcrypto' + self.version + '.so') - def get_recipe_env(self, arch=None): env = super(OpenSSLRecipe, self).get_recipe_env(arch) env['OPENSSL_VERSION'] = self.version @@ -129,8 +132,5 @@ def build_arch(self, arch): shprint(sh.make, 'build_libs', _env=env) - self.install_libs(arch, 'libssl' + self.version + '.so', - 'libcrypto' + self.version + '.so') - recipe = OpenSSLRecipe() From 0aa70836ce73c7348e61a573fb929883566065f9 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 11:48:14 +0200 Subject: [PATCH 07/24] [recipe-lib] Make png a library recipe --- pythonforandroid/recipes/png/__init__.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pythonforandroid/recipes/png/__init__.py b/pythonforandroid/recipes/png/__init__.py index a6e281a570..a49f28ea2c 100644 --- a/pythonforandroid/recipes/png/__init__.py +++ b/pythonforandroid/recipes/png/__init__.py @@ -2,7 +2,6 @@ from pythonforandroid.logger import shprint from pythonforandroid.util import current_directory from multiprocessing import cpu_count -from os.path import join, exists import sh @@ -10,14 +9,9 @@ class PngRecipe(Recipe): name = 'png' version = 'v1.6.37' url = 'https://github.com/glennrp/libpng/archive/{version}.zip' - - def should_build(self, arch): - return not exists( - join(self.get_build_dir(arch.arch), '.libs', 'libpng16.so') - ) + built_libraries = {'libpng16.so': '.libs'} def build_arch(self, arch): - super(PngRecipe, self).build_arch(arch) build_dir = self.get_build_dir(arch.arch) with current_directory(build_dir): env = self.get_recipe_env(arch) @@ -37,7 +31,6 @@ def build_arch(self, arch): _env=env, ) shprint(sh.make, '-j', str(cpu_count()), _env=env) - self.install_libs(arch, join(build_dir, '.libs', 'libpng16.so')) recipe = PngRecipe() From 5342814e5684f31b08621c6f3fcad2808857b2d6 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 11:49:38 +0200 Subject: [PATCH 08/24] [recipe-lib] Make jpeg a library recipe --- pythonforandroid/recipes/jpeg/__init__.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/pythonforandroid/recipes/jpeg/__init__.py b/pythonforandroid/recipes/jpeg/__init__.py index 507ba775e4..1937ecb1d8 100644 --- a/pythonforandroid/recipes/jpeg/__init__.py +++ b/pythonforandroid/recipes/jpeg/__init__.py @@ -1,9 +1,8 @@ from pythonforandroid.recipe import Recipe from pythonforandroid.logger import shprint from pythonforandroid.util import current_directory -from os.path import join, exists +from os.path import join from os import environ, uname -from glob import glob import sh @@ -16,15 +15,11 @@ class JpegRecipe(Recipe): name = 'jpeg' version = '2.0.1' url = 'https://github.com/libjpeg-turbo/libjpeg-turbo/archive/{version}.tar.gz' # noqa + built_libraries = {'libjpeg.a': '.', 'libturbojpeg.a': '.'} # we will require this below patch to build the shared library # patches = ['remove-version.patch'] - def should_build(self, arch): - return not exists(join(self.get_build_dir(arch.arch), - 'libturbojpeg.a')) - def build_arch(self, arch): - super(JpegRecipe, self).build_arch(arch) build_dir = self.get_build_dir(arch.arch) # TODO: Fix simd/neon @@ -59,10 +54,6 @@ def build_arch(self, arch): _env=env) shprint(sh.make, _env=env) - # copy static libs to libs collection - for lib in glob(join(build_dir, '*.a')): - shprint(sh.cp, '-L', lib, self.ctx.libs_dir) - def get_recipe_env(self, arch=None, with_flags_in_cc=False): env = environ.copy() From ce035b0b79088cb836c67a3bebac0aae3a0718c4 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 11:51:02 +0200 Subject: [PATCH 09/24] [recipe-lib] Make freetype a library recipe and ... also make the imports from the right module --- pythonforandroid/recipes/freetype/__init__.py | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/pythonforandroid/recipes/freetype/__init__.py b/pythonforandroid/recipes/freetype/__init__.py index bea70cdafc..cf8de5ee85 100644 --- a/pythonforandroid/recipes/freetype/__init__.py +++ b/pythonforandroid/recipes/freetype/__init__.py @@ -1,7 +1,7 @@ -from pythonforandroid.toolchain import Recipe +from pythonforandroid.recipe import Recipe from pythonforandroid.logger import shprint, info from pythonforandroid.util import current_directory -from os.path import exists, join +from os.path import join, exists from multiprocessing import cpu_count import sh @@ -26,16 +26,7 @@ class FreetypeRecipe(Recipe): version = '2.5.5' url = 'http://download.savannah.gnu.org/releases/freetype/freetype-{version}.tar.gz' # noqa - - def should_build(self, arch): - return not exists( - join( - self.get_build_dir(arch.arch), - 'objs', - '.libs', - 'libfreetype.so', - ) - ) + built_libraries = {'libfreetype.so': 'objs/.libs'} def get_recipe_env(self, arch=None, with_harfbuzz=False): env = super(FreetypeRecipe, self).get_recipe_env(arch) @@ -111,11 +102,14 @@ def build_arch(self, arch, with_harfbuzz=False): # First build, install the compiled lib, and clean build env shprint(sh.make, 'install', _env=env) shprint(sh.make, 'distclean', _env=env) - else: - # Second build (or the first if harfbuzz not enabled), now we - # copy definitive libs to libs collection. Be sure to link your - # recipes to the definitive library, located at: objs/.libs - self.install_libs(arch, 'objs/.libs/libfreetype.so') + + def install_libraries(self, arch): + # This library it's special because the first time we built it may not + # generate the expected library, because it can depend on harfbuzz, so + # we will make sure to only install it when the library exists + if not exists(list(self.get_libraries(arch))[0]): + return + self.install_libs(arch, *self.get_libraries(arch)) recipe = FreetypeRecipe() From 644682fef30da0847e76f9b4a31d6eceb5736861 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 11:51:44 +0200 Subject: [PATCH 10/24] [recipe-lib] Make harfbuzz a library recipe and ... also make the imports from the right module --- pythonforandroid/recipes/harfbuzz/__init__.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/pythonforandroid/recipes/harfbuzz/__init__.py b/pythonforandroid/recipes/harfbuzz/__init__.py index c2922540ac..17c72a27e2 100644 --- a/pythonforandroid/recipes/harfbuzz/__init__.py +++ b/pythonforandroid/recipes/harfbuzz/__init__.py @@ -1,8 +1,8 @@ -from pythonforandroid.toolchain import Recipe +from pythonforandroid.recipe import Recipe from pythonforandroid.util import current_directory from pythonforandroid.logger import shprint from multiprocessing import cpu_count -from os.path import exists, join +from os.path import join import sh @@ -23,13 +23,7 @@ class HarfbuzzRecipe(Recipe): version = '0.9.40' url = 'http://www.freedesktop.org/software/harfbuzz/release/harfbuzz-{version}.tar.bz2' # noqa opt_depends = ['freetype'] - - def should_build(self, arch): - return not exists( - join( - self.get_build_dir(arch.arch), 'src', '.libs', 'libharfbuzz.so' - ) - ) + built_libraries = {'libharfbuzz.so': 'src/.libs'} def get_recipe_env(self, arch=None): env = super(HarfbuzzRecipe, self).get_recipe_env(arch) @@ -68,12 +62,12 @@ def build_arch(self, arch): _env=env, ) shprint(sh.make, '-j', str(cpu_count()), _env=env) - self.install_libs(arch, join('src', '.libs', 'libharfbuzz.so')) if 'freetype' in self.ctx.recipe_build_order: - # Rebuild freetype with harfbuzz support + # Rebuild/install freetype with harfbuzz support freetype = self.get_recipe('freetype', self.ctx) freetype.build_arch(arch, with_harfbuzz=True) + freetype.install_libraries(arch) recipe = HarfbuzzRecipe() From 158b1c040b26a4abb8c100c3921adfba0e915a18 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 11:53:15 +0200 Subject: [PATCH 11/24] [recipe-lib] Make libcurl a library recipe and ... also make the imports from the right module --- pythonforandroid/recipes/libcurl/__init__.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/pythonforandroid/recipes/libcurl/__init__.py b/pythonforandroid/recipes/libcurl/__init__.py index e8cc86042d..27b142e071 100644 --- a/pythonforandroid/recipes/libcurl/__init__.py +++ b/pythonforandroid/recipes/libcurl/__init__.py @@ -1,20 +1,18 @@ import sh -from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory -from os.path import exists, join +from pythonforandroid.recipe import Recipe +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint +from os.path import join from multiprocessing import cpu_count class LibcurlRecipe(Recipe): version = '7.55.1' url = 'https://curl.haxx.se/download/curl-7.55.1.tar.gz' + built_libraries = {'libcurl.so': 'dist/lib'} depends = ['openssl'] - def should_build(self, arch): - super(LibcurlRecipe, self).should_build(arch) - return not exists(join(self.ctx.get_libs_dir(arch.arch), 'libcurl.so')) - def build_arch(self, arch): - super(LibcurlRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) r = self.get_recipe('openssl', self.ctx) @@ -31,10 +29,6 @@ def build_arch(self, arch): _env=env) shprint(sh.make, '-j', str(cpu_count()), _env=env) shprint(sh.make, 'install', _env=env) - shutil.copyfile('{}/lib/libcurl.so'.format(dst_dir), - join( - self.ctx.get_libs_dir(arch.arch), - 'libcurl.so')) recipe = LibcurlRecipe() From e2fc6ea059e071cbc8dc28a04979113f0c188b2f Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 11:55:54 +0200 Subject: [PATCH 12/24] [recipe-lib] Make libzbar a library recipe and ... also make the imports from the right module --- pythonforandroid/recipes/libzbar/__init__.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pythonforandroid/recipes/libzbar/__init__.py b/pythonforandroid/recipes/libzbar/__init__.py index 43ae34cc9d..8d7d48e2b7 100644 --- a/pythonforandroid/recipes/libzbar/__init__.py +++ b/pythonforandroid/recipes/libzbar/__init__.py @@ -1,6 +1,7 @@ import os -from pythonforandroid.toolchain import shprint, current_directory from pythonforandroid.recipe import Recipe +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint from multiprocessing import cpu_count import sh @@ -15,9 +16,7 @@ class LibZBarRecipe(Recipe): patches = ["werror.patch"] - def should_build(self, arch): - return not os.path.exists( - os.path.join(self.ctx.get_libs_dir(arch.arch), 'libzbar.so')) + built_libraries = {'libzbar.so': 'zbar/.libs'} def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = super(LibZBarRecipe, self).get_recipe_env(arch, with_flags_in_cc) @@ -28,7 +27,6 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): return env def build_arch(self, arch): - super(LibZBarRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): shprint(sh.Command('autoreconf'), '-vif', _env=env) @@ -50,8 +48,6 @@ def build_arch(self, arch): '--enable-static=no', _env=env) shprint(sh.make, '-j' + str(cpu_count()), _env=env) - libs = ['zbar/.libs/libzbar.so'] - self.install_libs(arch, *libs) recipe = LibZBarRecipe() From 8829783c35f2bebd08418aa54e096a5ce172daa9 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 12:34:11 +0200 Subject: [PATCH 13/24] [recipe-lib] Make libiconv a library recipe and ... also make the imports from the right module --- pythonforandroid/recipes/libiconv/__init__.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pythonforandroid/recipes/libiconv/__init__.py b/pythonforandroid/recipes/libiconv/__init__.py index 4a64669202..6bde196e56 100644 --- a/pythonforandroid/recipes/libiconv/__init__.py +++ b/pythonforandroid/recipes/libiconv/__init__.py @@ -1,5 +1,5 @@ -import os -from pythonforandroid.toolchain import shprint, current_directory +from pythonforandroid.logger import shprint +from pythonforandroid.util import current_directory from pythonforandroid.recipe import Recipe from multiprocessing import cpu_count import sh @@ -11,14 +11,11 @@ class LibIconvRecipe(Recipe): url = 'https://ftp.gnu.org/pub/gnu/libiconv/libiconv-{version}.tar.gz' - patches = ['libiconv-1.15-no-gets.patch'] + built_libraries = {'libiconv.so': 'lib/.libs'} - def should_build(self, arch): - return not os.path.exists( - os.path.join(self.ctx.get_libs_dir(arch.arch), 'libiconv.so')) + patches = ['libiconv-1.15-no-gets.patch'] def build_arch(self, arch): - super(LibIconvRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): shprint( @@ -27,8 +24,6 @@ def build_arch(self, arch): '--prefix=' + self.ctx.get_python_install_dir(), _env=env) shprint(sh.make, '-j' + str(cpu_count()), _env=env) - libs = ['lib/.libs/libiconv.so'] - self.install_libs(arch, *libs) recipe = LibIconvRecipe() From 7124f52ebfc8b0910c17c57e8ce566e6fc168d02 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 12:43:43 +0200 Subject: [PATCH 14/24] [recipe-lib] Make libexpat a library recipe and ... also: - make the imports from the right module - remove hardcoded arch --- pythonforandroid/recipes/libexpat/__init__.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/pythonforandroid/recipes/libexpat/__init__.py b/pythonforandroid/recipes/libexpat/__init__.py index ecf5265db3..614b0df0ff 100644 --- a/pythonforandroid/recipes/libexpat/__init__.py +++ b/pythonforandroid/recipes/libexpat/__init__.py @@ -1,38 +1,32 @@ import sh -from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory -from os.path import exists, join +from pythonforandroid.recipe import Recipe +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint +from os.path import join from multiprocessing import cpu_count class LibexpatRecipe(Recipe): version = 'master' url = 'https://github.com/libexpat/libexpat/archive/{version}.zip' + built_libraries = {'libexpat.so': 'dist/lib'} depends = [] - def should_build(self, arch): - super(LibexpatRecipe, self).should_build(arch) - return not exists( - join(self.ctx.get_libs_dir(arch.arch), 'libexpat.so')) - def build_arch(self, arch): - super(LibexpatRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) with current_directory(join(self.get_build_dir(arch.arch), 'expat')): dst_dir = join(self.get_build_dir(arch.arch), 'dist') shprint(sh.Command('./buildconf.sh'), _env=env) shprint( sh.Command('./configure'), - '--host=arm-linux-androideabi', + '--host={}'.format(arch.command_prefix), '--enable-shared', '--without-xmlwf', '--prefix={}'.format(dst_dir), _env=env) shprint(sh.make, '-j', str(cpu_count()), _env=env) shprint(sh.make, 'install', _env=env) - shutil.copyfile( - '{}/lib/libexpat.so'.format(dst_dir), - join(self.ctx.get_libs_dir(arch.arch), 'libexpat.so')) recipe = LibexpatRecipe() From 1be2cba209ef1f01671ec4fa43df4b0ba7aac39c Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 12:48:14 +0200 Subject: [PATCH 15/24] [recipe-lib] Make libogg a library recipe --- pythonforandroid/recipes/libogg/__init__.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pythonforandroid/recipes/libogg/__init__.py b/pythonforandroid/recipes/libogg/__init__.py index 064189eb7d..a96eca9c34 100644 --- a/pythonforandroid/recipes/libogg/__init__.py +++ b/pythonforandroid/recipes/libogg/__init__.py @@ -1,14 +1,12 @@ -from pythonforandroid.recipe import NDKRecipe +from pythonforandroid.recipe import Recipe from pythonforandroid.toolchain import current_directory, shprint -from os.path import join import sh -class OggRecipe(NDKRecipe): +class OggRecipe(Recipe): version = '1.3.3' url = 'http://downloads.xiph.org/releases/ogg/libogg-{version}.tar.gz' - - generated_libraries = ['libogg.so'] + built_libraries = {'libogg.so': 'src/.libs'} def build_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): @@ -20,7 +18,6 @@ def build_arch(self, arch): configure = sh.Command('./configure') shprint(configure, *flags, _env=env) shprint(sh.make, _env=env) - self.install_libs(arch, join('src', '.libs', 'libogg.so')) recipe = OggRecipe() From 723cf9d7db901adcfe78302d4624c418c4b04cc4 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 12:49:47 +0200 Subject: [PATCH 16/24] [recipe-lib] Make libxml2 a library recipe and ... also make the imports from the right module --- pythonforandroid/recipes/libxml2/__init__.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/pythonforandroid/recipes/libxml2/__init__.py b/pythonforandroid/recipes/libxml2/__init__.py index cdeaf88d95..bcccc4ebd8 100644 --- a/pythonforandroid/recipes/libxml2/__init__.py +++ b/pythonforandroid/recipes/libxml2/__init__.py @@ -1,6 +1,7 @@ from pythonforandroid.recipe import Recipe -from pythonforandroid.toolchain import shprint, shutil, current_directory -from os.path import exists, join +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint +from os.path import exists import sh @@ -9,14 +10,9 @@ class Libxml2Recipe(Recipe): url = 'http://xmlsoft.org/sources/libxml2-{version}.tar.gz' depends = [] patches = ['add-glob.c.patch'] - - def should_build(self, arch): - super(Libxml2Recipe, self).should_build(arch) - return not exists( - join(self.get_build_dir(arch.arch), '.libs', 'libxml2.a')) + built_libraries = {'libxml2.a': '.libs'} def build_arch(self, arch): - super(Libxml2Recipe, self).build_arch(arch) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): @@ -46,9 +42,6 @@ def build_arch(self, arch): # we'll need the glob dependency which is a big headache shprint(sh.make, "libxml2.la", _env=env) - shutil.copyfile('.libs/libxml2.a', - join(self.ctx.libs_dir, 'libxml2.a')) - def get_recipe_env(self, arch): env = super(Libxml2Recipe, self).get_recipe_env(arch) env['CONFIG_SHELL'] = '/bin/bash' From 2482bcbed52244e69f56634511c7dbae8863f7a0 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 12:53:08 +0200 Subject: [PATCH 17/24] [recipe-lib] Make libxslt a library recipe and ... also make the imports from the right module --- pythonforandroid/recipes/libxslt/__init__.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/pythonforandroid/recipes/libxslt/__init__.py b/pythonforandroid/recipes/libxslt/__init__.py index 076d6cc6a1..b22e2b328a 100644 --- a/pythonforandroid/recipes/libxslt/__init__.py +++ b/pythonforandroid/recipes/libxslt/__init__.py @@ -1,5 +1,6 @@ from pythonforandroid.recipe import Recipe -from pythonforandroid.toolchain import shprint, shutil, current_directory +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint from os.path import exists, join import sh @@ -9,16 +10,14 @@ class LibxsltRecipe(Recipe): url = 'http://xmlsoft.org/sources/libxslt-{version}.tar.gz' depends = ['libxml2'] patches = ['fix-dlopen.patch'] + built_libraries = { + 'libxslt.a': 'libxslt/.libs', + 'libexslt.a': 'libexslt/.libs' + } call_hostpython_via_targetpython = False - def should_build(self, arch): - return not exists( - join(self.get_build_dir(arch.arch), - 'libxslt', '.libs', 'libxslt.a')) - def build_arch(self, arch): - super(LibxsltRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) build_dir = self.get_build_dir(arch.arch) with current_directory(build_dir): @@ -45,11 +44,6 @@ def build_arch(self, arch): _env=env) shprint(sh.make, "V=1", _env=env) - shutil.copyfile('libxslt/.libs/libxslt.a', - join(self.ctx.libs_dir, 'libxslt.a')) - shutil.copyfile('libexslt/.libs/libexslt.a', - join(self.ctx.libs_dir, 'libexslt.a')) - def get_recipe_env(self, arch): env = super(LibxsltRecipe, self).get_recipe_env(arch) env['CONFIG_SHELL'] = '/bin/bash' From 42f69fbff5090703478d6e10416e76e8bf9ceb23 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 13:01:59 +0200 Subject: [PATCH 18/24] [recipe-lib] Make libshine a library recipe and ... also: - make the imports from the right module - remove the hardcoded cpu count when compiling --- pythonforandroid/recipes/libshine/__init__.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pythonforandroid/recipes/libshine/__init__.py b/pythonforandroid/recipes/libshine/__init__.py index fe9b5b589c..7d114d25a1 100644 --- a/pythonforandroid/recipes/libshine/__init__.py +++ b/pythonforandroid/recipes/libshine/__init__.py @@ -1,5 +1,8 @@ -from pythonforandroid.toolchain import Recipe, current_directory, shprint -from os.path import exists, join, realpath +from pythonforandroid.recipe import Recipe +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint +from multiprocessing import cpu_count +from os.path import realpath import sh @@ -7,9 +10,7 @@ class LibShineRecipe(Recipe): version = 'c72aba9031bde18a0995e7c01c9b53f2e08a0e46' url = 'https://github.com/toots/shine/archive/{version}.zip' - def should_build(self, arch): - build_dir = self.get_build_dir(arch.arch) - return not exists(join(build_dir, 'lib', 'libshine.a')) + built_libraries = {'libshine.a': 'lib'} def build_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): @@ -23,7 +24,7 @@ def build_arch(self, arch): '--enable-static', '--prefix={}'.format(realpath('.')), _env=env) - shprint(sh.make, '-j4', _env=env) + shprint(sh.make, '-j', str(cpu_count()), _env=env) shprint(sh.make, 'install', _env=env) From ccc960d831dedfce3a0dee4abfd27c1022423ccc Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 13:36:39 +0200 Subject: [PATCH 19/24] [recipe-lib] Make libx264 a library recipe and ... MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit also:   - make the imports from the right module   - remove the hardcoded cpu count when compiling --- pythonforandroid/recipes/libx264/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pythonforandroid/recipes/libx264/__init__.py b/pythonforandroid/recipes/libx264/__init__.py index 89d48c8410..13ba31f066 100644 --- a/pythonforandroid/recipes/libx264/__init__.py +++ b/pythonforandroid/recipes/libx264/__init__.py @@ -1,15 +1,15 @@ -from pythonforandroid.toolchain import Recipe, current_directory, shprint -from os.path import exists, join, realpath +from pythonforandroid.recipe import Recipe +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint +from multiprocessing import cpu_count +from os.path import realpath import sh class LibX264Recipe(Recipe): version = 'x264-snapshot-20171218-2245-stable' # using mirror url since can't use ftp url = 'http://mirror.yandex.ru/mirrors/ftp.videolan.org/x264/snapshots/{version}.tar.bz2' - - def should_build(self, arch): - build_dir = self.get_build_dir(arch.arch) - return not exists(join(build_dir, 'lib', 'libx264.a')) + built_libraries = {'libx264.a': 'lib'} def build_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): @@ -29,7 +29,7 @@ def build_arch(self, arch): '--enable-static', '--prefix={}'.format(realpath('.')), _env=env) - shprint(sh.make, '-j4', _env=env) + shprint(sh.make, '-j', str(cpu_count()), _env=env) shprint(sh.make, 'install', _env=env) From 64c609c216518a7dec6d94d0635f80e271b58f14 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 13:39:53 +0200 Subject: [PATCH 20/24] [recipe-lib] Make libglob a library recipe --- pythonforandroid/recipes/libglob/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pythonforandroid/recipes/libglob/__init__.py b/pythonforandroid/recipes/libglob/__init__.py index e0fccfecfe..4c69657c87 100644 --- a/pythonforandroid/recipes/libglob/__init__.py +++ b/pythonforandroid/recipes/libglob/__init__.py @@ -3,13 +3,13 @@ available via '-lglob' LDFLAG """ from os.path import exists, join -from pythonforandroid.recipe import CompiledComponentsPythonRecipe +from pythonforandroid.recipe import Recipe from pythonforandroid.toolchain import current_directory from pythonforandroid.logger import info, shprint import sh -class LibGlobRecipe(CompiledComponentsPythonRecipe): +class LibGlobRecipe(Recipe): """Make a glob.h and glob.so for the python_install_dir()""" version = '0.0.1' url = None @@ -20,6 +20,7 @@ class LibGlobRecipe(CompiledComponentsPythonRecipe): # https://raw.githubusercontent.com/white-gecko/TokyoCabinet/master/glob.c # and pushed in via patch name = 'libglob' + built_libraries = {'libglob.so': '.'} depends = [('hostpython2', 'hostpython3')] patches = ['glob.patch'] @@ -60,7 +61,6 @@ def build_arch(self, arch): cflags.extend(['-shared', '-I.', 'glob.o', '-o', 'libglob.so']) cflags.extend(env['LDFLAGS'].split()) shprint(cc, *cflags, _env=env) - shprint(sh.cp, 'libglob.so', join(self.ctx.libs_dir, arch.arch)) recipe = LibGlobRecipe() From 0ac8c9de0fb53d1931266bba8e8bcbf8dee35ed7 Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 13:44:09 +0200 Subject: [PATCH 21/24] [recipe-lib] Make libsodium a library recipe and ... MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit also:   - make the imports from the right module   - fix hardcoded arch   - enable cpu count when compiling --- .../recipes/libsodium/__init__.py | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/pythonforandroid/recipes/libsodium/__init__.py b/pythonforandroid/recipes/libsodium/__init__.py index 9911e36baa..8165ebc57e 100644 --- a/pythonforandroid/recipes/libsodium/__init__.py +++ b/pythonforandroid/recipes/libsodium/__init__.py @@ -1,5 +1,7 @@ -from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory -from os.path import exists, join +from pythonforandroid.recipe import Recipe +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint +from multiprocessing import cpu_count import sh @@ -8,19 +10,21 @@ class LibsodiumRecipe(Recipe): url = 'https://github.com/jedisct1/libsodium/releases/download/{version}/libsodium-{version}.tar.gz' depends = [] patches = ['size_max_fix.patch'] - - def should_build(self, arch): - super(LibsodiumRecipe, self).should_build(arch) - return not exists(join(self.ctx.get_libs_dir(arch.arch), 'libsodium.so')) + built_libraries = {'libsodium.so': 'src/libsodium/.libs'} def build_arch(self, arch): - super(LibsodiumRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): bash = sh.Command('bash') - shprint(bash, 'configure', '--disable-soname-versions', '--host=arm-linux-androideabi', '--enable-shared', _env=env) - shprint(sh.make, _env=env) - shutil.copyfile('src/libsodium/.libs/libsodium.so', join(self.ctx.get_libs_dir(arch.arch), 'libsodium.so')) + shprint( + bash, + 'configure', + '--disable-soname-versions', + '--host={}'.format(arch.command_prefix), + '--enable-shared', + _env=env, + ) + shprint(sh.make, '-j', str(cpu_count()), _env=env) def get_recipe_env(self, arch): env = super(LibsodiumRecipe, self).get_recipe_env(arch) From 5919a32d99e02fe36e093c763b979e775f8201fd Mon Sep 17 00:00:00 2001 From: opacam Date: Fri, 26 Jul 2019 13:54:05 +0200 Subject: [PATCH 22/24] [recipe-lib] Make libsecp256k1 a library recipe and ... also make the imports from the right module --- pythonforandroid/recipes/libsecp256k1/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pythonforandroid/recipes/libsecp256k1/__init__.py b/pythonforandroid/recipes/libsecp256k1/__init__.py index a8552577eb..caa5a6fc37 100644 --- a/pythonforandroid/recipes/libsecp256k1/__init__.py +++ b/pythonforandroid/recipes/libsecp256k1/__init__.py @@ -1,4 +1,5 @@ -from pythonforandroid.toolchain import shprint, current_directory +from pythonforandroid.logger import shprint +from pythonforandroid.util import current_directory from pythonforandroid.recipe import Recipe from multiprocessing import cpu_count from os.path import exists @@ -7,10 +8,11 @@ class LibSecp256k1Recipe(Recipe): + built_libraries = {'libsecp256k1.so': '.libs'} + url = 'https://github.com/bitcoin-core/secp256k1/archive/master.zip' def build_arch(self, arch): - super(LibSecp256k1Recipe, self).build_arch(arch) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): if not exists('configure'): @@ -25,8 +27,6 @@ def build_arch(self, arch): '--enable-module-ecdh', _env=env) shprint(sh.make, '-j' + str(cpu_count()), _env=env) - libs = ['.libs/libsecp256k1.so'] - self.install_libs(arch, *libs) recipe = LibSecp256k1Recipe() From de769393a02c6703a1321c2a0efde85bf495cfc7 Mon Sep 17 00:00:00 2001 From: opacam Date: Wed, 31 Jul 2019 09:16:28 +0200 Subject: [PATCH 23/24] [tests] Add tests for library recipe --- tests/test_recipe.py | 55 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/test_recipe.py b/tests/test_recipe.py index 9685b15d68..fe4d8c853a 100644 --- a/tests/test_recipe.py +++ b/tests/test_recipe.py @@ -7,6 +7,9 @@ from backports import tempfile from pythonforandroid.build import Context from pythonforandroid.recipe import Recipe, import_recipe +from pythonforandroid.archs import ArchAarch_64 +from pythonforandroid.bootstrap import Bootstrap +from test_bootstrap import BaseClassSetupBootstrap def patch_logger(level): @@ -176,3 +179,55 @@ def test_download_file_scheme_https_oserror(self): assert m_urlretrieve.call_args_list == expected_call_args_list expected_call_args_list = [mock.call(1)] * (retry - 1) assert m_sleep.call_args_list == expected_call_args_list + + +class TestLibraryRecipe(BaseClassSetupBootstrap, unittest.TestCase): + def setUp(self): + """ + Initialize a Context with a Bootstrap and a Distribution to properly + test an library recipe, to do so we reuse `BaseClassSetupBootstrap` + """ + super(TestLibraryRecipe, self).setUp() + self.ctx.bootstrap = Bootstrap().get_bootstrap('sdl2', self.ctx) + self.setUp_distribution_with_bootstrap(self.ctx.bootstrap) + + def test_built_libraries(self): + """The openssl recipe is a library recipe, so it should have set the + attribute `built_libraries`, but not the case of `pyopenssl` recipe. + """ + recipe = Recipe.get_recipe('openssl', self.ctx) + self.assertTrue(recipe.built_libraries) + + recipe = Recipe.get_recipe('pyopenssl', self.ctx) + self.assertFalse(recipe.built_libraries) + + @mock.patch('pythonforandroid.recipe.exists') + def test_should_build(self, mock_exists): + arch = ArchAarch_64(self.ctx) + recipe = Recipe.get_recipe('openssl', self.ctx) + recipe.ctx = self.ctx + self.assertFalse(recipe.should_build(arch)) + + mock_exists.return_value = False + self.assertTrue(recipe.should_build(arch)) + + @mock.patch('pythonforandroid.recipe.Recipe.get_libraries') + @mock.patch('pythonforandroid.recipe.Recipe.install_libs') + def test_install_libraries(self, mock_install_libs, mock_get_libraries): + mock_get_libraries.return_value = { + '/build_lib/libsample1.so', + '/build_lib/libsample2.so', + } + self.ctx.recipe_build_order = [ + "hostpython3", + "openssl", + "python3", + "sdl2", + "kivy", + ] + arch = ArchAarch_64(self.ctx) + recipe = Recipe.get_recipe('openssl', self.ctx) + recipe.install_libraries(arch) + mock_install_libs.assert_called_once_with( + arch, *mock_get_libraries.return_value + ) From 2c2cd3b9634e433abdbb9979e3ff1051b99a08a1 Mon Sep 17 00:00:00 2001 From: opacam Date: Sun, 25 Aug 2019 13:33:16 +0200 Subject: [PATCH 24/24] [NDK19] Fix libglob for android NDK r19 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - change the `ARG_MAX` define, because it's already defined at `sysroot/usr/include/linux/limits.h` - Replaced `size_t` by Including the header. Because found the solution at stackoverflow: As per C99, §7.17, size_t is not a builtin type but defined in See also: - https://travis-ci.org/kivy/python-for-android/jobs/576392711#L5992-L6013 - https://stackoverflow.com/questions/26410466/gcc-linaro-compiler-throws-error-unknown-type-name-size-t --- pythonforandroid/recipes/libglob/glob.patch | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pythonforandroid/recipes/libglob/glob.patch b/pythonforandroid/recipes/libglob/glob.patch index c7fe81738f..ee71719a1e 100644 --- a/pythonforandroid/recipes/libglob/glob.patch +++ b/pythonforandroid/recipes/libglob/glob.patch @@ -911,7 +911,7 @@ diff -Nur /tmp/x/glob.c libglob/glob.c diff -Nur /tmp/x/glob.h libglob/glob.h --- /tmp/x/glob.h 1969-12-31 19:00:00.000000000 -0500 +++ libglob/glob.h 2017-08-19 15:22:18.367109399 -0400 -@@ -0,0 +1,102 @@ +@@ -0,0 +1,104 @@ +/* + * Copyright (c) 1989, 1993 + * The Regents of the University of California. All rights reserved. @@ -952,10 +952,12 @@ diff -Nur /tmp/x/glob.h libglob/glob.h + +#include +#include ++#ifndef ARG_MAX +#define ARG_MAX 6553 ++#endif + +#ifndef _SIZE_T_DECLARED -+typedef __size_t size_t; ++#include +#define _SIZE_T_DECLARED +#endif +