8000 Add automatic expansion of --requirement list · kivy/python-for-android@82125ef · GitHub
[go: up one dir, main page]
Skip to content

Commit 82125ef

Browse files
pdallairpdallair
committed
Add automatic expansion of --requirement list
Related to issue #2529
1 parent a9eee16 commit 82125ef

File tree

3 files changed

+224
-10
lines changed

3 files changed

+224
-10
lines changed

‎pythonforandroid/toolchain.py

Lines changed: 129 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
66
This module defines the entry point for command line and programmatic use.
77
"""
8-
98
from os import environ
109
from pythonforandroid import __version__
1110
from pythonforandroid.pythonpackage import get_dep_names_of_package
@@ -654,9 +653,12 @@ def add_parser(subparsers, *args, **kwargs):
654653
"pyproject.toml"))):
655654
have_setup_py_or_similar = True
656655

657-
# Process requirements and put version in environ
658-
if hasattr(args, 'requirements'):
659-
requirements = []
656+
# Process requirements and put version in environ:
657+
if hasattr(args, 'requirements') and args.requirements:
658+
all_recipes = [
659+
recipe.lower() for recipe in
660+
set(Recipe.list_recipes(self.ctx))
661+
]
660662

661663
# Add dependencies from setup.py, but only if they are recipes
662664
# (because otherwise, setup.py itself will install them later)
@@ -675,10 +677,6 @@ def add_parser(subparsers, *args, **kwargs):
675677
)
676678
]
677679
info("Dependencies obtained: " + str(dependencies))
678-
all_recipes = [
679-
recipe.lower() for recipe in
680-
set(Recipe.list_recipes(self.ctx))
681-
]
682680
dependencies = set(dependencies).intersection(
683681
set(all_recipes)
684682
)
@@ -694,7 +692,126 @@ def add_parser(subparsers, *args, **kwargs):
694692
"package? Will continue WITHOUT setup.py deps."
695693
)
696694

697-
# Parse --requirements argument list:
695+
non_recipe_requirements = []
696+
for requirement in args.requirements.split(','):
697+
requirement_name = re.sub(r'==\d+(\.\d+)*', '', requirement)
698+
if requirement_name not in all_recipes:
699+
non_recipe_requirements.append(requirement)
700+
args.requirements = re.sub(
701+
r',?{}'.format(requirement), '', args.requirements)
702+
703+
# Compile "non-recipe" requirements' dependencies and add to list.
704+
# Otherwise, only recipe requirements' dependencies get installed.
705+
# More info https://github.com/kivy/python-for-android/issues/2529
706+
if non_recipe_requirements:
707+
info("Compiling dependencies for: "
708+
"{}".format(non_recipe_requirements))
709+
710+
output = shprint(
711+
sh.bash, '-c',
712+
"echo -e '{}' > requirements.in && "
713+
"pip-compile -v --dry-run --annotation-style=line && "
714+
"rm requirements.in".format(
715+
'\n'.join(non_recipe_requirements)))
716+
717+
# Parse pip-compile output
718+
parsed_requirement_info_list = []
719+
for line in output.splitlines():
720+
match_data = re.match(
721+
r'^([\w.-]+)==(\d+(\.\d+)*).*'
722+
r'#\s+via\s+([\w\s,.-]+)', line)
723+
724+
if match_data:
725+
parent_requirements = match_data.group(4).split(', ')
726+
requirement_name = match_data.group(1)
727+
requirement_version = match_data.group(2)
728+
729+
# Requirement is a "non-recipe" one we started with.
730+
if '-r requirements.in' in parent_requirements:
731+
parent_requirements.remove('-r requirements.in')
732+
733+
parsed_requirement_info_list.append([
734+
requirement_name,
735+
requirement_version,
736+
parent_requirements])
737+
738+
info("Requirements obtained from pip-compile: "
739+
"{}".format(["{}=={}".format(x[0], x[1])
740+
for x in parsed_requirement_info_list]))
741+
742+
# Remove indirect requirements ultimately installed by a recipe
743+
original_parsed_requirement_count = -1
744+
while len(parsed_requirement_info_list) != \
745+
original_parsed_requirement_count:
746+
747+
original_parsed_requirement_count = \
748+
len(parsed_requirement_info_list)
749+
750+
for i, parsed_requirement_info in \
751+
enumerate(reversed(parsed_requirement_info_list)):
752+
753+
index = original_parsed_requirement_count - i - 1
754+
requirement_name, requirement_version, \
755+
parent_requirements = parsed_requirement_info
756+
757+
# If any parent requirement has a recipe, this
758+
# requirement ought also to be installed by it.
759+
# Hence, it's better not to add this requirement the
760+
# expanded list.
761+
parent_requirements_with_recipe = list(
762+
set(parent_requirements).intersection(
763+
set(all_recipes)))
764+
765+
# Any parent requirement removed for the expanded list
766+
# implies that it and its own requirements (including
767+
# this requirement) will be installed by a recipe.
768+
# Hence, it's better not to add this requirement the
769+
# expanded list.
770+
requirement_name_list = \
771+
[x[0] for x in parsed_requirement_info_list]
772+
parent_requirements_still_in_list = list(
773+
set(parent_requirements).intersection(
774+
set(requirement_name_list)))
775+
776+
is_ultimately_installed_by_a_recipe = \
777+
len(parent_requirements) and \
778+
(parent_requirements_with_recipe or
779+
len(parent_requirements_still_in_list) !=
780+
len(parent_requirements))
781+
782+
if is_ultimately_installed_by_a_recipe:
783+
info(
784+
'{} will be installed by a recipe. Removing '
785+
'it from requirement list expansion.'.format(
786+
requirement_name))
787+
del parsed_requirement_info_list[index]
788+
789+
for parsed_requirement_info in parsed_requirement_info_list:
790+
requirement_name, requirement_version, \
791+
parent_requirements = parsed_requirement_info
792+
793+
# If the requirement has a recipe, don't use specific
794+
# version constraints determined by pip-compile. Some
795+
# recipes may not support the specified version. Therefor,
796+
# it's probably safer to just let them use their default
797+
# version. User can still force the usage of specific
798+
# version by explicitly declaring it with --requirements.
799+
requirement_has_recipe = requirement_name in all_recipes
800+
requirement_str = \
801+
requirement_name if requirement_has_recipe else \
802+
'{}=={}'.format(requirement_name, requirement_version)
803+
804+
requirement_names_arg = re.sub(
805+
r'==\d+(\.\d+)*', '', args.requirements).split(',')
806+
807+
# This expansion was carried out based on "non-recipe"
808+
# requirements. Hence,the counter-part, requirements
809+
# with a recipe, may already be part of list.
810+
if requirement_name not in requirement_names_arg:
811+
args.requirements += ',' + requirement_str
812+
813+
# Handle specific version requirement constraints (e.g. foo==x.y)
814+
requirements = []
698815
for requirement in split_argument_list(args.requirements):
699816
if "==" in requirement:
700817
requirement, version = requirement.split(u"==", 1)
@@ -704,6 +821,9 @@ def add_parser(subparsers, *args, **kwargs):
704821
requirements.append(requirement)
705822
args.requirements = u",".join(requirements)
706823

824+
info('Expanded Requirements List: '
825+
'{}'.format(args.requirements.split(',')))
826+
707827
self.warn_on_deprecated_args(args)
708828

709829
self.storage_dir = args.storage_dir

‎setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
install_reqs = [
2323
'appdirs', 'colorama>=0.3.3', 'jinja2', 'six',
2424
'enum34; python_version<"3.4"', 'sh>=1.10; sys_platform!="nt"',
25-
'pep517<0.7.0', 'toml',
25+
'pep517<0.7.0', 'toml', 'pip-tools'
2626
]
2727
# (pep517 and toml are used by pythonpackage.py)
2828

‎tests/test_toolchain.py

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import io
2+
import os
23
import sys
34
from os.path import join
45
import pytest
@@ -131,6 +132,99 @@ def test_create_no_sdk_dir(self):
131132
assert ex_info.value.message == (
132133
'Android SDK dir was not specified, exiting.')
133134

135+
def test_create_with_complex_requirements(self):
136+
requirements = [
137+
'python3==3.8.10', # direct requirement with recipe using version constraint
138+
'pandas', # direct requirement with recipe (no version constraint)
139+
'mfpymake==1.2.2', # direct requirement without recipe using version constraint
140+
'telenium', # direct requirement without recipe (no version constraint)
141+
'numpy==1.21.4', # indirect requirement with recipe using version constraint
142+
'mako==1.1.5', # indirect requirement without recipe using version constraint
143+
# There's no reason to specify an indirect requirement unless we want to install a specific version.
144+
]
145+
argv = [
146+
'toolchain.py',
147+
'create',
148+
'--sdk-dir=/tmp/android-sdk',
149+
'--ndk-dir=/tmp/android-ndk',
150+
'--bootstrap=service_only',
151+
'--requirements={}'.format(','.join(requirements)),
152+
'--dist-name=test_toolchain',
153+
'--activity-class-name=abc.myapp.android.CustomPythonActivity',
154+
'--service-class-name=xyz.myapp.android.CustomPythonService',
155+
'--arch=arm64-v8a',
156+
'--arch=armeabi-v7a'
157+
]
158+
with patch_sys_argv(argv), mock.patch(
159+
'pythonforandroid.build.get_available_apis'
160+
) as m_get_available_apis, mock.patch(
161+
'pythonforandroid.build.get_toolchain_versions'
162+
) as m_get_toolchain_versions, mock.patch(
163+
'pythonforandroid.build.get_ndk_sysroot'
164+
) as m_get_ndk_sysroot, mock.patch(
165+
'pythonforandroid.toolchain.build_recipes'
166+
) as m_build_recipes, mock.patch(
167+
'pythonforandroid.bootstraps.service_only.'
168+
'ServiceOnlyBootstrap.assemble_distribution'
169+
) as m_run_distribute:
170+
m_get_available_apis.return_value = [27]
171+
m_get_toolchain_versions.return_value = (['4.9'], True)
172+
m_get_ndk_sysroot.return_value = (
173+
join(get_ndk_standalone("/tmp/android-ndk"), "sysroot"),
174+
True,
175+
)
176+
tchain = ToolchainCL()
177+
assert tchain.ctx.activity_class_name == 'abc.myapp.android.CustomPythonActivity'
178+
assert tchain.ctx.service_class_name == 'xyz.myapp.android.CustomPythonService'
179+
assert m_get_available_apis.call_args_list in [
180+
[mock.call('/tmp/android-sdk')], # linux case
181+
[mock.call('/private/tmp/android-sdk')] # macos case
182+
]
183+
for callargs in m_get_toolchain_versions.call_args_list:
184+
assert callargs in [
185+
mock.call("/tmp/android-ndk", mock.ANY), # linux case
186+
mock.call("/private/tmp/android-ndk", mock.ANY), # macos case
187+
]
188+
build_order = [
189+
'android', 'cython', 'genericndkbuild', 'hostpython3', 'libbz2',
190+
'libffi', 'liblzma', 'numpy', 'openssl', 'pandas', 'pyjnius',
191+
'python3', 'pytz', 'setuptools', 'six', 'sqlite3'
192+
]
193+
python_modules = [
194+
'certifi', 'charset-normalizer', 'cheroot', 'cherrypy',
195+
'idna', 'importlib-resources', 'jaraco.classes',
196+
'jaraco.collections', 'jaraco.functools', 'jaraco.text',
197+
'json-rpc', 'mako', 'markupsafe', 'mfpymake', 'more-itertools',
198+
'networkx', 'portend', 'python-dateutil', 'requests', 'telenium',
199+
'tempora', 'urllib3', 'werkzeug', 'ws4py', 'zc.lockfile', 'zipp'
200+
]
201+
context = mock.ANY
202+
project_dir = None
203+
204+
# The pip-compile tool used to expanded the list of requirements
205+
# doesn't always return results in the same order.
206+
# _Call object and its properties are immutable, so we create a
207+
# new one with the same values.
208+
m_build_recipes_call_args = mock.call(
209+
sorted(m_build_recipes.call_args_list[0][0][0]),
210+
sorted(m_build_recipes.call_args_list[0][0][1]),
211+
m_build_recipes.call_args_list[0][0][2],
212+
m_build_recipes.call_args_list[0][0][3],
213+
ignore_project_setup_py=m_build_recipes.call_args_list[0][1]['ignore_project_setup_py']
214+
)
215+
assert m_build_recipes_call_args == mock.call(
216+
sorted(build_order),
217+
sorted(python_modules),
218+
context,
219+
project_dir,
220+
ignore_project_setup_py=False
221+
)
222+
assert m_run_distribute.call_args_list == [mock.call()]
223+
assert 'VERSION_python3' in os.environ and os.environ['VERSION_python3'] == '3.8.10'
224+
assert 'VERSION_mfpymake' in os.environ and os.environ['VERSION_mfpymake'] == '1.2.2'
225+
assert 'VERSION_numpy' in os.environ and os.environ['VERSION_numpy'] == '1.21.4'
226+
assert 'VERSION_mako' in os.environ and os.environ['VERSION_mako'] == '1.1.5'
227+
134228
@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires python3")
135229
def test_recipes(self):
136230
"""

0 commit comments

Comments
 (0)
0