8000 Add automatic expansion of --requirement list · kivy/python-for-android@b822898 · GitHub
[go: up one dir, main page]
Skip to content

Commit b822898

Browse files
pdallairpdallair
committed
Add automatic expansion of --requirement list
Related to issue #2529
1 parent 06dd1e0 commit b822898

File tree

3 files changed

+400
-61
lines changed

3 files changed

+400
-61
lines changed

‎pythonforandroid/toolchain.py

Lines changed: 260 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
66
This module defines the entry point for command line and programmatic use.
77
"""
8-
98
from os import environ
109
from pythonforandroid import __version__
1110
from pythonforandroid.pythonpackage import get_dep_names_of_package
@@ -227,6 +226,251 @@ def split_argument_list(arg_list):
227226
return re.split(r'[ ,]+', arg_list)
228227

229228

229+
def __expand_requirements_arg_from_project_files(ctx, args):
230+
"""Parse additional requirements from setup.py or pyproject.toml file
231+
and add to --requirements arg if --use_setup_py argument was specified"""
232+
has_setup_py_or_toml = False
233+
if getattr(args, "private", None) is not None:
234+
project_dir = getattr(args, "private")
235+
has_setup_py = os.path.exists(
236+
os.path.join(project_dir, "setup.py"))
237+
has_toml = os.path.exists(
238+
os.path.join(project_dir, "pyproject.toml"))
239+
has_setup_py_or_toml = has_setup_py or has_toml
240+
241+
# Add dependencies from setup.py, but only if they are recipes
242+
# (because otherwise, setup.py itself will install them later)
243+
if has_setup_py_or_toml and getattr(args, "use_setup_py", False):
244+
try:
245+
info("Analyzing package dependencies. MAY TAKE A WHILE.")
246+
# Get all the dependencies corresponding to a recipe:
247+
dependencies = [
248+
dep.lower() for dep in
249+
get_dep_names_of_package(
250+
args.private,
251+
keep_version_pins=True,
252+
recursive=True,
253+
verbose=True,
254+
)
255+
]
256+
info("Dependencies obtained: " + str(dependencies))
257+
dependencies = [
258+
dependencie for dependencie in dependencies
259+
if has_a_recipe(ctx, dependencie)
260+
]
261+
262+
# Add dependencies to argument list:
263+
if len(dependencies) > 0:
264+
if len(args.requirements) > 0:
265+
args.requirements += u","
266+
args.requirements += u",".join(dependencies)
267+
268+
except ValueError:
269+
# Not a python package, apparently.
270+
warning(
271+
"Processing failed, is this project a valid "
272+
"package? Will continue WITHOUT setup.py deps."
273+
)
274+
275+
276+
def has_a_recipe(ctx, requirement):
277+
all_recipes = [
278+
recipe.lower() for recipe in
279+
set(Recipe.list_recipes(ctx))
280+
]
281+
requirement_name = requirement.split('==')[0]
282+
return requirement_name in all_recipes
283+
284+
285+
def __run_pip_compile(requirements):
286+
return shprint(
287+
sh.bash, '-c',
288+
"echo -e '{}' > requirements.in && "
289+
"{} -m piptools compile --dry-run --annotation-style=line && "
290+
"rm requirements.in".format(
291+
'\n'.join(requirements), sys.executable))
292+
293+
294+
def __parse_pip_compile_output(output):
295+
parsed_requirement_info_list = []
296+
for line in output.splitlines():
297+
match_data = re.match(
298+
r'^([\w.-]+)(==([^\s]+)|\s+@\s+([^\s]+)).*'
299+
r'#\s+via\s+([\w\s,.-]+)', line)
300+
301+
if match_data:
302+
parent_requirements = match_data.group(5).split(', ')
303+
requirement_name = match_data.group(1)
304+
requirement_version = match_data.group(3)
305+
requirement_url = match_data.group(4)
306+
307+
# Requirement is a "non-recipe" one we started with.
308+
if '-r requirements.in' in parent_requirements:
309+
parent_requirements.remove('-r requirements.in')
310+
311+
if requirement_url:
312+
# For wtv reason, pip-compile output truncates the slashes.
313+
requirement_url = requirement_url.replace('file:/', 'file:///')
314+
315+
parsed_requirement_info_list.append([
316+
requirement_name,
317+
requirement_version,
318+
requirement_url,
319+
parent_requirements])
320+
321+
return parsed_requirement_info_list
322+
323+
324+
def __run_pip_compile_and_parse_output(requirements):
325+
return __parse_pip_compile_output(__run_pip_compile(requirements))
326+
327+
328+
def __is_requirement_indirectly_installed_by_recipe(
329+
ctx, current_requirement_info, remaining_requirement_names):
330+
331+
requirement_name, requirement_version, \
332+
requirement_url, parent_requirements = current_requirement_info
333+
334+
# If any parent requirement has a recipe, this
335+
# requirement ought also to be installed by it.
336+
# Hence, it's better not to add this requirement the
337+
# expanded list.
338+
parent_requirements_with_recipe = [
339+
parent_requirement
340+
for parent_requirement in parent_requirements
341+
if has_a_recipe(ctx, parent_requirement)
342+
]
343+
344+
# Any parent requirement removed for the expanded list
345+
# implies that it and its own requirements (including
346+
# this requirement) will be installed by a recipe.
347+
# Hence, it's better not to add this requirement the
348+
# expanded list.
349+
parent_requirements_not_in_list = [
350+
parent_requirement
351+
for parent_requirement in parent_requirements
352+
if parent_requirement not in remaining_requirement_names
353+
]
354+
355+
is_indrectly_installed_by_a_recipe = \
356+
len(parent_requirements) and \
357+
(parent_requirements_with_recipe or parent_requirements_not_in_list)
358+
359+
if is_indrectly_installed_by_a_recipe and parent_requirements_with_recipe:
360+
info('Concluding that {} is installed by {} recipe(s).'.format(requirement_name, parent_requirements_with_recipe))
361+
362+
elif is_indrectly_installed_by_a_recipe and parent_requirements_not_in_list:
363+
info('Previously concluded that {} is/are installed by recipe(s). Consequently, so will {}.'.format(
364+
parent_requirements_not_in_list, requirement_name))
365+
366+
return is_indrectly_installed_by_a_recipe
367+
368+
369+
def __prune_requirements_installed_by_recipe(ctx, requirement_info_list):
370+
original_requirement_count = -1
371+
372+
while len(requirement_info_list) != original_requirement_count:
373+
original_requirement_count = len(requirement_info_list)
374+
375+
for i, requirement_info in enumerate(reversed(requirement_info_list)):
376+
index = original_requirement_count - i - 1
377+
378+
remaining_requirement_names = \
379+
[x[0] for x in requirement_info_list]
380+
381+
if __is_requirement_indirectly_installed_by_recipe(
382+
ctx, requirement_info, remaining_requirement_names):
383+
info('\tRemoving {} from requirement list expansion.'.format(
384+
requirement_info[0]))
385+
386+
del requirement_info_list[index]
387+
388+
389+
def __add_compiled_requirements_to_args(
390+
ctx, args, compiled_requirment_info_list):
391+
392+
for requirement_info in compiled_requirment_info_list:
393+
requirement_name, requirement_version, \
394+
requirement_url, parent_requirements = requirement_info
395+
396+
# If the requirement has a recipe, don't use specific
397+
# version constraints determined by pip-compile. Some
398+
# recipes may not support the specified version. Therefor,
399+
# it's probably safer to just let them use their default
400+
# version. User can still force the usage of specific
401+
# version by explicitly declaring it with --requirements.
402+
requirement_str = \
403+
requirement_name if has_a_recipe(ctx, requirement_name) else \
404+
'{}=={}'.format(requirement_name, requirement_version)
405+
406+
requirement_names_arg = split_argument_list(re.sub(
407+
r'==[^\s,]+', '', args.requirements))
408+
409+
# This expansion was carried out based on "non-recipe"
410+
# requirements. Hence, the counter-part, requirements
411+
# with a recipe, may already be part of list.
412+
if not (requirement_url or requirement_name in requirement_names_arg):
413+
args.requirements += ',' + requirement_str
414+
415+
elif requirement_url and requirement_url not in requirement_names_arg:
416+
args.requirements += ',' + requirement_url
417+
418+
419+
def __expand_requirements_arg_from_pip_compile(ctx, args):
420+
"""Use pip-compile to generate requirement dependencies and add to
421+
--requirements command line argument."""
422+
423+
non_recipe_requirements = [
424+
requirement for requirement in split_argument_list(args.requirements)
425+
if not has_a_recipe(ctx, requirement)
426+
]
427+
non_recipe_requirements_regex = \
428+
r',?\s+' + r'|,?\s+'.join(non_recipe_requirements)
429+
args.requirements = \
430+
re.sub(non_recipe_requirements_regex, '', args.requirements)
431+
432+
# Compile "non-recipe" requirements' dependencies and add to
433+
# args.requirement. Otherwise, only recipe requirements'
434+
# dependencies would get installed.
435+
# More info https://github.com/kivy/python-for-android/issues/2529
436+
if non_recipe_requirements:
437+
info("Compiling dependencies for: "
438+
"{}".format(non_recipe_requirements))
439+
440+
parsed_requirement_info_list = \
441+
__run_pip_compile_and_parse_output(non_recipe_requirements)
442+
443+
info("Requirements obtained from pip-compile: "
444+
"{}".format(["{}{}".format(x[0], '==' + x[1] if x[1] else '[' + x[2] + ']')
445+
for x in parsed_requirement_info_list]))
446+
447+
__prune_requirements_installed_by_recipe(
448+
ctx, parsed_requirement_info_list)
449+
450+
info("Requirements remaining after recipe dependency \"prunage\": "
451+
"{}".format(["{}{}".format(x[0], '==' + x[1] if x[1] else '[' + x[2] + ']')
452+
for x in parsed_requirement_info_list]))
453+
454+
__add_compiled_requirements_to_args(
455+
ctx, args, parsed_requirement_info_list)
456+
457+
458+
def expand_requirements_args(ctx, args):
459+
"""Expand --requirements arg value to include what may have not
460+
been specified by the user, such as:
461+
* requirements specified in local project setup.py or pyproject.toml
462+
(if --use_setup_py was used)
463+
* indirect requirements (i.e., the requirements of our requirements).
464+
(e.g., if user specifies beautifulsoup4, the appropriate version of
465+
soupsieve is added).
466+
"""
467+
__expand_requirements_arg_from_project_files(ctx, args)
468+
__expand_requirements_arg_from_pip_compile(ctx, args)
469+
470+
info('Expanded Requirements List: '
471+
'{}'.format(split_argument_list(args.requirements)))
472+
473+
230474
class NoAbbrevParser(argparse.ArgumentParser):
231475
"""We want to disable argument abbreviation so as not to interfere
232476
with passing through arguments to build.py, but in python2 argparse
@@ -645,65 +889,6 @@ def add_parser(subparsers, *args, **kwargs):
645889
self.ctx.with_debug_symbols = getattr(
646890
args, "with_debug_symbols", False
647891
)
648-
649-
have_setup_py_or_similar = False
650-
if getattr(args, "private", None) is not None:
651-
project_dir = getattr(args, "private")
652-
if (os.path.exists(os.path.join(project_dir, "setup.py")) or
653-
os.path.exists(os.path.join(project_dir,
654-
"pyproject.toml"))):
655-
have_setup_py_or_similar = True
656-
657-
# Process requirements and put version in environ
658-
if hasattr(args, 'requirements'):
659-
requirements = []
660-
661-
# Add dependencies from setup.py, but only if they are recipes
662-
# (because otherwise, setup.py itself will install them later)
663-
if (have_setup_py_or_similar and
664-
getattr(args, "use_setup_py", False)):
665-
try:
666-
info("Analyzing package dependencies. MAY TAKE A WHILE.")
667-
# Get all the dependencies corresponding to a recipe:
668-
dependencies = [
669-
dep.lower() for dep in
670-
get_dep_names_of_package(
671-
args.private,
672-
keep_version_pins=True,
673-
recursive=True,
674-
verbose=True,
675-
)
676-
]
677-
info("Dependencies obtained: " + str(dependencies))
678-
all_recipes = [
679-
recipe.lower() for recipe in
680-
set(Recipe.list_recipes(self.ctx))
681-
]
682-
dependencies = set(dependencies).intersection(
683-
set(all_recipes)
684-
)
685-
# Add dependencies to argument list:
686-
if len(dependencies) > 0:
687-
if len(args.requirements) > 0:
688-
args.requirements += u","
689-
args.requirements += u",".join(dependencies)
690-
except ValueError:
691-
# Not a python package, apparently.
692-
warning(
693-
"Processing failed, is this project a valid "
694-
"package? Will continue WITHOUT setup.py deps."
695-
)
696-
697-
# Parse --requirements argument list:
698-
for requirement in split_argument_list(args.requirements):
699-
if "==" in requirement:
700-
requirement, version = requirement.split(u"==", 1)
701-
os.environ["VERSION_{}".format(requirement)] = version
702-
info('Recipe {}: version "{}" requested'.format(
703-
requirement, version))
704-
requirements.append(requirement)
705-
args.requirements = u",".join(requirements)
706-
707892
self.warn_on_deprecated_args(args)
708893

709894
self.storage_dir = args.storage_dir
@@ -723,6 +908,21 @@ def add_parser(subparsers, *args, **kwargs):
723908
self.ctx.activity_class_name = args.activity_class_name
724909
self.ctx.service_class_name = args.service_class_name
725910

911+
# Process requirements and put version in environ:
912+
if getattr(args, 'requirements', []):
913+
expand_requirements_args(self.ctx, args)
914+
915+
# Handle specific version requirement constraints (e.g. foo==x.y)
916+
requirements = []
917+
for requirement in split_argument_list(args.requirements):
918+
if "==" in requirement:
919+
requirement, version = requirement.split(u"==", 1)
920+
os.environ["VERSION_{}".format(requirement)] = version
921+
info('Recipe {}: version "{}" requested'.format(
922+
requirement, version))
923+
requirements.append(requirement)
924+
args.requirements = u",".join(requirements)
925+
726926
# Each subparser corresponds to a method
727927
command = args.subparser_name.replace('-', '_')
728928
getattr(self, command)(args)

‎setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
install_reqs = [
2323
'appdirs', 'colorama>=0.3.3', 'jinja2', 'six',
2424
'enum34; python_version<"3.4"', 'sh>=1.10; sys_platform!="nt"',
25-
'pep517<0.7.0', 'toml',
25+
'pep517<0.7.0', 'toml', 'pip-tools'
2626
]
2727
# (pep517 and toml are used by pythonpackage.py)
2828

0 commit comments

Comments
 (0)
0