10000 Add automatic expansion of --requirement list · kivy/python-for-android@7647153 · GitHub
[go: up one dir, main page]
Skip to content

Commit 7647153

Browse files
pdallairpdallair
committed
Add automatic expansion of --requirement list
Related to issue #2529
1 parent 06dd1e0 commit 7647153

File tree

3 files changed

+398
-61
lines changed

3 files changed

+398
-61
lines changed

‎pythonforandroid/toolchain.py

Lines changed: 258 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
66
This module defines the entry point for command line and programmatic use.
77
"""
8-
98
from os import environ
109
from pythonforandroid import __version__
1110
from pythonforandroid.pythonpackage import get_dep_names_of_package
@@ -227,6 +226,249 @@ def split_argument_list(arg_list):
227226
return re.split(r'[ ,]+', arg_list)
228227

229228

229+
def __expand_requirements_arg_from_project_files(ctx, args):
230+
"""Parse additional requirements from setup.py or pyproject.toml file
231+
and add to --requirements arg if --use_setup_py argument was specified"""
232+
all_recipes = [
233+
recipe.lower() for recipe in
234+
set(Recipe.list_recipes(ctx))
235+
]
236+
237+
has_setup_py_or_toml = False
238+
if getattr(args, "private", None) is not None:
239+
project_dir = getattr(args, "private")
240+
has_setup_py = os.path.exists(
241+
os.path.join(project_dir, "setup.py"))
242+
has_toml = os.path.exists(
243+
os.path.join(project_dir, "pyproject.toml"))
244+
has_setup_py_or_toml = has_setup_py or has_toml
245+
246+
# Add dependencies from setup.py, but only if they are recipes
247+
# (because otherwise, setup.py itself will install them later)
248+
if has_setup_py_or_toml and getattr(args, "use_setup_py", False):
249+
try:
250+
info("Analyzing package dependencies. MAY TAKE A WHILE.")
251+
# Get all the dependencies corresponding to a recipe:
252+
dependencies = [
253+
dep.lower() for dep in
254+
get_dep_names_of_package(
255+
args.private,
256+
keep_version_pins=True,
257+
recursive=True,
258+
verbose=True,
259+
)
260+
]
261+
info("Dependencies obtained: " + str(dependencies))
262+
dependencies = set(dependencies).intersection(
263+
set(all_recipes)
264+
)
265+
266+
# Add dependencies to argument list:
267+
if len(dependencies) > 0:
268+
if len(args.requirements) > 0:
269+
args.requirements += u","
270+
args.requirements += u",".join(dependencies)
271+
272+
except ValueError:
273+
# Not a python package, apparently.
274+
warning(
275+
"Processing failed, is this project a valid "
276+
"package? Will continue WITHOUT setup.py deps."
277+
)
278+
279+
280+
def has_a_recipe(ctx, requirement):
281+
all_recipes = [
282+
recipe.lower() for recipe in
283+
set(Recipe.list_recipes(ctx))
284+
]
285+
requirement_name = re.sub(r'==\d+(\.\d+)*', '', requirement)
286+
return requirement_name in all_recipes
287+
288+
289+
def __run_pip_compile(requirements):
290+
return shprint(
291+
sh.bash, '-c',
292+
"echo -e '{}' > requirements.in && "
293+
"{} -m piptools compile --dry-run --annotation-style=line && "
294+
"rm requirements.in".format(
295+
'\n'.join(requirements), sys.executable))
296+
297+
298+
def __parse_pip_compile_output(output):
299+
parsed_requirement_info_list = []
300+
for line in output.splitlines():
301+
match_data = re.match(
302+
r'^([\w.-]+)==(\d+(\.\d+)*).*'
303+
r'#\s+via\s+([\w\s,.-]+)', line)
304+
305+
if match_data:
306+
parent_requirements = match_data.group(4).split(', ')
307+
requirement_name = match_data.group(1)
308+
requirement_version = match_data.group(2)
309+
310+
# Requirement is a "non-recipe" one we started with.
311+
if '-r requirements.in' in parent_requirements:
312+
parent_requirements.remove('-r requirements.in')
313+
314+
parsed_requirement_info_list.append([
315+
requirement_name,
316+
requirement_version,
317+
parent_requirements])
318+
319+
return parsed_requirement_info_list
320+
321+
322+
def __run_pip_compile_and_parse_output(requirements):
323+
return __parse_pip_compile_output(__run_pip_compile(requirements))
324+
325+
326+
def __is_requirement_installed_by_recipe(
327+
ctx, current_requirement_info, remaining_requirement_names):
328+
329+
requirement_name, requirement_version, \
330+
parent_requirements = current_requirement_info
331+
332+
# If any parent requirement has a recipe, this
333+
# requirement ought also to be installed by it.
334+
# Hence, it's better not to add this requirement the
335+
# expanded list.
336+
parent_requirements_with_recipe = [
337+
parent_requirement
338+
for parent_requirement in parent_requirements
339+
if has_a_recipe(ctx, parent_requirement)
340+
]
341+
342+
# Any parent requirement removed for the expanded list
343+
# implies that it and its own requirements (including
344+
# this requirement) will be installed by a recipe.
345+
# Hence, it's better not to add this requirement the
346+
# expanded list.
347+
parent_requirements_still_in_list = [
348+
parent_requirement
349+
for parent_requirement in parent_requirements
350+
if parent_requirement in remaining_requirement_names
351+
]
352+
353+
is_installed_by_a_recipe = \
354+
len(parent_requirements) and \
355+
(parent_requirements_with_recipe or
356+
len(parent_requirements_still_in_list) !=
357+
len(parent_requirements))
358+
359+
if is_installed_by_a_recipe:
360+
info('{}\n\t{}\n\t{}\n\t{}'.format(
361+
requirement_name,
362+
parent_requirements,
363+
parent_requirements_with_recipe,
364+
parent_requirements_still_in_list))
365+
366+
return is_installed_by_a_recipe
367+
368+
369+
def __prune_requirements_installed_by_recipe(ctx, requirement_info_list):
370+
original_requirement_count = -1
371+
372+
while len(requirement_info_list) != original_requirement_count:
373+
original_requirement_count = len(requirement_info_list)
374+
375+
for i, requirement_info in enumerate(reversed(requirement_info_list)):
376+
index = original_requirement_count - i - 1
377+
378+
remaining_requirement_names = \
379+
[x[0] for x in requirement_info_list]
380+
381+
if __is_requirement_installed_by_recipe(
382+
ctx, requirement_info, remaining_requirement_names):
383+
info('{} will be installed by a recipe. Removing '
384+
'it from requirement list expansion.'.format(
385+
requirement_info[0]))
386+
387+
del requirement_info_list[index]
388+
389+
390+
def __add_compiled_requirements_to_args(
391+
ctx, args, compiled_requirment_info_list):
392+
393+
for requirement_info in compiled_requirment_info_list:
394+
requirement_name, requirement_version, \
395+
parent_requirements = requirement_info
396+
397+
# If the requirement has a recipe, don't use specific
398+
# version constraints determined by pip-compile. Some
399+
# recipes may not support the specified version. Therefor,
400+
# it's probably safer to just let them use their default
401+
# version. User can still force the usage of specific
402+
# version by explicitly declaring it with --requirements.
403+
requirement_str = \
404+
requirement_name if has_a_recipe(ctx, requirement_name) else \
405+
'{}=={}'.format(requirement_name, requirement_version)
406+
407+
requirement_names_arg = split_argument_list(re.sub(
408+
r'==\d+(\.\d+)*', '', args.requirements))
409+
410+
# This expansion was carried out based on "non-recipe"
411+
# requirements. Hence,the counter-part, requirements
412+
# with a recipe, may already be part of list.
413+
if requirement_name not in requirement_names_arg:
414+
args.requirements += ',' + requirement_str
415+
416+
417+
def __expand_requirements_arg_from_pip_compile(ctx, args):
418+
"""Use pip-compile to generate requirement dependencies and add to
419+
--requirements command line argument."""
420+
421+
non_recipe_requirements = [
422+
requirement for requirement in split_argument_list(args.requirements)
423+
if not has_a_recipe(ctx, requirement)
424+
]
425+
non_recipe_requirements_regex = \
426+
r',?\s+' + r'|,?\s+'.join(non_recipe_requirements)
427+
args.requirements = \
428+
re.sub(non_recipe_requirements_regex, '', args.requirements)
429+
430+
# Compile "non-recipe" requirements' dependencies and add to
431+
# args.requirement. Otherwise, only recipe requirements'
432+
# dependencies would get installed.
433+
# More info https://github.com/kivy/python-for-android/issues/2529
434+
if non_recipe_requirements:
435+
info("Compiling dependencies for: "
436+
"{}".format(non_recipe_requirements))
437+
438+
parsed_requirement_info_list = \
439+
__run_pip_compile_and_parse_output(non_recipe_requirements)
440+
441+
info("Requirements obtained from pip-compile: "
442+
"{}".format(["{}=={}".format(x[0], x[1])
443+
for x in parsed_requirement_info_list]))
444+
445+
__prune_requirements_installed_by_recipe(
446+
ctx, parsed_requirement_info_list)
447+
448+
info("Requirements remaining after recipe dependency \"prunage\": "
449+
"{}".format(["{}=={}".format(x[0], x[1])
450+
for x in parsed_requirement_info_list]))
451+
452+
__add_compiled_requirements_to_args(
453+
ctx, args, parsed_requirement_info_list)
454+
455+
456+
def expand_requirements_args(ctx, args):
457+
"""Expand --requirements arg value to include what may have not
458+
been specified by the user, such as:
459+
* requirements specified in local project setup.py or pyproject.toml
460+
(if --use_setup_py was used)
461+
* indirect requirements (i.e., the requirements of our requirements).
462+
(e.g., if user specifies beautifulsoup4, the appropriate version of
463+
soupsieve is added).
464+
"""
465+
__expand_requirements_arg_from_project_files(ctx, args)
466+
__expand_requirements_arg_from_pip_compile(ctx, args)
467+
468+
info('Expanded Requirements List: '
469+
'{}'.format(split_argument_list(args.requirements)))
470+
471+
230472
class NoAbbrevParser(argparse.ArgumentParser):
231473
"""We want to disable argument abbreviation so as not to interfere
232474
with passing through arguments to build.py, but in python2 argparse
@@ -645,65 +887,6 @@ def add_parser(subparsers, *args, **kwargs):
645887
self.ctx.with_debug_symbols = getattr(
646888
args, "with_debug_symbols", False
647889
)
648-
649-
have_setup_py_or_similar = False
650-
if getattr(args, "private", None) is not None:
651-
project_dir = getattr(args, "private")
652-
if (os.path.exists(os.path.join(project_dir, "setup.py")) or
653-
os.path.exists(os.path.join(project_dir,
654-
"pyproject.toml"))):
655-
have_setup_py_or_similar = True
656-
657-
# Process requirements and put version in environ
658-
if hasattr(args, 'requirements'):
659-
requirements = []
660-
661-
# Add dependencies from setup.py, but only if they are recipes
662-
# (because otherwise, setup.py itself will install them later)
663-
if (have_setup_py_or_similar and
664-
getattr(args, "use_setup_py", False)):
665-
try:
666-
info("Analyzing package dependencies. MAY TAKE A WHILE.")
667-
# Get all the dependencies corresponding to a recipe:
668-
dependencies = [
669-
dep.lower() for dep in
670-
get_dep_names_of_package(
671-
args.private,
672-
keep_version_pins=True,
673-
recursive=True,
674-
verbose=True,
675-
)
676-
]
677-
info("Dependencies obtained: " + str(dependencies))
678-
all_recipes = [
679-
recipe.lower() for recipe in
680-
set(Recipe.list_recipes(self.ctx))
681-
]
682-
dependencies = set(dependencies).intersection(
683-
set(all_recipes)
684-
)
685-
# Add dependencies to argument list:
686-
if len(dependencies) > 0:
687-
if len(args.requirements) > 0:
688-
args.requirements += u","
689-
args.requirements += u",".join(dependencies)
690-
except ValueError:
691-
# Not a python package, apparently.
692-
warning(
693-
"Processing failed, is this project a valid "
694-
"package? Will continue WITHOUT setup.py deps."
695-
)
696-
697-
# Parse --requirements argument list:
698-
for requirement in split_argument_list(args.requirements):
699-
if "==" in requirement:
700-
requirement, version = requirement.split(u"==", 1)
701-
os.environ["VERSION_{}".format(requirement)] = version
702-
info('Recipe {}: version "{}" requested'.format(
703-
requirement, version))
704-
requirements.append(requirement)
705-
args.requirements = u",".join(requirements)
706-
707890
self.warn_on_deprecated_args(args)
708891

709892
self.storage_dir = args.storage_dir
@@ -723,6 +906,21 @@ def add_parser(subparsers, *args, **kwargs):
723906
self.ctx.activity_class_name = args.activity_class_name
724907
self.ctx.service_class_name = args.service_class_name
725908

909+
# Process requirements and put version in environ:
910+
if getattr(args, 'requirements', []):
911+
expand_requirements_args(self.ctx, args)
912+
913+
# Handle specific version requirement constraints (e.g. foo==x.y)
914+
requirements = []
915+
for requirement in split_argument_list(args.requirements):
916+
if "==" in requirement:
917+
requirement, version = requirement.split(u"==", 1)
918+
os.environ["VERSION_{}".format(requirement)] = version
919+
info('Recipe {}: version "{}" requested'.format(
920+
requirement, version))
921+
requirements.append(requirement)
922+
args.requirements = u",".join(requirements)
923+
726924
# Each subparser corresponds to a method
727925
command = args.subparser_name.replace('-', '_')
728926
getattr(self, command)(args)

‎setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
install_reqs = [
2323
'appdirs', 'colorama>=0.3.3', 'jinja2', 'six',
2424
'enum34; python_version<"3.4"', 'sh>=1.10; sys_platform!="nt"',
25-
'pep517<0.7.0', 'toml',
25+
'pep517<0.7.0', 'toml', 'pip-tools'
2626
]
2727
# (pep517 and toml are used by pythonpackage.py)
2828

0 commit comments

Comments
 (0)
0