From 3e39ca01b533670a1f106ab390e93cd704da207b Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sun, 23 Oct 2022 17:44:18 +0200 Subject: [PATCH] Fixes an issue regarding blacklist and bytecode compile + some cleanup --- doc/source/buildoptions.rst | 2 +- doc/source/launcher.rst | 2 +- pythonforandroid/bdistapk.py | 2 +- .../bootstraps/common/build/build.py | 82 ++++++++----------- .../common/build/jni/application/src/start.c | 12 +-- .../java/org/kivy/android/PythonActivity.java | 3 +- .../java/org/kivy/android/PythonActivity.java | 3 +- .../java/org/kivy/android/PythonActivity.java | 3 +- pythonforandroid/build.py | 1 - pythonforandroid/recipes/python3/__init__.py | 6 +- 10 files changed, 44 insertions(+), 72 deletions(-) diff --git a/doc/source/buildoptions.rst b/doc/source/buildoptions.rst index d9f97175a2..ebe2d14e4f 100644 --- a/doc/source/buildoptions.rst +++ b/doc/source/buildoptions.rst @@ -89,7 +89,7 @@ options (this list may not be exhaustive): - ``--service``: A service name and the Python script it should run. See :ref:`arbitrary_scripts_services`. - ``--add-source``: Add a source directory to the app's Java code. -- ``--no-compile-pyo``: Do not optimise .py files to .pyo. +- ``--no-byte-compile-python``: Skip byte compile for .py files. - ``--enable-androidx``: Enable AndroidX support library. diff --git a/doc/source/launcher.rst b/doc/source/launcher.rst index f863e493fc..cc4ccf6ea5 100644 --- a/doc/source/launcher.rst +++ b/doc/source/launcher.rst @@ -48,7 +48,7 @@ grab an old (cached) package instead of a fresh one. .. warning:: Do not use any of `--private`, `--public`, `--dir` or other arguments for - adding `main.py` or `main.pyo` to the app. The argument `--launcher` is + adding `main.py` or `main.pyc` to the app. The argument `--launcher` is above them and tells the p4a to build the launcher version of the APK. Usage diff --git a/pythonforandroid/bdistapk.py b/pythonforandroid/bdistapk.py index cc1487c85e..bcf77cd60d 100644 --- a/pythonforandroid/bdistapk.py +++ b/pythonforandroid/bdistapk.py @@ -108,7 +108,7 @@ def prepare_build_dir(self): makedirs(new_dir) print('Including {}'.format(filen)) copyfile(filen, join(bdist_dir, filen)) - if basename(filen) in ('main.py', 'main.pyo'): + if basename(filen) in ('main.py', 'main.pyc'): main_py_dirs.append(filen) # This feels ridiculous, but how else to define the main.py dir? diff --git a/pythonforandroid/bootstraps/common/build/build.py b/pythonforandroid/bootstraps/common/build/build.py index 6885a333df..213abfa8a5 100644 --- a/pythonforandroid/bootstraps/common/build/build.py +++ b/pythonforandroid/bootstraps/common/build/build.py @@ -40,10 +40,6 @@ def get_hostpython(): return get_dist_info_for('hostpython') -def get_python_version(): - return get_dist_info_for('python_version') - - def get_bootstrap_name(): return get_dist_info_for('bootstrap') @@ -58,7 +54,6 @@ def get_bootstrap_name(): curdir = dirname(__file__) PYTHON = get_hostpython() -PYTHON_VERSION = get_python_version() if PYTHON is not None and not exists(PYTHON): PYTHON = None @@ -73,17 +68,16 @@ def get_bootstrap_name(): '~', '*.bak', '*.swp', + + # Android artifacts + '*.apk', + '*.aab', ] -# pyc/py -if PYTHON is not None: - BLACKLIST_PATTERNS.append('*.py') WHITELIST_PATTERNS = [] if get_bootstrap_name() in ('sdl2', 'webview', 'service_only'): WHITELIST_PATTERNS.append('pyconfig.h') -python_files = [] - environment = jinja2.Environment(loader=jinja2.FileSystemLoader( join(curdir, 'templates'))) @@ -150,23 +144,11 @@ def listfiles(d): yield fn -def make_tar(tfn, source_dirs, ignore_path=[], optimize_python=True): +def make_tar(tfn, source_dirs, byte_compile_python=False, optimize_python=True): ''' Make a zip file `fn` from the contents of source_dis. ''' - # selector function - def select(fn): - rfn = realpath(fn) - for p in ignore_path: - if p.endswith('/'): - p = p[:-1] - if rfn.startswith(p): - return False - if rfn in python_files: - return False - return not is_blacklist(fn) - def clean(tinfo): """cleaning function (for reproducible builds)""" tinfo.uid = tinfo.gid = 0 @@ -178,9 +160,12 @@ def clean(tinfo): files = [] for sd in source_dirs: sd = realpath(sd) - compile_dir(sd, optimize_python=optimize_python) - files += [(x, relpath(realpath(x), sd)) for x in listfiles(sd) - if select(x)] + for fn in listfiles(sd): + if is_blacklist(fn): + continue + if fn.endswith('.py') and byte_compile_python: + fn = compile_py_file(fn, optimize_python=optimize_python) + files.append((fn, relpath(realpath(fn), sd))) files.sort() # deterministic # create tar.gz of thoses files @@ -210,18 +195,15 @@ def clean(tinfo): gf.close() -def compile_dir(dfn, optimize_python=True): +def compile_py_file(python_file, optimize_python=True): ''' - Compile *.py in directory `dfn` to *.pyo + Compile python_file to *.pyc and return the filename of the *.pyc file. ''' if PYTHON is None: return - if int(PYTHON_VERSION[0]) >= 3: - args = [PYTHON, '-m', 'compileall', '-b', '-f', dfn] - else: - args = [PYTHON, '-m', 'compileall', '-f', dfn] + args = [PYTHON, '-m', 'compileall', '-b', '-f', python_file] if optimize_python: # -OO = strip docstrings args.insert(1, '-OO') @@ -233,16 +215,18 @@ def compile_dir(dfn, optimize_python=True): 'error, see logs above') exit(1) + return ".".join([os.path.splitext(python_file)[0], "pyc"]) + def make_package(args): - # If no launcher is specified, require a main.py/main.pyo: + # If no launcher is specified, require a main.py/main.pyc: if (get_bootstrap_name() != "sdl" or args.launcher is None) and \ get_bootstrap_name() not in ["webview", "service_library"]: # (webview doesn't need an entrypoint, apparently) if args.private is None or ( not exists(join(realpath(args.private), 'main.py')) and - not exists(join(realpath(args.private), 'main.pyo'))): - print('''BUILD FAILURE: No main.py(o) found in your app directory. This + not exists(join(realpath(args.private), 'main.pyc'))): + print('''BUILD FAILURE: No main.py(c) found in your app directory. This file must exist to act as the entry point for you app. If your app is started by a file with a different name, rename it to main.py or add a main.py that loads it.''') @@ -290,7 +274,6 @@ def make_package(args): variants = [ copy_path, copy_path.partition(".")[0] + ".pyc", - copy_path.partition(".")[0] + ".pyo", ] # Check in all variants with all possible endings: for variant in variants: @@ -326,11 +309,17 @@ def make_package(args): for arch in get_dist_info_for("archs"): libs_dir = f"libs/{arch}" make_tar( - join(libs_dir, 'libpybundle.so'), [f'_python_bundle__{arch}'], args.ignore_path, - optimize_python=args.optimize_python) + join(libs_dir, "libpybundle.so"), + [f"_python_bundle__{arch}"], + byte_compile_python=args.byte_compile_python, + optimize_python=args.optimize_python, + ) make_tar( - join(assets_dir, 'private.tar'), private_tar_dirs, args.ignore_path, - optimize_python=args.optimize_python) + join(assets_dir, "private.tar"), + private_tar_dirs, + byte_compile_python=args.byte_compile_python, + optimize_python=args.optimize_python, + ) finally: for directory in _temp_dirs_to_clean: shutil.rmtree(directory) @@ -824,8 +813,6 @@ def parse_args_and_make_package(args=None): ap.add_argument('--try-system-python-compile', dest='try_system_python_compile', action='store_true', help='Use the system python during compileall if possible.') - ap.add_argument('--no-compile-pyo', dest='no_compile_pyo', action='store_true', - help='Do not optimise .py files to .pyo.') ap.add_argument('--sign', action='store_true', help=('Try to sign the APK with your credentials. You must set ' 'the appropriate environment variables.')) @@ -844,9 +831,12 @@ def parse_args_and_make_package(args=None): 'files (containing your main.py entrypoint). ' 'See https://developer.android.com/guide/topics/data/' 'autobackup#IncludingFiles for more information')) + ap.add_argument('--no-byte-compile-python', dest='byte_compile_python', + action='store_false', default=True, + help='Skip byte compile for .py files.') ap.add_argument('--no-optimize-python', dest='optimize_python', action='store_false', default=True, - help=('Whether to compile to optimised .pyo files, using -OO ' + help=('Whether to compile to optimised .pyc files, using -OO ' '(strips docstrings and asserts)')) ap.add_argument('--extra-manifest-xml', default='', help=('Extra xml to write directly inside the element of' @@ -881,8 +871,6 @@ def _read_configuration(): args = ap.parse_args(args) - args.ignore_path = [] - if args.name and args.name[0] == '"' and args.name[-1] == '"': args.name = args.name[1:-1] @@ -925,10 +913,6 @@ def _read_configuration(): else: PYTHON = python_executable - if args.no_compile_pyo: - PYTHON = None - BLACKLIST_PATTERNS.remove('*.py') - if args.blacklist: with open(args.blacklist) as fd: patterns = [x.strip() for x in fd.read().splitlines() diff --git a/pythonforandroid/bootstraps/common/build/jni/application/src/start.c b/pythonforandroid/bootstraps/common/build/jni/application/src/start.c index d46b1fae0d..bad52186f8 100644 --- a/pythonforandroid/bootstraps/common/build/jni/application/src/start.c +++ b/pythonforandroid/bootstraps/common/build/jni/application/src/start.c @@ -251,14 +251,10 @@ int main(int argc, char *argv[]) { */ LOGP("Run user program, change dir and execute entrypoint"); - /* Get the entrypoint, search the .pyo then .py + /* Get the entrypoint, search the .pyc then .py */ char *dot = strrchr(env_entrypoint, '.'); -#if PY_MAJOR_VERSION > 2 char *ext = ".pyc"; -#else - char *ext = ".pyo"; -#endif if (dot <= 0) { LOGP("Invalid entrypoint, abort."); return -1; @@ -281,14 +277,10 @@ int main(int argc, char *argv[]) { strcpy(entrypoint, env_entrypoint); } } else if (!strcmp(dot, ".py")) { - /* if .py is passed, check the pyo version first */ + /* if .py is passed, check the pyc version first */ strcpy(entrypoint, env_entrypoint); entrypoint[strlen(env_entrypoint) + 1] = '\0'; -#if PY_MAJOR_VERSION > 2 entrypoint[strlen(env_entrypoint)] = 'c'; -#else - entrypoint[strlen(env_entrypoint)] = 'o'; -#endif if (!file_exists(entrypoint)) { /* fallback on pure python version */ if (!file_exists(env_entrypoint)) { diff --git a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java index ded381f161..361975a4cf 100644 --- a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java @@ -419,11 +419,10 @@ public void run() { } public String getEntryPoint(String search_dir) { - /* Get the main file (.pyc|.pyo|.py) depending on if we + /* Get the main file (.pyc|.py) depending on if we * have a compiled version or not. */ List entryPoints = new ArrayList(); - entryPoints.add("main.pyo"); // python 2 compiled files entryPoints.add("main.pyc"); // python 3 compiled files for (String value : entryPoints) { File mainFile = new File(search_dir + "/" + value); diff --git a/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java index 87ea061c41..57112dd555 100644 --- a/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java @@ -46,11 +46,10 @@ public String getAppRoot() { } public String getEntryPoint(String search_dir) { - /* Get the main file (.pyc|.pyo|.py) depending on if we + /* Get the main file (.pyc|.py) depending on if we * have a compiled version or not. */ List entryPoints = new ArrayList(); - entryPoints.add("main.pyo"); // python 2 compiled files entryPoints.add("main.pyc"); // python 3 compiled files for (String value : entryPoints) { File mainFile = new File(search_dir + "/" + value); diff --git a/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java index 8aa308b24a..2f0afdc6f4 100644 --- a/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java @@ -68,11 +68,10 @@ public String getAppRoot() { } public String getEntryPoint(String search_dir) { - /* Get the main file (.pyc|.pyo|.py) depending on if we + /* Get the main file (.pyc|.py) depending on if we * have a compiled version or not. */ List entryPoints = new ArrayList(); - entryPoints.add("main.pyo"); // python 2 compiled files entryPoints.add("main.pyc"); // python 3 compiled files for (String value : entryPoints) { File mainFile = new File(search_dir + "/" + value); diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index 42b6b52add..645b368d00 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -456,7 +456,6 @@ def has_package(self, name, arch=None): return (exists(join(site_packages_dir, name)) or exists(join(site_packages_dir, name + '.py')) or exists(join(site_packages_dir, name + '.pyc')) or - exists(join(site_packages_dir, name + '.pyo')) or exists(join(site_packages_dir, name + '.so')) or glob.glob(join(site_packages_dir, name + '-*.egg'))) diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index c3c28c70fb..5894b3f9b0 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -364,11 +364,11 @@ def create_python_bundle(self, dirn, arch): self.major_minor_version_string )) - # Compile to *.pyc/*.pyo the python modules + # Compile to *.pyc the python modules self.compile_python_files(modules_build_dir) - # Compile to *.pyc/*.pyo the standard python library + # Compile to *.pyc the standard python library self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib')) - # Compile to *.pyc/*.pyo the other python packages (site-packages) + # Compile to *.pyc the other python packages (site-packages) self.compile_python_files(self.ctx.get_python_install_dir(arch.arch)) # Bundle compiled python modules to a folder