From 71cccb3a12ebcf7c79ce8bf695da13dae02a7be6 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 24 Apr 2025 14:07:57 +0100 Subject: [PATCH 01/13] Add `android.py env` command --- Android/android.py | 97 ++++++++++++++++++++++++++++++---------------- 1 file changed, 64 insertions(+), 33 deletions(-) diff --git a/Android/android.py b/Android/android.py index 1b20820b784371..b60995a7897264 100755 --- a/Android/android.py +++ b/Android/android.py @@ -22,9 +22,13 @@ SCRIPT_NAME = Path(__file__).name ANDROID_DIR = Path(__file__).resolve().parent -CHECKOUT = ANDROID_DIR.parent +PYTHON_DIR = ANDROID_DIR.parent +in_source_tree = ( + ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists() +) + TESTBED_DIR = ANDROID_DIR / "testbed" -CROSS_BUILD_DIR = CHECKOUT / "cross-build" +CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" HOSTS = ["aarch64-linux-android", "x86_64-linux-android"] APP_ID = "org.python.testbed" @@ -74,41 +78,62 @@ def subdir(*parts, create=False): def run(command, *, host=None, env=None, log=True, **kwargs): kwargs.setdefault("check", True) + if env is None: env = os.environ.copy() - original_env = env.copy() - if host: - env_script = ANDROID_DIR / "android-env.sh" - env_output = subprocess.run( - f"set -eu; " - f"HOST={host}; " - f"PREFIX={subdir(host)}/prefix; " - f". {env_script}; " - f"export", - check=True, shell=True, text=True, stdout=subprocess.PIPE - ).stdout - - for line in env_output.splitlines(): - # We don't require every line to match, as there may be some other - # output from installing the NDK. - if match := re.search( - "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line - ): - key, value = match[2], match[3] - if env.get(key) != value: - print(line) - env[key] = value - - if env == original_env: - raise ValueError(f"Found no variables in {env_script.name} output:\n" - + env_output) + env.update(android_env(host)) if log: print(">", " ".join(map(str, command))) return subprocess.run(command, env=env, **kwargs) +def print_env(context): + android_env(getattr(context, "host", None)) + + +def android_env(host): + if host: + prefix = subdir(host) / "prefix" + else: + prefix = ANDROID_DIR / "prefix" + sysconfigdata_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py") + host = re.fullmatch( + r"_sysconfigdata__android_(.+).py", next(sysconfigdata_files).name + )[1] + + env_script = ANDROID_DIR / "android-env.sh" + env_output = subprocess.run( + f"set -eu; " + f"export HOST={host}; " + f"PREFIX={prefix}; " + f". {env_script}; " + f"export", + check=True, shell=True, capture_output=True, text=True, + ).stdout + + env = {} + for line in env_output.splitlines(): + # We don't require every line to match, as there may be some other + # output from installing the NDK. + if match := re.search( + "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line + ): + key, value = match[2], match[3] + if os.environ.get(key) != value: + env[key] = value + + if not env: + raise ValueError(f"Found no variables in {env_script.name} output:\n" + + env_output) + + # Format the environment so it can be pasted into a shell. + for key, value in sorted(env.items()): + print(f"export {key}={shlex.quote(value)}") + return env + + def build_python_path(): """The path to the build Python binary.""" build_dir = subdir("build") @@ -127,7 +152,7 @@ def configure_build_python(context): clean("build") os.chdir(subdir("build", create=True)) - command = [relpath(CHECKOUT / "configure")] + command = [relpath(PYTHON_DIR / "configure")] if context.args: command.extend(context.args) run(command) @@ -168,7 +193,7 @@ def configure_host_python(context): os.chdir(host_dir) command = [ # Basic cross-compiling configuration - relpath(CHECKOUT / "configure"), + relpath(PYTHON_DIR / "configure"), f"--host={context.host}", f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", f"--with-build-python={build_python_path()}", @@ -624,8 +649,7 @@ def parse_args(): configure_build = subcommands.add_parser("configure-build", help="Run `configure` for the " "build Python") - make_build = subcommands.add_parser("make-build", - help="Run `make` for the build Python") + subcommands.add_parser("make-build", help="Run `make` for the build Python") configure_host = subcommands.add_parser("configure-host", help="Run `configure` for Android") make_host = subcommands.add_parser("make-host", @@ -637,16 +661,22 @@ def parse_args(): test = subcommands.add_parser( "test", help="Run the test suite") package = subcommands.add_parser("package", help="Make a release package") + env = subcommands.add_parser("env", help="Print environment variables") # Common arguments for subcommand in build, configure_build, configure_host: subcommand.add_argument( "--clean", action="store_true", default=False, dest="clean", help="Delete the relevant build and prefix directories first") - for subcommand in [build, configure_host, make_host, package]: + + host_commands = [build, configure_host, make_host, package] + if in_source_tree: + host_commands.append(env) + for subcommand in host_commands: subcommand.add_argument( "host", metavar="HOST", choices=HOSTS, help="Host triplet: choices=[%(choices)s]") + for subcommand in build, configure_build, configure_host: subcommand.add_argument("args", nargs="*", help="Extra arguments to pass to `configure`") @@ -690,6 +720,7 @@ def main(): "build-testbed": build_testbed, "test": run_testbed, "package": package, + "env": print_env, } try: From e3d27acaa6619ea3a5c509657c6d9b693456eef2 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 24 Apr 2025 17:14:42 +0100 Subject: [PATCH 02/13] Miscellaneous cleanups --- Android/README.md | 8 +++++--- Android/android.py | 20 +++++++++----------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Android/README.md b/Android/README.md index 789bcbe5edff44..6cabd6ba5d6844 100644 --- a/Android/README.md +++ b/Android/README.md @@ -25,11 +25,13 @@ it: `android-sdk/cmdline-tools/latest`. * `export ANDROID_HOME=/path/to/android-sdk` -The `android.py` script also requires the following commands to be on the `PATH`: +The `android.py` script will automatically use the SDK's `sdkmanager` to install +any packages it needs. + +The script also requires the following commands to be on the `PATH`: * `curl` * `java` (or set the `JAVA_HOME` environment variable) -* `tar` ## Building @@ -97,7 +99,7 @@ similar to the `Android` directory of the CPython source tree. The Python test suite can be run on Linux, macOS, or Windows: * On Linux, the emulator needs access to the KVM virtualization interface, and - a DISPLAY environment variable pointing at an X server. + a DISPLAY environment variable pointing at an X server. Xvfb is acceptable. The test suite can usually be run on a device with 2 GB of RAM, but this is borderline, so you may need to increase it to 4 GB. As of Android diff --git a/Android/android.py b/Android/android.py index b60995a7897264..89d927c5fc5dd3 100755 --- a/Android/android.py +++ b/Android/android.py @@ -163,19 +163,19 @@ def make_build_python(context): run(["make", "-j", str(os.cpu_count())]) -def unpack_deps(host): +def unpack_deps(host, prefix_dir): deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4", "sqlite-3.49.1-0", "xz-5.4.6-1"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") - run(["tar", "-xf", filename]) + shutil.unpack_archive(filename, prefix_dir) os.remove(filename) def download(url, target_dir="."): out_path = f"{target_dir}/{basename(url)}" - run(["curl", "-Lf", "-o", out_path, url]) + run(["curl", "-Lf", "--retry", "5", "--retry-all-errors", "-o", out_path, url]) return out_path @@ -187,8 +187,7 @@ def configure_host_python(context): prefix_dir = host_dir / "prefix" if not prefix_dir.exists(): prefix_dir.mkdir() - os.chdir(prefix_dir) - unpack_deps(context.host) + unpack_deps(context.host, prefix_dir) os.chdir(host_dir) command = [ @@ -266,16 +265,15 @@ def setup_sdk(): # the Gradle wrapper is not included in the CPython repository. Instead, we # extract it from the Gradle GitHub repository. def setup_testbed(): - # The Gradle version used for the build is specified in - # testbed/gradle/wrapper/gradle-wrapper.properties. This wrapper version - # doesn't need to match, as any version of the wrapper can download any - # version of Gradle. - version = "8.9.0" paths = ["gradlew", "gradlew.bat", "gradle/wrapper/gradle-wrapper.jar"] - if all((TESTBED_DIR / path).exists() for path in paths): return + # The wrapper version isn't important, as any version of the wrapper can + # download any version of Gradle. The Gradle version actually used for the + # build is specified in testbed/gradle/wrapper/gradle-wrapper.properties. + version = "8.9.0" + for path in paths: out_path = TESTBED_DIR / path out_path.parent.mkdir(exist_ok=True) From 24b082f74ea346408effe7b9546ecf4f52737fe3 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 24 Apr 2025 17:29:13 +0100 Subject: [PATCH 03/13] Prefer 'encoding' to 'text' Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Android/android.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Android/android.py b/Android/android.py index 89d927c5fc5dd3..9845a4f22288c1 100755 --- a/Android/android.py +++ b/Android/android.py @@ -110,7 +110,7 @@ def android_env(host): f"PREFIX={prefix}; " f". {env_script}; " f"export", - check=True, shell=True, capture_output=True, text=True, + check=True, shell=True, capture_output=True, encoding='utf-8', ).stdout env = {} From b7461d3326fc96287d9fbb1e18908cc6da3483e6 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 24 Apr 2025 17:53:42 +0100 Subject: [PATCH 04/13] Clarify sysconfigdata file discovery --- Android/android.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Android/android.py b/Android/android.py index 9845a4f22288c1..b410f3e2eec22f 100755 --- a/Android/android.py +++ b/Android/android.py @@ -98,10 +98,9 @@ def android_env(host): prefix = subdir(host) / "prefix" else: prefix = ANDROID_DIR / "prefix" - sysconfigdata_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py") - host = re.fullmatch( - r"_sysconfigdata__android_(.+).py", next(sysconfigdata_files).name - )[1] + sysconfig_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py") + sysconfig_filename = next(sysconfig_files).name + host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1] env_script = ANDROID_DIR / "android-env.sh" env_output = subprocess.run( From c7cdb984dfc413313eeea10f57905d008d0507f6 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 25 Apr 2025 16:54:09 +0100 Subject: [PATCH 05/13] Rename api_level environment variable to ANDROID_API_LEVEL --- Android/android-env.sh | 4 ++-- Android/testbed/app/build.gradle.kts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Android/android-env.sh b/Android/android-env.sh index bab4130c9e92d0..ae1385034a37f2 100644 --- a/Android/android-env.sh +++ b/Android/android-env.sh @@ -3,7 +3,7 @@ : "${HOST:?}" # GNU target triplet # You may also override the following: -: "${api_level:=24}" # Minimum Android API level the build will run on +: "${ANDROID_API_LEVEL:=24}" # Minimum Android API level the build will run on : "${PREFIX:-}" # Path in which to find required libraries @@ -43,7 +43,7 @@ fi toolchain=$(echo "$ndk"/toolchains/llvm/prebuilt/*) export AR="$toolchain/bin/llvm-ar" export AS="$toolchain/bin/llvm-as" -export CC="$toolchain/bin/${clang_triplet}${api_level}-clang" +export CC="$toolchain/bin/${clang_triplet}${ANDROID_API_LEVEL}-clang" export CXX="${CC}++" export LD="$toolchain/bin/ld" export NM="$toolchain/bin/llvm-nm" diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts index c627cb1b0e0b22..2a284f619db9ec 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Android/testbed/app/build.gradle.kts @@ -85,7 +85,7 @@ android { minSdk = androidEnvFile.useLines { for (line in it) { - """api_level:=(\d+)""".toRegex().find(line)?.let { + """ANDROID_API_LEVEL:=(\d+)""".toRegex().find(line)?.let { return@useLines it.groupValues[1].toInt() } } From f29e1779c31c1deb8a09d83959724cdea5acaeb9 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 30 Apr 2025 10:45:21 +0100 Subject: [PATCH 06/13] Environment variable cleanups --- Android/android.py | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/Android/android.py b/Android/android.py index b410f3e2eec22f..0722da2df2ba8a 100755 --- a/Android/android.py +++ b/Android/android.py @@ -82,15 +82,30 @@ def run(command, *, host=None, env=None, log=True, **kwargs): if env is None: env = os.environ.copy() if host: - env.update(android_env(host)) + # The -I and -L arguments used when building Python should not be reused + # when building third-party extension modules, so pass them via the + # NODIST environment variables. + host_env = android_env(host) + for name in ["CFLAGS", "CXXFLAGS", "LDFLAGS"]: + flags = [] + nodist = [] + for word in host_env[name].split(): + (nodist if word.startswith(("-I", "-L")) else flags).append(word) + host_env[name] = " ".join(flags) + host_env[f"{name}_NODIST"] = " ".join(nodist) + + print_env(host_env) + env.update(host_env) if log: print(">", " ".join(map(str, command))) return subprocess.run(command, env=env, **kwargs) -def print_env(context): - android_env(getattr(context, "host", None)) +# Format the environment so it can be pasted into a shell. +def print_env(env): + for key, value in sorted(env.items()): + print(f"export {key}={shlex.quote(value)}") def android_env(host): @@ -126,10 +141,6 @@ def android_env(host): if not env: raise ValueError(f"Found no variables in {env_script.name} output:\n" + env_output) - - # Format the environment so it can be pasted into a shell. - for key, value in sorted(env.items()): - print(f"export {key}={shlex.quote(value)}") return env @@ -220,9 +231,12 @@ def make_host_python(context): for pattern in ("include/python*", "lib/libpython*", "lib/python*"): delete_glob(f"{prefix_dir}/{pattern}") + # The Android environment variables were already captured in the Makefile by + # `configure`, and passing them again when running `make` may cause some + # flags to be duplicated. So we don't use the `host` argument here. os.chdir(host_dir) - run(["make", "-j", str(os.cpu_count())], host=context.host) - run(["make", "install", f"prefix={prefix_dir}"], host=context.host) + run(["make", "-j", str(os.cpu_count())]) + run(["make", "install", f"prefix={prefix_dir}"]) def build_all(context): @@ -628,6 +642,10 @@ def package(context): print(f"Wrote {package_path}") +def env(context): + print_env(android_env(getattr(context, "host", None))) + + # Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated # by the buildbot worker, we'll make an attempt to clean up our subprocesses. def install_signal_handler(): @@ -717,7 +735,7 @@ def main(): "build-testbed": build_testbed, "test": run_testbed, "package": package, - "env": print_env, + "env": env, } try: From fc8c1e1fd7e8bf947ea16b0cc47efd336f8c45d9 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 2 May 2025 15:57:49 +0100 Subject: [PATCH 07/13] Add `android.py test` -c and -m options --- Android/android.py | 68 ++++++++++++++----- .../java/org/python/testbed/PythonSuite.kt | 8 +-- .../java/org/python/testbed/MainActivity.kt | 27 ++++++-- .../{main.py => android_testbed_main.py} | 10 ++- 4 files changed, 82 insertions(+), 31 deletions(-) rename Android/testbed/app/src/main/python/{main.py => android_testbed_main.py} (84%) diff --git a/Android/android.py b/Android/android.py index 0722da2df2ba8a..eafbc56f46fecf 100755 --- a/Android/android.py +++ b/Android/android.py @@ -98,10 +98,19 @@ def run(command, *, host=None, env=None, log=True, **kwargs): env.update(host_env) if log: - print(">", " ".join(map(str, command))) + print(">", join_command(command)) return subprocess.run(command, env=env, **kwargs) +# Format a command so it can be copied into a shell. Like shlex.join, but also +# accepts arguments which are Paths, or a single string/Path outside of a list. +def join_command(args): + if isinstance(args, (str, Path)): + return str(args) + else: + return shlex.join(map(str, args)) + + # Format the environment so it can be pasted into a shell. def print_env(env): for key, value in sorted(env.items()): @@ -512,24 +521,42 @@ async def gradle_task(context): task_prefix = "connected" env["ANDROID_SERIAL"] = context.connected + hidden_output = [] + + def log(line): + # Gradle may take several minutes to install SDK packages, so it's worth + # showing those messages even in non-verbose mode. + if context.verbose or line.startswith('Preparing "Install'): + sys.stdout.write(line) + else: + hidden_output.append(line) + + if context.command: + mode = "-c" + module = context.command + else: + mode = "-m" + module = context.module or "test" + args = [ gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", - "-Pandroid.testInstrumentationRunnerArguments.pythonArgs=" - + shlex.join(context.args), + ] + [ + f"-Pandroid.testInstrumentationRunnerArguments.python{name}={value}" + for name, value in [ + ("Mode", mode), + ("Module", module), + ("Args", join_command(context.args)), + ] ] - hidden_output = [] + log("> " + join_command(args)) + try: async with async_process( *args, cwd=TESTBED_DIR, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) as process: while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - # Gradle may take several minutes to install SDK packages, so - # it's worth showing those messages even in non-verbose mode. - if context.verbose or line.startswith('Preparing "Install'): - sys.stdout.write(line) - else: - hidden_output.append(line) + log(line) status = await wait_for(process.wait(), timeout=1) if status == 0: @@ -701,6 +728,7 @@ def parse_args(): "-v", "--verbose", action="count", default=0, help="Show Gradle output, and non-Python logcat messages. " "Use twice to include high-volume messages which are rarely useful.") + device_group = test.add_mutually_exclusive_group(required=True) device_group.add_argument( "--connected", metavar="SERIAL", help="Run on a connected device. " @@ -708,8 +736,17 @@ def parse_args(): device_group.add_argument( "--managed", metavar="NAME", help="Run on a Gradle-managed device. " "These are defined in `managedDevices` in testbed/app/build.gradle.kts.") + + mode_group = test.add_mutually_exclusive_group() + mode_group.add_argument( + "-c", dest="command", help="Execute the given Python code.") + mode_group.add_argument( + "-m", dest="module", help="Execute the module with the given name.") + test.epilog = ( + "If neither -c nor -m are passed, the default is '-m test', which will " + "run Python's own test suite.") test.add_argument( - "args", nargs="*", help=f"Arguments for `python -m test`. " + "args", nargs="*", help=f"Arguments to add to sys.argv. " f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.") return parser.parse_args() @@ -756,14 +793,9 @@ def print_called_process_error(e): if not content.endswith("\n"): stream.write("\n") - # Format the command so it can be copied into a shell. shlex uses single - # quotes, so we surround the whole command with double quotes. - args_joined = ( - e.cmd if isinstance(e.cmd, str) - else " ".join(shlex.quote(str(arg)) for arg in e.cmd) - ) + # shlex uses single quotes, so we surround the command with double quotes. print( - f'Command "{args_joined}" returned exit status {e.returncode}' + f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}' ) diff --git a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt index 0e888ab71d87da..94be52dd2dc870 100644 --- a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt +++ b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt @@ -17,11 +17,11 @@ class PythonSuite { fun testPython() { val start = System.currentTimeMillis() try { - val context = + val status = PythonTestRunner( InstrumentationRegistry.getInstrumentation().targetContext - val args = - InstrumentationRegistry.getArguments().getString("pythonArgs", "") - val status = PythonTestRunner(context).run(args) + ).run( + InstrumentationRegistry.getArguments() + ) assertEquals(0, status) } finally { // Make sure the process lives long enough for the test script to diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt index c4bf6cbe83d8cd..db716d2b49e49c 100644 --- a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -15,17 +15,29 @@ class MainActivity : AppCompatActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) setContentView(R.layout.activity_main) - val status = PythonTestRunner(this).run("-W -uall") + val status = PythonTestRunner(this).run("-m", "test", "-W -uall") findViewById(R.id.tvHello).text = "Exit status $status" } } class PythonTestRunner(val context: Context) { - /** @param args Extra arguments for `python -m test`. - * @return The Python exit status: zero if the tests passed, nonzero if - * they failed. */ - fun run(args: String = "") : Int { + fun run(instrumentationArgs: Bundle) = run( + instrumentationArgs.getString("pythonMode")!!, + instrumentationArgs.getString("pythonModule")!!, + instrumentationArgs.getString("pythonArgs")!!, + ) + + /** Run Python. + * + * @param mode Either "-c" or "-m". + * @param module Python statements for "-c" mode, or a module name for + * "-m" mode. + * @param args Arguments to add to sys.argv. Will be parsed by `shlex.split`. + * @return The Python exit status: zero on success, nonzero on failure. */ + fun run(mode: String, module: String, args: String = "") : Int { + Os.setenv("PYTHON_MODE", mode, true) + Os.setenv("PYTHON_MODULE", module, true) Os.setenv("PYTHON_ARGS", args, true) // Python needs this variable to help it find the temporary directory, @@ -36,8 +48,9 @@ class PythonTestRunner(val context: Context) { System.loadLibrary("main_activity") redirectStdioToLogcat() - // The main module is in src/main/python/main.py. - return runPython(pythonHome.toString(), "main") + // The main module is in src/main/python. We don't simply call it + // "main", as that could clash with third-party test code. + return runPython(pythonHome.toString(), "android_testbed_main") } private fun extractAssets() : File { diff --git a/Android/testbed/app/src/main/python/main.py b/Android/testbed/app/src/main/python/android_testbed_main.py similarity index 84% rename from Android/testbed/app/src/main/python/main.py rename to Android/testbed/app/src/main/python/android_testbed_main.py index d6941b14412fcc..71ac47e5734306 100644 --- a/Android/testbed/app/src/main/python/main.py +++ b/Android/testbed/app/src/main/python/android_testbed_main.py @@ -26,7 +26,13 @@ # test_signals in test_threadsignals.py. signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) +mode = os.environ["PYTHON_MODE"] +module = os.environ["PYTHON_MODULE"] sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"]) -# The test module will call sys.exit to indicate whether the tests passed. -runpy.run_module("test") +if mode == "-c": + exec(module, {}) +elif mode == "-m": + runpy.run_module(module, run_name="__main__", alter_sys=True) +else: + raise ValueError(f"unknown mode: {mode}") From b273bc743dacb24128ed4bb11b6c4c0b4d498222 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 3 May 2025 14:29:55 +0100 Subject: [PATCH 08/13] Add `android.py test` --site-packages and --cwd options --- Android/README.md | 4 ++ Android/android.py | 54 +++++++++++-------- Android/testbed/app/build.gradle.kts | 18 +++++++ .../java/org/python/testbed/MainActivity.kt | 4 +- .../src/main/python/android_testbed_main.py | 10 ++++ Doc/using/android.rst | 9 ++++ 6 files changed, 76 insertions(+), 23 deletions(-) diff --git a/Android/README.md b/Android/README.md index 6cabd6ba5d6844..c42eb627006e6a 100644 --- a/Android/README.md +++ b/Android/README.md @@ -156,6 +156,10 @@ repository's `Lib` directory will be picked up immediately. Changes in C files, and architecture-specific files such as sysconfigdata, will not take effect until you re-run `android.py make-host` or `build`. +The testbed app can also be used to test third-party packages. For more details, +run `android.py test --help`, paying attention to the options `--site-packages`, +`--cwd`, `-c` and `-m`. + ## Using in your own app diff --git a/Android/android.py b/Android/android.py index eafbc56f46fecf..40c962ed4715a4 100755 --- a/Android/android.py +++ b/Android/android.py @@ -14,7 +14,7 @@ from contextlib import asynccontextmanager from datetime import datetime, timezone from glob import glob -from os.path import basename, relpath +from os.path import abspath, basename, relpath from pathlib import Path from subprocess import CalledProcessError from tempfile import TemporaryDirectory @@ -541,12 +541,17 @@ def log(line): args = [ gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", ] + [ + # Build-time properties + f"-Ppython.{name}={value}" + for name, value in [ + ("sitePackages", context.site_packages), ("cwd", context.cwd) + ] if value + ] + [ + # Runtime properties f"-Pandroid.testInstrumentationRunnerArguments.python{name}={value}" for name, value in [ - ("Mode", mode), - ("Module", module), - ("Args", join_command(context.args)), - ] + ("Mode", mode), ("Module", module), ("Args", join_command(context.args)) + ] if value ] log("> " + join_command(args)) @@ -684,24 +689,24 @@ def signal_handler(*args): def parse_args(): parser = argparse.ArgumentParser() - subcommands = parser.add_subparsers(dest="subcommand") + subcommands = parser.add_subparsers(dest="subcommand", required=True) # Subcommands - build = subcommands.add_parser("build", help="Build everything") - configure_build = subcommands.add_parser("configure-build", - help="Run `configure` for the " - "build Python") - subcommands.add_parser("make-build", help="Run `make` for the build Python") - configure_host = subcommands.add_parser("configure-host", - help="Run `configure` for Android") - make_host = subcommands.add_parser("make-host", - help="Run `make` for Android") - subcommands.add_parser( - "clean", help="Delete all build and prefix directories") + build = subcommands.add_parser( + "build", help="Run configure-build, make-build, configure-host and " + "make-host") + configure_build = subcommands.add_parser( + "configure-build", help="Run `configure` for the build Python") subcommands.add_parser( - "build-testbed", help="Build the testbed app") - test = subcommands.add_parser( - "test", help="Run the test suite") + "make-build", help="Run `make` for the build Python") + configure_host = subcommands.add_parser( + "configure-host", help="Run `configure` for Android") + make_host = subcommands.add_parser( + "make-host", help="Run `make` for Android") + + subcommands.add_parser("clean", help="Delete all build directories") + subcommands.add_parser("build-testbed", help="Build the testbed app") + test = subcommands.add_parser("test", help="Run the testbed app") package = subcommands.add_parser("package", help="Make a release package") env = subcommands.add_parser("env", help="Print environment variables") @@ -709,7 +714,7 @@ def parse_args(): for subcommand in build, configure_build, configure_host: subcommand.add_argument( "--clean", action="store_true", default=False, dest="clean", - help="Delete the relevant build and prefix directories first") + help="Delete the relevant build directories first") host_commands = [build, configure_host, make_host, package] if in_source_tree: @@ -737,6 +742,13 @@ def parse_args(): "--managed", metavar="NAME", help="Run on a Gradle-managed device. " "These are defined in `managedDevices` in testbed/app/build.gradle.kts.") + test.add_argument( + "--site-packages", metavar="DIR", type=abspath, + help="Directory to copy as the app's site-packages.") + test.add_argument( + "--cwd", metavar="DIR", type=abspath, + help="Directory to copy as the app's working directory.") + mode_group = test.add_mutually_exclusive_group() mode_group.add_argument( "-c", dest="command", help="Execute the given Python code.") diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts index 2a284f619db9ec..92cffd61f86876 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Android/testbed/app/build.gradle.kts @@ -205,11 +205,29 @@ androidComponents.onVariants { variant -> into("site-packages") { from("$projectDir/src/main/python") + + val sitePackages = findProperty("python.sitePackages") as String? + if (!sitePackages.isNullOrEmpty()) { + if (!file(sitePackages).exists()) { + throw GradleException("$sitePackages does not exist") + } + from(sitePackages) + } } duplicatesStrategy = DuplicatesStrategy.EXCLUDE exclude("**/__pycache__") } + + into("cwd") { + val cwd = findProperty("python.cwd") as String? + if (!cwd.isNullOrEmpty()) { + if (!file(cwd).exists()) { + throw GradleException("$cwd does not exist") + } + from(cwd) + } + } } } diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt index db716d2b49e49c..ef28948486fb52 100644 --- a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -25,7 +25,7 @@ class PythonTestRunner(val context: Context) { fun run(instrumentationArgs: Bundle) = run( instrumentationArgs.getString("pythonMode")!!, instrumentationArgs.getString("pythonModule")!!, - instrumentationArgs.getString("pythonArgs")!!, + instrumentationArgs.getString("pythonArgs") ?: "", ) /** Run Python. @@ -35,7 +35,7 @@ class PythonTestRunner(val context: Context) { * "-m" mode. * @param args Arguments to add to sys.argv. Will be parsed by `shlex.split`. * @return The Python exit status: zero on success, nonzero on failure. */ - fun run(mode: String, module: String, args: String = "") : Int { + fun run(mode: String, module: String, args: String) : Int { Os.setenv("PYTHON_MODE", mode, true) Os.setenv("PYTHON_MODULE", module, true) Os.setenv("PYTHON_ARGS", args, true) diff --git a/Android/testbed/app/src/main/python/android_testbed_main.py b/Android/testbed/app/src/main/python/android_testbed_main.py index 71ac47e5734306..31b8e5343a8449 100644 --- a/Android/testbed/app/src/main/python/android_testbed_main.py +++ b/Android/testbed/app/src/main/python/android_testbed_main.py @@ -30,9 +30,19 @@ module = os.environ["PYTHON_MODULE"] sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"]) +cwd = f"{sys.prefix}/cwd" +if not os.path.exists(cwd): + # Empty directories are lost in the asset packing/unpacking process. + os.mkdir(cwd) +os.chdir(cwd) + if mode == "-c": + # In -c mode, sys.path starts with an empty string, which means whatever the current + # working directory is at the moment of each import. + sys.path.insert(0, "") exec(module, {}) elif mode == "-m": + sys.path.insert(0, os.getcwd()) runpy.run_module(module, run_name="__main__", alter_sys=True) else: raise ValueError(f"unknown mode: {mode}") diff --git a/Doc/using/android.rst b/Doc/using/android.rst index 65bf23dc994856..cb762310328f1c 100644 --- a/Doc/using/android.rst +++ b/Doc/using/android.rst @@ -63,3 +63,12 @@ link to the relevant file. * Add code to your app to :source:`start Python in embedded mode `. This will need to be C code called via JNI. + +Building a Python package for Android +------------------------------------- + +Python packages can be built for Android as wheels and released on PyPI. The +recommended tool for doing this is `cibuildwheel +`__, which automates +all the details of setting up a cross-compilation environment, building the +wheel, and testing it on an emulator. From f36273a2cf393339338a319f54553ab1f5871835 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 29 May 2025 14:37:21 +0100 Subject: [PATCH 09/13] Revert use of NODIST environment variables --- Android/android.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/Android/android.py b/Android/android.py index 40c962ed4715a4..5e986e59110efd 100755 --- a/Android/android.py +++ b/Android/android.py @@ -78,22 +78,11 @@ def subdir(*parts, create=False): def run(command, *, host=None, env=None, log=True, **kwargs): kwargs.setdefault("check", True) - if env is None: env = os.environ.copy() + if host: - # The -I and -L arguments used when building Python should not be reused - # when building third-party extension modules, so pass them via the - # NODIST environment variables. host_env = android_env(host) - for name in ["CFLAGS", "CXXFLAGS", "LDFLAGS"]: - flags = [] - nodist = [] - for word in host_env[name].split(): - (nodist if word.startswith(("-I", "-L")) else flags).append(word) - host_env[name] = " ".join(flags) - host_env[f"{name}_NODIST"] = " ".join(nodist) - print_env(host_env) env.update(host_env) From 51e1460c5646d597e9bd76bd57bef4c0fb31d1ad Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 2 Jun 2025 22:24:25 +0100 Subject: [PATCH 10/13] Update to current Android Gradle plugin version, which gives better error messages when failing to start the emulator --- Android/testbed/build.gradle.kts | 2 +- Android/testbed/gradle/wrapper/gradle-wrapper.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Android/testbed/build.gradle.kts b/Android/testbed/build.gradle.kts index 4d1d6f87594da3..451517b3f1aeab 100644 --- a/Android/testbed/build.gradle.kts +++ b/Android/testbed/build.gradle.kts @@ -1,5 +1,5 @@ // Top-level build file where you can add configuration options common to all sub-projects/modules. plugins { - id("com.android.application") version "8.6.1" apply false + id("com.android.application") version "8.10.0" apply false id("org.jetbrains.kotlin.android") version "1.9.22" apply false } diff --git a/Android/testbed/gradle/wrapper/gradle-wrapper.properties b/Android/testbed/gradle/wrapper/gradle-wrapper.properties index 36529c896426b0..5d42fbae084da1 100644 --- a/Android/testbed/gradle/wrapper/gradle-wrapper.properties +++ b/Android/testbed/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ #Mon Feb 19 20:29:06 GMT 2024 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 497e84c92cec3823aa6add8283862f060fd2d7b0 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 3 Jun 2025 19:26:36 +0100 Subject: [PATCH 11/13] Update to an NDK version which is pre-installed on GitHub Actions --- Android/android-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Android/android-env.sh b/Android/android-env.sh index ae1385034a37f2..7b381a013cf0ba 100644 --- a/Android/android-env.sh +++ b/Android/android-env.sh @@ -24,7 +24,7 @@ fail() { # * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md # where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.: # https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md -ndk_version=27.1.12297006 +ndk_version=27.2.12479018 ndk=$ANDROID_HOME/ndk/$ndk_version if ! [ -e "$ndk" ]; then From ddfb875de4d4974f67e4dd8c7421b9b8f68cc126 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 3 Jun 2025 19:28:08 +0100 Subject: [PATCH 12/13] Logging improvements --- Android/android.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Android/android.py b/Android/android.py index 5e986e59110efd..c8f235724f7d9e 100755 --- a/Android/android.py +++ b/Android/android.py @@ -264,7 +264,12 @@ def setup_sdk(): if not all((android_home / "licenses" / path).exists() for path in [ "android-sdk-arm-dbt-license", "android-sdk-license" ]): - run([sdkmanager, "--licenses"], text=True, input="y\n" * 100) + run( + [sdkmanager, "--licenses"], + text=True, + capture_output=True, + input="y\n" * 100, + ) # Gradle may install this automatically, but we can't rely on that because # we need to run adb within the logcat task. @@ -542,6 +547,8 @@ def log(line): ("Mode", mode), ("Module", module), ("Args", join_command(context.args)) ] if value ] + if context.verbose >= 2: + args.append("--info") log("> " + join_command(args)) try: From 6f185281d13784b0472bcf34bf8d9fd03a722435 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 4 Jun 2025 15:22:39 +0100 Subject: [PATCH 13/13] Update to bzip2-1.0.8-3 --- Android/android.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Android/android.py b/Android/android.py index c8f235724f7d9e..551168fc4b2f5a 100755 --- a/Android/android.py +++ b/Android/android.py @@ -172,12 +172,13 @@ def make_build_python(context): def unpack_deps(host, prefix_dir): + os.chdir(prefix_dir) deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" - for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4", + for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.15-4", "sqlite-3.49.1-0", "xz-5.4.6-1"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") - shutil.unpack_archive(filename, prefix_dir) + shutil.unpack_archive(filename) os.remove(filename)