From 3aea414aa2e5f1e0e915f58a434c01428a90382c Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 23 May 2025 19:52:32 -0700 Subject: [PATCH 01/43] refactor: also rename host toolchain impl function name (#2930) The implementation function name got missed when the repo rule name itself was changed. --- python/private/toolchains_repo.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index cf4373932b..2476889583 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -309,7 +309,7 @@ actions.""", environ = [REPO_DEBUG_ENV_VAR], ) -def _host_toolchain_impl(rctx): +def _host_compatible_python_repo(rctx): rctx.file("BUILD.bazel", _HOST_TOOLCHAIN_BUILD_CONTENT) os_name = repo_utils.get_platforms_os_name(rctx) @@ -380,7 +380,7 @@ def _host_toolchain_impl(rctx): # NOTE: The term "toolchain" is a misnomer for this rule. This doesn't define # a repo with toolchains or toolchain implementations. host_compatible_python_repo = repository_rule( - _host_toolchain_impl, + _host_compatible_python_repo, doc = """\ Creates a repository with a shorter name meant to be used in the repository_ctx, which needs to have `symlinks` for the interpreter. This is separate from the From 28fda8664a1e89f2f055ed7183ad28dbdbeaafc9 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 24 May 2025 13:35:03 -0700 Subject: [PATCH 02/43] tests: refactor py_reconfig rules so less boilerplate is needed to add attrs (#2933) Just some minor refactoring of the py_reconfig rule so that it's easier to add attributes that affect transition state. After this, just two spots have to be modified to add an attribute (map of attrs, map of attr to transition label). --- tests/support/sh_py_run_test.bzl | 82 ++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 4 deletions(-) diff --git a/tests/support/sh_py_run_test.bzl b/tests/support/sh_py_run_test.bzl index 1a61de9bd3..04a2883fde 100644 --- a/tests/support/sh_py_run_test.bzl +++ b/tests/support/sh_py_run_test.bzl @@ -13,14 +13,88 @@ # limitations under the License. """Run a py_binary with altered config settings in an sh_test. -This facilitates verify running binaries with different outer environmental -settings and verifying their output without the overhead of a bazel-in-bazel -integration test. +This facilitates verify running binaries with different configuration settings +without the overhead of a bazel-in-bazel integration test. """ load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("//python/private:attr_builders.bzl", "attrb") # buildifier: disable=bzl-visibility +load("//python/private:py_binary_macro.bzl", "py_binary_macro") # buildifier: disable=bzl-visibility +load("//python/private:py_binary_rule.bzl", "create_py_binary_rule_builder") # buildifier: disable=bzl-visibility +load("//python/private:py_test_macro.bzl", "py_test_macro") # buildifier: disable=bzl-visibility +load("//python/private:py_test_rule.bzl", "create_py_test_rule_builder") # buildifier: disable=bzl-visibility load("//python/private:toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") # buildifier: disable=bzl-visibility -load(":py_reconfig.bzl", "py_reconfig_binary") +load("//tests/support:support.bzl", "VISIBLE_FOR_TESTING") + +def _perform_transition_impl(input_settings, attr, base_impl): + settings = {k: input_settings[k] for k in _RECONFIG_INHERITED_OUTPUTS if k in input_settings} + settings.update(base_impl(input_settings, attr)) + + settings[VISIBLE_FOR_TESTING] = True + settings["//command_line_option:build_python_zip"] = attr.build_python_zip + + for attr_name, setting_label in _RECONFIG_ATTR_SETTING_MAP.items(): + if getattr(attr, attr_name): + settings[setting_label] = getattr(attr, attr_name) + return settings + +# Attributes that, if non-falsey (`if attr.`), will copy their +# value into the output settings +_RECONFIG_ATTR_SETTING_MAP = { + "bootstrap_impl": "//python/config_settings:bootstrap_impl", + "extra_toolchains": "//command_line_option:extra_toolchains", + "python_src": "//python/bin:python_src", + "venvs_site_packages": "//python/config_settings:venvs_site_packages", + "venvs_use_declare_symlink": "//python/config_settings:venvs_use_declare_symlink", +} + +_RECONFIG_INPUTS = _RECONFIG_ATTR_SETTING_MAP.values() +_RECONFIG_OUTPUTS = _RECONFIG_INPUTS + [ + "//command_line_option:build_python_zip", + VISIBLE_FOR_TESTING, +] +_RECONFIG_INHERITED_OUTPUTS = [v for v in _RECONFIG_OUTPUTS if v in _RECONFIG_INPUTS] + +_RECONFIG_ATTRS = { + "bootstrap_impl": attrb.String(), + "build_python_zip": attrb.String(default = "auto"), + "extra_toolchains": attrb.StringList( + doc = """ +Value for the --extra_toolchains flag. + +NOTE: You'll likely have to also specify //tests/support/cc_toolchains:all (or some CC toolchain) +to make the RBE presubmits happy, which disable auto-detection of a CC +toolchain. +""", + ), + "python_src": attrb.Label(), + "venvs_site_packages": attrb.String(), + "venvs_use_declare_symlink": attrb.String(), +} + +def _create_reconfig_rule(builder): + builder.attrs.update(_RECONFIG_ATTRS) + + base_cfg_impl = builder.cfg.implementation() + builder.cfg.set_implementation(lambda *args: _perform_transition_impl(base_impl = base_cfg_impl, *args)) + builder.cfg.update_inputs(_RECONFIG_INPUTS) + builder.cfg.update_outputs(_RECONFIG_OUTPUTS) + return builder.build() + +_py_reconfig_binary = _create_reconfig_rule(create_py_binary_rule_builder()) + +_py_reconfig_test = _create_reconfig_rule(create_py_test_rule_builder()) + +def py_reconfig_test(**kwargs): + """Create a py_test with customized build settings for testing. + + Args: + **kwargs: kwargs to pass along to _py_reconfig_test. + """ + py_test_macro(_py_reconfig_test, **kwargs) + +def py_reconfig_binary(**kwargs): + py_binary_macro(_py_reconfig_binary, **kwargs) def sh_py_run_test(*, name, sh_src, py_src, **kwargs): """Run a py_binary within a sh_test. From e73dccf7b1827b1ea1216646ac82c97ae8d1e64b Mon Sep 17 00:00:00 2001 From: Chris Chua Date: Sun, 25 May 2025 13:21:44 +0800 Subject: [PATCH 03/43] feat: add shebang attribute on py_console_script_binary (#2867) # Background Use case: user is setting up the environment for a docker image, and needs a bash executable from the py_console_script (e.g. to run `ray` from command line without full bazel bootstrapping). User is responsible of setting up the right paths (and hermeticity concerns). There's no change in default behavior per this diff. Previously, prior to Bazel mod, this was possible and simple through the use of `rules_python_wheel_entry_points` ([per here](https://github.com/bazel-contrib/rules_python/blob/9dfa3abba293488a9a1899832a340f7b44525cad/python/private/pypi/whl_library.bzl#L507)) but these are not reachable now via Bazel mod. # Approach Add a shebang attribute that allows users of the console binary to use it like a binary executable. This is similar to the functionality that came with wheel entry points here: https://github.com/bazel-contrib/rules_python/blob/9dfa3abba293488a9a1899832a340f7b44525cad/python/private/pypi/whl_library.bzl#L507 With this change, one can specify a shebang like: ```starlark py_console_script_binary( name = "yamllint", pkg = "@pip//yamllint", shebang = "#!/usr/bin/env python3", ) ``` Summary: - Update tests - Add test for this functionality - Leave default to without shebang so this is a non-breaking change - Documentation (want to hear more about the general approach first, and also want to hear whether this warrants specific docs, or can just leave it to API docs) --------- Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> --- docs/_includes/py_console_script_binary.md | 23 ++++++++++++- python/private/py_console_script_binary.bzl | 4 +++ python/private/py_console_script_gen.bzl | 5 +++ python/private/py_console_script_gen.py | 11 +++++- .../py_console_script_gen_test.py | 34 +++++++++++++++++++ 5 files changed, 75 insertions(+), 2 deletions(-) diff --git a/docs/_includes/py_console_script_binary.md b/docs/_includes/py_console_script_binary.md index aa356e0e94..d327091630 100644 --- a/docs/_includes/py_console_script_binary.md +++ b/docs/_includes/py_console_script_binary.md @@ -48,6 +48,26 @@ py_console_script_binary( ) ``` +#### Adding a Shebang Line + +You can specify a shebang line for the generated binary, useful for Unix-like +systems where the shebang line determines which interpreter is used to execute +the script, per [PEP441]: + +```starlark +load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") + +py_console_script_binary( + name = "black", + pkg = "@pip//black", + shebang = "#!/usr/bin/env python3", +) +``` + +Note that to execute via the shebang line, you need to ensure the specified +Python interpreter is available in the environment. + + #### Using a specific Python Version directly from a Toolchain :::{deprecated} 1.1.0 The toolchain specific `py_binary` and `py_test` symbols are aliases to the regular rules. @@ -70,4 +90,5 @@ py_console_script_binary( ``` [specification]: https://packaging.python.org/en/latest/specifications/entry-points/ -[`py_console_script_binary.binary_rule`]: #py_console_script_binary_binary_rule \ No newline at end of file +[`py_console_script_binary.binary_rule`]: #py_console_script_binary_binary_rule +[PEP441]: https://peps.python.org/pep-0441/#minimal-tooling-the-zipapp-module diff --git a/python/private/py_console_script_binary.bzl b/python/private/py_console_script_binary.bzl index 154fa3bf2f..d98457dbe1 100644 --- a/python/private/py_console_script_binary.bzl +++ b/python/private/py_console_script_binary.bzl @@ -52,6 +52,7 @@ def py_console_script_binary( entry_points_txt = None, script = None, binary_rule = py_binary, + shebang = "", **kwargs): """Generate a py_binary for a console_script entry_point. @@ -68,6 +69,8 @@ def py_console_script_binary( binary_rule: {type}`callable`, The rule/macro to use to instantiate the target. It's expected to behave like {obj}`py_binary`. Defaults to {obj}`py_binary`. + shebang: {type}`str`, The shebang to use for the entry point python file. + Defaults to empty string. **kwargs: Extra parameters forwarded to `binary_rule`. """ main = "rules_python_entry_point_{}.py".format(name) @@ -81,6 +84,7 @@ def py_console_script_binary( out = main, console_script = script, console_script_guess = name, + shebang = shebang, visibility = ["//visibility:private"], ) diff --git a/python/private/py_console_script_gen.bzl b/python/private/py_console_script_gen.bzl index 7dd4dd2dad..de016036b2 100644 --- a/python/private/py_console_script_gen.bzl +++ b/python/private/py_console_script_gen.bzl @@ -42,6 +42,7 @@ def _py_console_script_gen_impl(ctx): args = ctx.actions.args() args.add("--console-script", ctx.attr.console_script) args.add("--console-script-guess", ctx.attr.console_script_guess) + args.add("--shebang", ctx.attr.shebang) args.add(entry_points_txt) args.add(ctx.outputs.out) @@ -81,6 +82,10 @@ py_console_script_gen = rule( doc = "Output file location.", mandatory = True, ), + "shebang": attr.string( + doc = "The shebang to use for the entry point python file.", + default = "", + ), "_tool": attr.label( default = ":py_console_script_gen_py", executable = True, diff --git a/python/private/py_console_script_gen.py b/python/private/py_console_script_gen.py index ffc4e81b3a..4b4f2f6986 100644 --- a/python/private/py_console_script_gen.py +++ b/python/private/py_console_script_gen.py @@ -44,7 +44,7 @@ _ENTRY_POINTS_TXT = "entry_points.txt" _TEMPLATE = """\ -import sys +{shebang}import sys # See @rules_python//python/private:py_console_script_gen.py for explanation if getattr(sys.flags, "safe_path", False): @@ -87,6 +87,7 @@ def run( out: pathlib.Path, console_script: str, console_script_guess: str, + shebang: str, ): """Run the generator @@ -94,6 +95,8 @@ def run( entry_points: The entry_points.txt file to be parsed. out: The output file. console_script: The console_script entry in the entry_points.txt file. + console_script_guess: The string used for guessing the console_script if it is not provided. + shebang: The shebang to use for the entry point python file. Defaults to empty string (no shebang). """ config = EntryPointsParser() config.read(entry_points) @@ -136,6 +139,7 @@ def run( with open(out, "w") as f: f.write( _TEMPLATE.format( + shebang=f"{shebang}\n" if shebang else "", module=module, attr=attr, entry_point=entry_point, @@ -154,6 +158,10 @@ def main(): required=True, help="The string used for guessing the console_script if it is not provided.", ) + parser.add_argument( + "--shebang", + help="The shebang to use for the entry point python file.", + ) parser.add_argument( "entry_points", metavar="ENTRY_POINTS_TXT", @@ -173,6 +181,7 @@ def main(): out=args.out, console_script=args.console_script, console_script_guess=args.console_script_guess, + shebang=args.shebang, ) diff --git a/tests/entry_points/py_console_script_gen_test.py b/tests/entry_points/py_console_script_gen_test.py index a5fceb67f9..1bbf5fbf25 100644 --- a/tests/entry_points/py_console_script_gen_test.py +++ b/tests/entry_points/py_console_script_gen_test.py @@ -47,6 +47,7 @@ def test_no_console_scripts_error(self): out=outfile, console_script=None, console_script_guess="", + shebang="", ) self.assertEqual( @@ -76,6 +77,7 @@ def test_no_entry_point_selected_error(self): out=outfile, console_script=None, console_script_guess="bar-baz", + shebang="", ) self.assertEqual( @@ -106,6 +108,7 @@ def test_incorrect_entry_point(self): out=outfile, console_script="baz", console_script_guess="", + shebang="", ) self.assertEqual( @@ -134,6 +137,7 @@ def test_a_single_entry_point(self): out=out, console_script=None, console_script_guess="foo", + shebang="", ) got = out.read_text() @@ -185,6 +189,7 @@ def test_a_second_entry_point_class_method(self): out=out, console_script="bar", console_script_guess="", + shebang="", ) got = out.read_text() @@ -192,6 +197,35 @@ def test_a_second_entry_point_class_method(self): self.assertRegex(got, "from foo\.baz import Bar") self.assertRegex(got, "sys\.exit\(Bar\.baz\(\)\)") + def test_shebang_included(self): + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + given_contents = ( + textwrap.dedent( + """ + [console_scripts] + foo = foo.bar:baz + """ + ).strip() + + "\n" + ) + entry_points = tmpdir / "entry_points.txt" + entry_points.write_text(given_contents) + out = tmpdir / "foo.py" + + shebang = "#!/usr/bin/env python3" + run( + entry_points=entry_points, + out=out, + console_script=None, + console_script_guess="foo", + shebang=shebang, + ) + + got = out.read_text() + + self.assertTrue(got.startswith(shebang + "\n")) + if __name__ == "__main__": unittest.main() From b40d96aba36d675c60b03424aa22f31c09e0ea4f Mon Sep 17 00:00:00 2001 From: Kayce Basques Date: Mon, 26 May 2025 06:36:58 -0700 Subject: [PATCH 04/43] fix: update the stub type alias names (#2929) Co-authored-by: Kayce Basques --- tools/precompiler/precompiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/precompiler/precompiler.py b/tools/precompiler/precompiler.py index 310f2eb097..e7c693c195 100644 --- a/tools/precompiler/precompiler.py +++ b/tools/precompiler/precompiler.py @@ -68,12 +68,12 @@ def _compile(options: "argparse.Namespace") -> None: # A stub type alias for readability. # See the Bazel WorkRequest object definition: # https://github.com/bazelbuild/bazel/blob/master/src/main/protobuf/worker_protocol.proto -JsonWorkerRequest = object +JsonWorkRequest = object # A stub type alias for readability. # See the Bazel WorkResponse object definition: # https://github.com/bazelbuild/bazel/blob/master/src/main/protobuf/worker_protocol.proto -JsonWorkerResponse = object +JsonWorkResponse = object class _SerialPersistentWorker: From dce5120249f62bd04d2bafa48fd053732854e1ad Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 28 May 2025 00:47:49 +0900 Subject: [PATCH 05/43] refactor: reimplement writing namespace pkgs in Starlark (#2882) With this PR I would like to facilitate the implementation of the venv layouts because we can in theory take the `srcs` and the `data` within the `py_library` and then use the `expand_template` to write the extra Python files if the namespace_pkgs flag is enabled. The old Python code has been removed and the extra generated files are written out with `bazel_skylib` `copy_file`. The implicit `namespace_pkg` init files are included to `py_library` if the `site-packages` config flag is set to false and I think this may help with continuing the implementation, but it currently is still not working as expected (see comment). Work towards #2156 --- python/config_settings/BUILD.bazel | 9 + python/private/pypi/BUILD.bazel | 5 + python/private/pypi/namespace_pkg_tmpl.py | 2 + python/private/pypi/namespace_pkgs.bzl | 83 ++++++++ .../private/pypi/whl_installer/arguments.py | 5 - .../pypi/whl_installer/wheel_installer.py | 32 +-- python/private/pypi/whl_library.bzl | 8 +- python/private/pypi/whl_library_targets.bzl | 50 +++-- tests/pypi/namespace_pkgs/BUILD.bazel | 5 + .../namespace_pkgs/namespace_pkgs_tests.bzl | 167 +++++++++++++++ tests/pypi/whl_installer/BUILD.bazel | 11 - tests/pypi/whl_installer/arguments_test.py | 1 - .../pypi/whl_installer/namespace_pkgs_test.py | 192 ------------------ .../whl_installer/wheel_installer_test.py | 1 - 14 files changed, 311 insertions(+), 260 deletions(-) create mode 100644 python/private/pypi/namespace_pkg_tmpl.py create mode 100644 python/private/pypi/namespace_pkgs.bzl create mode 100644 tests/pypi/namespace_pkgs/BUILD.bazel create mode 100644 tests/pypi/namespace_pkgs/namespace_pkgs_tests.bzl delete mode 100644 tests/pypi/whl_installer/namespace_pkgs_test.py diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index 1772a3403e..ee15828fa5 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -217,6 +217,15 @@ string_flag( visibility = ["//visibility:public"], ) +config_setting( + name = "is_venvs_site_packages", + flag_values = { + ":venvs_site_packages": VenvsSitePackages.YES, + }, + # NOTE: Only public because it is used in whl_library repos. + visibility = ["//visibility:public"], +) + define_pypi_internal_flags( name = "define_pypi_internal_flags", ) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 84e0535289..e9036c3013 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -18,6 +18,11 @@ package(default_visibility = ["//:__subpackages__"]) licenses(["notice"]) +exports_files( + srcs = ["namespace_pkg_tmpl.py"], + visibility = ["//visibility:public"], +) + filegroup( name = "distribution", srcs = glob( diff --git a/python/private/pypi/namespace_pkg_tmpl.py b/python/private/pypi/namespace_pkg_tmpl.py new file mode 100644 index 0000000000..a21b846e76 --- /dev/null +++ b/python/private/pypi/namespace_pkg_tmpl.py @@ -0,0 +1,2 @@ +# __path__ manipulation added by bazel-contrib/rules_python to support namespace pkgs. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/python/private/pypi/namespace_pkgs.bzl b/python/private/pypi/namespace_pkgs.bzl new file mode 100644 index 0000000000..bf4689a5ea --- /dev/null +++ b/python/private/pypi/namespace_pkgs.bzl @@ -0,0 +1,83 @@ +"""Utilities to get where we should write namespace pkg paths.""" + +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + +_ext = struct( + py = ".py", + pyd = ".pyd", + so = ".so", + pyc = ".pyc", +) + +_TEMPLATE = Label("//python/private/pypi:namespace_pkg_tmpl.py") + +def _add_all(dirname, dirs): + dir_path = "." + for dir_name in dirname.split("/"): + dir_path = "{}/{}".format(dir_path, dir_name) + dirs[dir_path[2:]] = None + +def get_files(*, srcs, ignored_dirnames = [], root = None): + """Get the list of filenames to write the namespace pkg files. + + Args: + srcs: {type}`src` a list of files to be passed to {bzl:obj}`py_library` + as `srcs` and `data`. This is usually a result of a {obj}`glob`. + ignored_dirnames: {type}`str` a list of patterns to ignore. + root: {type}`str` the prefix to use as the root. + + Returns: + {type}`src` a list of paths to write the namespace pkg `__init__.py` file. + """ + dirs = {} + ignored = {i: None for i in ignored_dirnames} + + if root: + _add_all(root, ignored) + + for file in srcs: + dirname, _, filename = file.rpartition("/") + + if filename == "__init__.py": + ignored[dirname] = None + dirname, _, _ = dirname.rpartition("/") + elif filename.endswith(_ext.py): + pass + elif filename.endswith(_ext.pyc): + pass + elif filename.endswith(_ext.pyd): + pass + elif filename.endswith(_ext.so): + pass + else: + continue + + if dirname in dirs or not dirname: + continue + + _add_all(dirname, dirs) + + return sorted([d for d in dirs if d not in ignored]) + +def create_inits(**kwargs): + """Create init files and return the list to be included `py_library` srcs. + + Args: + **kwargs: passed to {obj}`get_files`. + + Returns: + {type}`list[str]` to be included as part of `py_library`. + """ + srcs = [] + for out in get_files(**kwargs): + src = "{}/__init__.py".format(out) + srcs.append(srcs) + + copy_file( + name = "_cp_{}_namespace".format(out), + src = _TEMPLATE, + out = src, + **kwargs + ) + + return srcs diff --git a/python/private/pypi/whl_installer/arguments.py b/python/private/pypi/whl_installer/arguments.py index ea609bef9d..57dae45ae9 100644 --- a/python/private/pypi/whl_installer/arguments.py +++ b/python/private/pypi/whl_installer/arguments.py @@ -57,11 +57,6 @@ def parser(**kwargs: Any) -> argparse.ArgumentParser: action="store", help="Additional data exclusion parameters to add to the pip packages BUILD file.", ) - parser.add_argument( - "--enable_implicit_namespace_pkgs", - action="store_true", - help="Disables conversion of implicit namespace packages into pkg-util style packages.", - ) parser.add_argument( "--environment", action="store", diff --git a/python/private/pypi/whl_installer/wheel_installer.py b/python/private/pypi/whl_installer/wheel_installer.py index 600d45f940..a6a9dd0429 100644 --- a/python/private/pypi/whl_installer/wheel_installer.py +++ b/python/private/pypi/whl_installer/wheel_installer.py @@ -27,7 +27,7 @@ from pip._vendor.packaging.utils import canonicalize_name -from python.private.pypi.whl_installer import arguments, namespace_pkgs, wheel +from python.private.pypi.whl_installer import arguments, wheel def _configure_reproducible_wheels() -> None: @@ -77,35 +77,10 @@ def _parse_requirement_for_extra( return None, None -def _setup_namespace_pkg_compatibility(wheel_dir: str) -> None: - """Converts native namespace packages to pkgutil-style packages - - Namespace packages can be created in one of three ways. They are detailed here: - https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package - - 'pkgutil-style namespace packages' (2) and 'pkg_resources-style namespace packages' (3) works in Bazel, but - 'native namespace packages' (1) do not. - - We ensure compatibility with Bazel of method 1 by converting them into method 2. - - Args: - wheel_dir: the directory of the wheel to convert - """ - - namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages( - wheel_dir, - ignored_dirnames=["%s/bin" % wheel_dir], - ) - - for ns_pkg_dir in namespace_pkg_dirs: - namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir) - - def _extract_wheel( wheel_file: str, extras: Dict[str, Set[str]], enable_pipstar: bool, - enable_implicit_namespace_pkgs: bool, platforms: List[wheel.Platform], installation_dir: Path = Path("."), ) -> None: @@ -116,15 +91,11 @@ def _extract_wheel( installation_dir: the destination directory for installation of the wheel. extras: a list of extras to add as dependencies for the installed wheel enable_pipstar: if true, turns off certain operations. - enable_implicit_namespace_pkgs: if true, disables conversion of implicit namespace packages and will unzip as-is """ whl = wheel.Wheel(wheel_file) whl.unzip(installation_dir) - if not enable_implicit_namespace_pkgs: - _setup_namespace_pkg_compatibility(installation_dir) - metadata = { "entry_points": [ { @@ -168,7 +139,6 @@ def main() -> None: wheel_file=whl, extras=extras, enable_pipstar=args.enable_pipstar, - enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs, platforms=arguments.get_platforms(args), ) return diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 17ee3d3cfe..c271449b3d 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -173,9 +173,6 @@ def _parse_optional_attrs(rctx, args, extra_pip_args = None): json.encode(struct(arg = rctx.attr.pip_data_exclude)), ] - if rctx.attr.enable_implicit_namespace_pkgs: - args.append("--enable_implicit_namespace_pkgs") - env = {} if rctx.attr.environment != None: for key, value in rctx.attr.environment.items(): @@ -389,6 +386,8 @@ def _whl_library_impl(rctx): metadata_name = metadata.name, metadata_version = metadata.version, requires_dist = metadata.requires_dist, + # TODO @aignas 2025-05-17: maybe have a build flag for this instead + enable_implicit_namespace_pkgs = rctx.attr.enable_implicit_namespace_pkgs, # TODO @aignas 2025-04-14: load through the hub: annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))), data_exclude = rctx.attr.pip_data_exclude, @@ -457,6 +456,8 @@ def _whl_library_impl(rctx): name = whl_path.basename, dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix), entry_points = entry_points, + # TODO @aignas 2025-05-17: maybe have a build flag for this instead + enable_implicit_namespace_pkgs = rctx.attr.enable_implicit_namespace_pkgs, # TODO @aignas 2025-04-14: load through the hub: dependencies = metadata["deps"], dependencies_by_platform = metadata["deps_by_platform"], @@ -580,7 +581,6 @@ attr makes `extra_pip_args` and `download_only` ignored.""", Label("//python/private/pypi/whl_installer:wheel.py"), Label("//python/private/pypi/whl_installer:wheel_installer.py"), Label("//python/private/pypi/whl_installer:arguments.py"), - Label("//python/private/pypi/whl_installer:namespace_pkgs.py"), ] + record_files.values(), ), "_rule_name": attr.string(default = "whl_library"), diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index e0c03a1505..3529566c49 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -30,6 +30,7 @@ load( "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", ) +load(":namespace_pkgs.bzl", "create_inits") load(":pep508_deps.bzl", "deps") def whl_library_targets_from_requires( @@ -113,6 +114,7 @@ def whl_library_targets( copy_executables = {}, entry_points = {}, native = native, + enable_implicit_namespace_pkgs = False, rules = struct( copy_file = copy_file, py_binary = py_binary, @@ -153,6 +155,8 @@ def whl_library_targets( data: {type}`list[str]` A list of labels to include as part of the `data` attribute in `py_library`. entry_points: {type}`dict[str, str]` The mapping between the script name and the python file to use. DEPRECATED. + enable_implicit_namespace_pkgs: {type}`boolean` generate __init__.py + files for namespace pkgs. native: {type}`native` The native struct for overriding in tests. rules: {type}`struct` A struct with references to rules for creating targets. """ @@ -293,6 +297,14 @@ def whl_library_targets( ) if hasattr(rules, "py_library"): + srcs = native.glob( + ["site-packages/**/*.py"], + exclude = srcs_exclude, + # Empty sources are allowed to support wheels that don't have any + # pure-Python code, e.g. pymssql, which is written in Cython. + allow_empty = True, + ) + # NOTE: pyi files should probably be excluded because they're carried # by the pyi_srcs attribute. However, historical behavior included # them in data and some tools currently rely on that. @@ -309,23 +321,31 @@ def whl_library_targets( if item not in _data_exclude: _data_exclude.append(item) + data = data + native.glob( + ["site-packages/**/*"], + exclude = _data_exclude, + ) + + pyi_srcs = native.glob( + ["site-packages/**/*.pyi"], + allow_empty = True, + ) + + if enable_implicit_namespace_pkgs: + srcs = srcs + getattr(native, "select", select)({ + Label("//python/config_settings:is_venvs_site_packages"): [], + "//conditions:default": create_inits( + srcs = srcs + data + pyi_srcs, + ignore_dirnames = [], # If you need to ignore certain folders, you can patch rules_python here to do so. + root = "site-packages", + ), + }) + rules.py_library( name = py_library_label, - srcs = native.glob( - ["site-packages/**/*.py"], - exclude = srcs_exclude, - # Empty sources are allowed to support wheels that don't have any - # pure-Python code, e.g. pymssql, which is written in Cython. - allow_empty = True, - ), - pyi_srcs = native.glob( - ["site-packages/**/*.pyi"], - allow_empty = True, - ), - data = data + native.glob( - ["site-packages/**/*"], - exclude = _data_exclude, - ), + srcs = srcs, + pyi_srcs = pyi_srcs, + data = data, # This makes this directory a top-level in the python import # search path for anything that depends on this. imports = ["site-packages"], diff --git a/tests/pypi/namespace_pkgs/BUILD.bazel b/tests/pypi/namespace_pkgs/BUILD.bazel new file mode 100644 index 0000000000..57f7962524 --- /dev/null +++ b/tests/pypi/namespace_pkgs/BUILD.bazel @@ -0,0 +1,5 @@ +load(":namespace_pkgs_tests.bzl", "namespace_pkgs_test_suite") + +namespace_pkgs_test_suite( + name = "namespace_pkgs_tests", +) diff --git a/tests/pypi/namespace_pkgs/namespace_pkgs_tests.bzl b/tests/pypi/namespace_pkgs/namespace_pkgs_tests.bzl new file mode 100644 index 0000000000..7ac938ff17 --- /dev/null +++ b/tests/pypi/namespace_pkgs/namespace_pkgs_tests.bzl @@ -0,0 +1,167 @@ +"" + +load("@rules_testing//lib:analysis_test.bzl", "test_suite") +load("//python/private/pypi:namespace_pkgs.bzl", "get_files") # buildifier: disable=bzl-visibility + +_tests = [] + +def test_in_current_dir(env): + srcs = [ + "foo/bar/biz.py", + "foo/bee/boo.py", + "foo/buu/__init__.py", + "foo/buu/bii.py", + ] + got = get_files(srcs = srcs) + expected = [ + "foo", + "foo/bar", + "foo/bee", + ] + env.expect.that_collection(got).contains_exactly(expected) + +_tests.append(test_in_current_dir) + +def test_find_correct_namespace_packages(env): + srcs = [ + "nested/root/foo/bar/biz.py", + "nested/root/foo/bee/boo.py", + "nested/root/foo/buu/__init__.py", + "nested/root/foo/buu/bii.py", + ] + + got = get_files(srcs = srcs, root = "nested/root") + expected = [ + "nested/root/foo", + "nested/root/foo/bar", + "nested/root/foo/bee", + ] + env.expect.that_collection(got).contains_exactly(expected) + +_tests.append(test_find_correct_namespace_packages) + +def test_ignores_empty_directories(_): + # because globs do not add directories, this test is not needed + pass + +_tests.append(test_ignores_empty_directories) + +def test_empty_case(env): + srcs = [ + "foo/__init__.py", + "foo/bar/__init__.py", + "foo/bar/biz.py", + ] + + got = get_files(srcs = srcs) + expected = [] + env.expect.that_collection(got).contains_exactly(expected) + +_tests.append(test_empty_case) + +def test_ignores_non_module_files_in_directories(env): + srcs = [ + "foo/__init__.pyi", + "foo/py.typed", + ] + + got = get_files(srcs = srcs) + expected = [] + env.expect.that_collection(got).contains_exactly(expected) + +_tests.append(test_ignores_non_module_files_in_directories) + +def test_parent_child_relationship_of_namespace_pkgs(env): + srcs = [ + "foo/bar/biff/my_module.py", + "foo/bar/biff/another_module.py", + ] + + got = get_files(srcs = srcs) + expected = [ + "foo", + "foo/bar", + "foo/bar/biff", + ] + env.expect.that_collection(got).contains_exactly(expected) + +_tests.append(test_parent_child_relationship_of_namespace_pkgs) + +def test_parent_child_relationship_of_namespace_and_standard_pkgs(env): + srcs = [ + "foo/bar/biff/__init__.py", + "foo/bar/biff/another_module.py", + ] + + got = get_files(srcs = srcs) + expected = [ + "foo", + "foo/bar", + ] + env.expect.that_collection(got).contains_exactly(expected) + +_tests.append(test_parent_child_relationship_of_namespace_and_standard_pkgs) + +def test_parent_child_relationship_of_namespace_and_nested_standard_pkgs(env): + srcs = [ + "foo/bar/__init__.py", + "foo/bar/biff/another_module.py", + "foo/bar/biff/__init__.py", + "foo/bar/boof/big_module.py", + "foo/bar/boof/__init__.py", + "fim/in_a_ns_pkg.py", + ] + + got = get_files(srcs = srcs) + expected = [ + "foo", + "fim", + ] + env.expect.that_collection(got).contains_exactly(expected) + +_tests.append(test_parent_child_relationship_of_namespace_and_nested_standard_pkgs) + +def test_recognized_all_nonstandard_module_types(env): + srcs = [ + "ayy/my_module.pyc", + "bee/ccc/dee/eee.so", + "eff/jee/aych.pyd", + ] + + expected = [ + "ayy", + "bee", + "bee/ccc", + "bee/ccc/dee", + "eff", + "eff/jee", + ] + got = get_files(srcs = srcs) + env.expect.that_collection(got).contains_exactly(expected) + +_tests.append(test_recognized_all_nonstandard_module_types) + +def test_skips_ignored_directories(env): + srcs = [ + "root/foo/boo/my_module.py", + "root/foo/bar/another_module.py", + ] + + expected = [ + "root/foo", + "root/foo/bar", + ] + got = get_files( + srcs = srcs, + ignored_dirnames = ["root/foo/boo"], + root = "root", + ) + env.expect.that_collection(got).contains_exactly(expected) + +_tests.append(test_skips_ignored_directories) + +def namespace_pkgs_test_suite(name): + test_suite( + name = name, + basic_tests = _tests, + ) diff --git a/tests/pypi/whl_installer/BUILD.bazel b/tests/pypi/whl_installer/BUILD.bazel index 040e4d765f..060d2bce62 100644 --- a/tests/pypi/whl_installer/BUILD.bazel +++ b/tests/pypi/whl_installer/BUILD.bazel @@ -16,17 +16,6 @@ py_test( ], ) -py_test( - name = "namespace_pkgs_test", - size = "small", - srcs = [ - "namespace_pkgs_test.py", - ], - deps = [ - ":lib", - ], -) - py_test( name = "platform_test", size = "small", diff --git a/tests/pypi/whl_installer/arguments_test.py b/tests/pypi/whl_installer/arguments_test.py index 5538054a59..2352d8e48b 100644 --- a/tests/pypi/whl_installer/arguments_test.py +++ b/tests/pypi/whl_installer/arguments_test.py @@ -36,7 +36,6 @@ def test_arguments(self) -> None: self.assertIn("requirement", args_dict) self.assertIn("extra_pip_args", args_dict) self.assertEqual(args_dict["pip_data_exclude"], []) - self.assertEqual(args_dict["enable_implicit_namespace_pkgs"], False) self.assertEqual(args_dict["extra_pip_args"], extra_pip_args) def test_deserialize_structured_args(self) -> None: diff --git a/tests/pypi/whl_installer/namespace_pkgs_test.py b/tests/pypi/whl_installer/namespace_pkgs_test.py deleted file mode 100644 index fbbd50926a..0000000000 --- a/tests/pypi/whl_installer/namespace_pkgs_test.py +++ /dev/null @@ -1,192 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pathlib -import shutil -import tempfile -import unittest -from typing import Optional, Set - -from python.private.pypi.whl_installer import namespace_pkgs - - -class TempDir: - def __init__(self) -> None: - self.dir = tempfile.mkdtemp() - - def root(self) -> str: - return self.dir - - def add_dir(self, rel_path: str) -> None: - d = pathlib.Path(self.dir, rel_path) - d.mkdir(parents=True) - - def add_file(self, rel_path: str, contents: Optional[str] = None) -> None: - f = pathlib.Path(self.dir, rel_path) - f.parent.mkdir(parents=True, exist_ok=True) - if contents: - with open(str(f), "w") as writeable_f: - writeable_f.write(contents) - else: - f.touch() - - def remove(self) -> None: - shutil.rmtree(self.dir) - - -class TestImplicitNamespacePackages(unittest.TestCase): - def assertPathsEqual(self, actual: Set[pathlib.Path], expected: Set[str]) -> None: - self.assertEqual(actual, {pathlib.Path(p) for p in expected}) - - def test_in_current_directory(self) -> None: - directory = TempDir() - directory.add_file("foo/bar/biz.py") - directory.add_file("foo/bee/boo.py") - directory.add_file("foo/buu/__init__.py") - directory.add_file("foo/buu/bii.py") - cwd = os.getcwd() - os.chdir(directory.root()) - expected = { - "foo", - "foo/bar", - "foo/bee", - } - try: - actual = namespace_pkgs.implicit_namespace_packages(".") - self.assertPathsEqual(actual, expected) - finally: - os.chdir(cwd) - directory.remove() - - def test_finds_correct_namespace_packages(self) -> None: - directory = TempDir() - directory.add_file("foo/bar/biz.py") - directory.add_file("foo/bee/boo.py") - directory.add_file("foo/buu/__init__.py") - directory.add_file("foo/buu/bii.py") - - expected = { - directory.root() + "/foo", - directory.root() + "/foo/bar", - directory.root() + "/foo/bee", - } - actual = namespace_pkgs.implicit_namespace_packages(directory.root()) - self.assertPathsEqual(actual, expected) - - def test_ignores_empty_directories(self) -> None: - directory = TempDir() - directory.add_file("foo/bar/biz.py") - directory.add_dir("foo/cat") - - expected = { - directory.root() + "/foo", - directory.root() + "/foo/bar", - } - actual = namespace_pkgs.implicit_namespace_packages(directory.root()) - self.assertPathsEqual(actual, expected) - - def test_empty_case(self) -> None: - directory = TempDir() - directory.add_file("foo/__init__.py") - directory.add_file("foo/bar/__init__.py") - directory.add_file("foo/bar/biz.py") - - actual = namespace_pkgs.implicit_namespace_packages(directory.root()) - self.assertEqual(actual, set()) - - def test_ignores_non_module_files_in_directories(self) -> None: - directory = TempDir() - directory.add_file("foo/__init__.pyi") - directory.add_file("foo/py.typed") - - actual = namespace_pkgs.implicit_namespace_packages(directory.root()) - self.assertEqual(actual, set()) - - def test_parent_child_relationship_of_namespace_pkgs(self): - directory = TempDir() - directory.add_file("foo/bar/biff/my_module.py") - directory.add_file("foo/bar/biff/another_module.py") - - expected = { - directory.root() + "/foo", - directory.root() + "/foo/bar", - directory.root() + "/foo/bar/biff", - } - actual = namespace_pkgs.implicit_namespace_packages(directory.root()) - self.assertPathsEqual(actual, expected) - - def test_parent_child_relationship_of_namespace_and_standard_pkgs(self): - directory = TempDir() - directory.add_file("foo/bar/biff/__init__.py") - directory.add_file("foo/bar/biff/another_module.py") - - expected = { - directory.root() + "/foo", - directory.root() + "/foo/bar", - } - actual = namespace_pkgs.implicit_namespace_packages(directory.root()) - self.assertPathsEqual(actual, expected) - - def test_parent_child_relationship_of_namespace_and_nested_standard_pkgs(self): - directory = TempDir() - directory.add_file("foo/bar/__init__.py") - directory.add_file("foo/bar/biff/another_module.py") - directory.add_file("foo/bar/biff/__init__.py") - directory.add_file("foo/bar/boof/big_module.py") - directory.add_file("foo/bar/boof/__init__.py") - directory.add_file("fim/in_a_ns_pkg.py") - - expected = { - directory.root() + "/foo", - directory.root() + "/fim", - } - actual = namespace_pkgs.implicit_namespace_packages(directory.root()) - self.assertPathsEqual(actual, expected) - - def test_recognized_all_nonstandard_module_types(self): - directory = TempDir() - directory.add_file("ayy/my_module.pyc") - directory.add_file("bee/ccc/dee/eee.so") - directory.add_file("eff/jee/aych.pyd") - - expected = { - directory.root() + "/ayy", - directory.root() + "/bee", - directory.root() + "/bee/ccc", - directory.root() + "/bee/ccc/dee", - directory.root() + "/eff", - directory.root() + "/eff/jee", - } - actual = namespace_pkgs.implicit_namespace_packages(directory.root()) - self.assertPathsEqual(actual, expected) - - def test_skips_ignored_directories(self): - directory = TempDir() - directory.add_file("foo/boo/my_module.py") - directory.add_file("foo/bar/another_module.py") - - expected = { - directory.root() + "/foo", - directory.root() + "/foo/bar", - } - actual = namespace_pkgs.implicit_namespace_packages( - directory.root(), - ignored_dirnames=[directory.root() + "/foo/boo"], - ) - self.assertPathsEqual(actual, expected) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/pypi/whl_installer/wheel_installer_test.py b/tests/pypi/whl_installer/wheel_installer_test.py index ef5a2483ab..7040b0cfd8 100644 --- a/tests/pypi/whl_installer/wheel_installer_test.py +++ b/tests/pypi/whl_installer/wheel_installer_test.py @@ -70,7 +70,6 @@ def test_wheel_exists(self) -> None: Path(self.wheel_path), installation_dir=Path(self.wheel_dir), extras={}, - enable_implicit_namespace_pkgs=False, platforms=[], enable_pipstar=False, ) From c0415c67e6f9c0951176354e0256a55e85e475aa Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 28 May 2025 00:54:48 +0900 Subject: [PATCH 06/43] cleanup(pycross): remove the partially migrated code (#2906) The migration effort has stalled and we closed the initiative. #1360 --- MODULE.bazel | 1 - WORKSPACE | 13 +- python/private/internal_dev_deps.bzl | 12 -- ...d-new-file-for-testing-patch-support.patch | 17 -- tests/pycross/BUILD.bazel | 64 ------ .../pycross/patched_py_wheel_library_test.py | 40 ---- tests/pycross/py_wheel_library_test.py | 46 ---- third_party/rules_pycross/LICENSE | 201 ------------------ .../rules_pycross/pycross/private/BUILD.bazel | 14 -- .../pycross/private/providers.bzl | 32 --- .../pycross/private/tools/BUILD.bazel | 26 --- .../pycross/private/tools/wheel_installer.py | 196 ----------------- .../pycross/private/wheel_library.bzl | 174 --------------- 13 files changed, 1 insertion(+), 835 deletions(-) delete mode 100644 tests/pycross/0001-Add-new-file-for-testing-patch-support.patch delete mode 100644 tests/pycross/BUILD.bazel delete mode 100644 tests/pycross/patched_py_wheel_library_test.py delete mode 100644 tests/pycross/py_wheel_library_test.py delete mode 100644 third_party/rules_pycross/LICENSE delete mode 100644 third_party/rules_pycross/pycross/private/BUILD.bazel delete mode 100644 third_party/rules_pycross/pycross/private/providers.bzl delete mode 100644 third_party/rules_pycross/pycross/private/tools/BUILD.bazel delete mode 100644 third_party/rules_pycross/pycross/private/tools/wheel_installer.py delete mode 100644 third_party/rules_pycross/pycross/private/wheel_library.bzl diff --git a/MODULE.bazel b/MODULE.bazel index d0f7cc4afa..fa24ed04ba 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -102,7 +102,6 @@ use_repo( internal_dev_deps, "buildkite_config", "rules_python_runtime_env_tc_info", - "wheel_for_testing", ) # Add gazelle plugin so that we can run the gazelle example as an e2e integration diff --git a/WORKSPACE b/WORKSPACE index 3ad83ca04b..dddc5105ed 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -78,7 +78,7 @@ python_register_multi_toolchains( python_versions = PYTHON_VERSIONS, ) -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") # Used for Bazel CI http_archive( @@ -155,14 +155,3 @@ pip_parse( load("@dev_pip//:requirements.bzl", docs_install_deps = "install_deps") docs_install_deps() - -# This wheel is purely here to validate the wheel extraction code. It's not -# intended for anything else. -http_file( - name = "wheel_for_testing", - downloaded_file_path = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - sha256 = "0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2", - urls = [ - "https://files.pythonhosted.org/packages/50/67/3e966d99a07d60a21a21d7ec016e9e4c2642a86fea251ec68677daf71d4d/numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - ], -) diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index 4f2cca0b42..600c934ace 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -14,23 +14,11 @@ """Module extension for internal dev_dependency=True setup.""" load("@bazel_ci_rules//:rbe_repo.bzl", "rbe_preconfig") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") load(":runtime_env_repo.bzl", "runtime_env_repo") def _internal_dev_deps_impl(mctx): _ = mctx # @unused - # This wheel is purely here to validate the wheel extraction code. It's not - # intended for anything else. - http_file( - name = "wheel_for_testing", - downloaded_file_path = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - sha256 = "0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2", - urls = [ - "https://files.pythonhosted.org/packages/50/67/3e966d99a07d60a21a21d7ec016e9e4c2642a86fea251ec68677daf71d4d/numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - ], - ) - # Creates a default toolchain config for RBE. # Use this as is if you are using the rbe_ubuntu16_04 container, # otherwise refer to RBE docs. diff --git a/tests/pycross/0001-Add-new-file-for-testing-patch-support.patch b/tests/pycross/0001-Add-new-file-for-testing-patch-support.patch deleted file mode 100644 index fcbc3096ef..0000000000 --- a/tests/pycross/0001-Add-new-file-for-testing-patch-support.patch +++ /dev/null @@ -1,17 +0,0 @@ -From b2ebe6fe67ff48edaf2ae937d24b1f0b67c16f81 Mon Sep 17 00:00:00 2001 -From: Philipp Schrader -Date: Thu, 28 Sep 2023 09:02:44 -0700 -Subject: [PATCH] Add new file for testing patch support - ---- - site-packages/numpy/file_added_via_patch.txt | 1 + - 1 file changed, 1 insertion(+) - create mode 100644 site-packages/numpy/file_added_via_patch.txt - -diff --git a/site-packages/numpy/file_added_via_patch.txt b/site-packages/numpy/file_added_via_patch.txt -new file mode 100644 -index 0000000..9d947a4 ---- /dev/null -+++ b/site-packages/numpy/file_added_via_patch.txt -@@ -0,0 +1 @@ -+Hello from a patch! diff --git a/tests/pycross/BUILD.bazel b/tests/pycross/BUILD.bazel deleted file mode 100644 index e90b60e17e..0000000000 --- a/tests/pycross/BUILD.bazel +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load("//python:py_test.bzl", "py_test") -load("//third_party/rules_pycross/pycross/private:wheel_library.bzl", "py_wheel_library") # buildifier: disable=bzl-visibility - -py_wheel_library( - name = "extracted_wheel_for_testing", - wheel = "@wheel_for_testing//file", -) - -py_test( - name = "py_wheel_library_test", - srcs = [ - "py_wheel_library_test.py", - ], - data = [ - ":extracted_wheel_for_testing", - ], - deps = [ - "//python/runfiles", - ], -) - -py_wheel_library( - name = "patched_extracted_wheel_for_testing", - patch_args = [ - "-p1", - ], - patch_tool = "patch", - patches = [ - "0001-Add-new-file-for-testing-patch-support.patch", - ], - target_compatible_with = select({ - # We don't have `patch` available on the Windows CI machines. - "@platforms//os:windows": ["@platforms//:incompatible"], - "//conditions:default": [], - }), - wheel = "@wheel_for_testing//file", -) - -py_test( - name = "patched_py_wheel_library_test", - srcs = [ - "patched_py_wheel_library_test.py", - ], - data = [ - ":patched_extracted_wheel_for_testing", - ], - deps = [ - "//python/runfiles", - ], -) diff --git a/tests/pycross/patched_py_wheel_library_test.py b/tests/pycross/patched_py_wheel_library_test.py deleted file mode 100644 index e1b404a0ef..0000000000 --- a/tests/pycross/patched_py_wheel_library_test.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest -from pathlib import Path - -from python.runfiles import runfiles - -RUNFILES = runfiles.Create() - - -class TestPyWheelLibrary(unittest.TestCase): - def setUp(self): - self.extraction_dir = Path( - RUNFILES.Rlocation( - "rules_python/tests/pycross/patched_extracted_wheel_for_testing" - ) - ) - self.assertTrue(self.extraction_dir.exists(), self.extraction_dir) - self.assertTrue(self.extraction_dir.is_dir(), self.extraction_dir) - - def test_patched_file_contents(self): - """Validate that the patch got applied correctly.""" - file = self.extraction_dir / "site-packages/numpy/file_added_via_patch.txt" - self.assertEqual(file.read_text(), "Hello from a patch!\n") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/pycross/py_wheel_library_test.py b/tests/pycross/py_wheel_library_test.py deleted file mode 100644 index 25d896a1ae..0000000000 --- a/tests/pycross/py_wheel_library_test.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest -from pathlib import Path - -from python.runfiles import runfiles - -RUNFILES = runfiles.Create() - - -class TestPyWheelLibrary(unittest.TestCase): - def setUp(self): - self.extraction_dir = Path( - RUNFILES.Rlocation("rules_python/tests/pycross/extracted_wheel_for_testing") - ) - self.assertTrue(self.extraction_dir.exists(), self.extraction_dir) - self.assertTrue(self.extraction_dir.is_dir(), self.extraction_dir) - - def test_file_presence(self): - """Validate that the basic file layout looks good.""" - for path in ( - "bin/f2py", - "site-packages/numpy.libs/libgfortran-daac5196.so.5.0.0", - "site-packages/numpy/dtypes.py", - "site-packages/numpy/core/_umath_tests.cpython-311-aarch64-linux-gnu.so", - ): - print(self.extraction_dir / path) - self.assertTrue( - (self.extraction_dir / path).exists(), f"{path} does not exist" - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/third_party/rules_pycross/LICENSE b/third_party/rules_pycross/LICENSE deleted file mode 100644 index 261eeb9e9f..0000000000 --- a/third_party/rules_pycross/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/third_party/rules_pycross/pycross/private/BUILD.bazel b/third_party/rules_pycross/pycross/private/BUILD.bazel deleted file mode 100644 index f59b087027..0000000000 --- a/third_party/rules_pycross/pycross/private/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2023 Jeremy Volkman. All rights reserved. -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/third_party/rules_pycross/pycross/private/providers.bzl b/third_party/rules_pycross/pycross/private/providers.bzl deleted file mode 100644 index 47fc9f7271..0000000000 --- a/third_party/rules_pycross/pycross/private/providers.bzl +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2023 Jeremy Volkman. All rights reserved. -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Python providers.""" - -PyWheelInfo = provider( - doc = "Information about a Python wheel.", - fields = { - "name_file": "File: A file containing the canonical name of the wheel.", - "wheel_file": "File: The wheel file itself.", - }, -) - -PyTargetEnvironmentInfo = provider( - doc = "A target environment description.", - fields = { - "file": "The JSON file containing target environment information.", - "python_compatible_with": "A list of constraints used to select this platform.", - }, -) diff --git a/third_party/rules_pycross/pycross/private/tools/BUILD.bazel b/third_party/rules_pycross/pycross/private/tools/BUILD.bazel deleted file mode 100644 index 41485c18a3..0000000000 --- a/third_party/rules_pycross/pycross/private/tools/BUILD.bazel +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2023 Jeremy Volkman. All rights reserved. -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load("//python:defs.bzl", "py_binary") - -py_binary( - name = "wheel_installer", - srcs = ["wheel_installer.py"], - visibility = ["//visibility:public"], - deps = [ - "//python/private/pypi/whl_installer:lib", - "@pypi__installer//:lib", - ], -) diff --git a/third_party/rules_pycross/pycross/private/tools/wheel_installer.py b/third_party/rules_pycross/pycross/private/tools/wheel_installer.py deleted file mode 100644 index a122e67733..0000000000 --- a/third_party/rules_pycross/pycross/private/tools/wheel_installer.py +++ /dev/null @@ -1,196 +0,0 @@ -# Copyright 2023 Jeremy Volkman. All rights reserved. -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -A tool that invokes pypa/build to build the given sdist tarball. -""" - -import argparse -import os -import shutil -import subprocess -import sys -import tempfile -from pathlib import Path -from typing import Any - -from installer import install -from installer.destinations import SchemeDictionaryDestination -from installer.sources import WheelFile - -from python.private.pypi.whl_installer import namespace_pkgs - - -def setup_namespace_pkg_compatibility(wheel_dir: Path) -> None: - """Converts native namespace packages to pkgutil-style packages - - Namespace packages can be created in one of three ways. They are detailed here: - https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package - - 'pkgutil-style namespace packages' (2) and 'pkg_resources-style namespace packages' (3) works in Bazel, but - 'native namespace packages' (1) do not. - - We ensure compatibility with Bazel of method 1 by converting them into method 2. - - Args: - wheel_dir: the directory of the wheel to convert - """ - - namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages( - str(wheel_dir), - ignored_dirnames=["%s/bin" % wheel_dir], - ) - - for ns_pkg_dir in namespace_pkg_dirs: - namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir) - - -def main(args: Any) -> None: - dest_dir = args.directory - lib_dir = dest_dir / "site-packages" - destination = SchemeDictionaryDestination( - scheme_dict={ - "platlib": str(lib_dir), - "purelib": str(lib_dir), - "headers": str(dest_dir / "include"), - "scripts": str(dest_dir / "bin"), - "data": str(dest_dir / "data"), - }, - interpreter="/usr/bin/env python3", # Generic; it's not feasible to run these scripts directly. - script_kind="posix", - bytecode_optimization_levels=[0, 1], - ) - - link_dir = Path(tempfile.mkdtemp()) - if args.wheel_name_file: - with open(args.wheel_name_file, "r") as f: - wheel_name = f.read().strip() - else: - wheel_name = os.path.basename(args.wheel) - - link_path = link_dir / wheel_name - os.symlink(os.path.join(os.getcwd(), args.wheel), link_path) - - try: - with WheelFile.open(link_path) as source: - install( - source=source, - destination=destination, - # Additional metadata that is generated by the installation tool. - additional_metadata={ - "INSTALLER": b"https://github.com/bazel-contrib/rules_python/tree/main/third_party/rules_pycross", - }, - ) - finally: - shutil.rmtree(link_dir, ignore_errors=True) - - setup_namespace_pkg_compatibility(lib_dir) - - if args.patch: - if not args.patch_tool and not args.patch_tool_target: - raise ValueError("Specify one of 'patch_tool' or 'patch_tool_target'.") - - patch_args = [ - args.patch_tool or Path.cwd() / args.patch_tool_target - ] + args.patch_arg - for patch in args.patch: - with patch.open("r") as stdin: - try: - subprocess.run( - patch_args, - stdin=stdin, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - cwd=args.directory, - ) - except subprocess.CalledProcessError as error: - print(f"Patch {patch} failed to apply:") - print(error.stdout.decode("utf-8")) - raise - - -def parse_flags(argv) -> Any: - parser = argparse.ArgumentParser(description="Extract a Python wheel.") - - parser.add_argument( - "--wheel", - type=Path, - required=True, - help="The wheel file path.", - ) - - parser.add_argument( - "--wheel-name-file", - type=Path, - required=False, - help="A file containing the canonical name of the wheel.", - ) - - parser.add_argument( - "--enable-implicit-namespace-pkgs", - action="store_true", - help="If true, disables conversion of implicit namespace packages and will unzip as-is.", - ) - - parser.add_argument( - "--directory", - type=Path, - help="The output path.", - ) - - parser.add_argument( - "--patch", - type=Path, - default=[], - action="append", - help="A patch file to apply.", - ) - - parser.add_argument( - "--patch-arg", - type=str, - default=[], - action="append", - help="An argument for the patch tool when applying the patches.", - ) - - parser.add_argument( - "--patch-tool", - type=str, - help=( - "The tool from PATH to invoke when applying patches. " - "If set, --patch-tool-target is ignored." - ), - ) - - parser.add_argument( - "--patch-tool-target", - type=Path, - help=( - "The path to the tool to invoke when applying patches. " - "Ignored when --patch-tool is set." - ), - ) - - return parser.parse_args(argv[1:]) - - -if __name__ == "__main__": - # When under `bazel run`, change to the actual working dir. - if "BUILD_WORKING_DIRECTORY" in os.environ: - os.chdir(os.environ["BUILD_WORKING_DIRECTORY"]) - - main(parse_flags(sys.argv)) diff --git a/third_party/rules_pycross/pycross/private/wheel_library.bzl b/third_party/rules_pycross/pycross/private/wheel_library.bzl deleted file mode 100644 index 00d85f71b1..0000000000 --- a/third_party/rules_pycross/pycross/private/wheel_library.bzl +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright 2023 Jeremy Volkman. All rights reserved. -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Implementation of the py_wheel_library rule.""" - -load("@bazel_skylib//lib:paths.bzl", "paths") -load("//python:py_info.bzl", "PyInfo") -load(":providers.bzl", "PyWheelInfo") - -def _py_wheel_library_impl(ctx): - out = ctx.actions.declare_directory(ctx.attr.name) - - wheel_target = ctx.attr.wheel - if PyWheelInfo in wheel_target: - wheel_file = wheel_target[PyWheelInfo].wheel_file - name_file = wheel_target[PyWheelInfo].name_file - else: - wheel_file = ctx.file.wheel - name_file = None - - args = ctx.actions.args().use_param_file("--flagfile=%s") - args.add("--wheel", wheel_file) - args.add("--directory", out.path) - args.add_all(ctx.files.patches, format_each = "--patch=%s") - args.add_all(ctx.attr.patch_args, format_each = "--patch-arg=%s") - args.add("--patch-tool", ctx.attr.patch_tool) - - tools = [] - inputs = [wheel_file] + ctx.files.patches - if name_file: - inputs.append(name_file) - args.add("--wheel-name-file", name_file) - - if ctx.attr.patch_tool_target: - args.add("--patch-tool-target", ctx.attr.patch_tool_target.files_to_run.executable) - tools.append(ctx.executable.patch_tool_target) - - if ctx.attr.enable_implicit_namespace_pkgs: - args.add("--enable-implicit-namespace-pkgs") - - # We apply patches in the same action as the extraction to minimize the - # number of times we cache the wheel contents. If we were to split this - # into 2 actions, then the wheel contents would be cached twice. - ctx.actions.run( - inputs = inputs, - outputs = [out], - executable = ctx.executable._tool, - tools = tools, - arguments = [args], - # Set environment variables to make generated .pyc files reproducible. - env = { - "PYTHONHASHSEED": "0", - "SOURCE_DATE_EPOCH": "315532800", - }, - mnemonic = "WheelInstall", - progress_message = "Installing %s" % ctx.file.wheel.basename, - ) - - has_py2_only_sources = ctx.attr.python_version == "PY2" - has_py3_only_sources = ctx.attr.python_version == "PY3" - if not has_py2_only_sources: - for d in ctx.attr.deps: - if d[PyInfo].has_py2_only_sources: - has_py2_only_sources = True - break - if not has_py3_only_sources: - for d in ctx.attr.deps: - if d[PyInfo].has_py3_only_sources: - has_py3_only_sources = True - break - - # TODO: Is there a more correct way to get this runfiles-relative import path? - imp = paths.join( - ctx.label.repo_name or ctx.workspace_name, # Default to the local workspace. - ctx.label.package, - ctx.label.name, - "site-packages", # we put lib files in this subdirectory. - ) - - imports = depset( - direct = [imp], - transitive = [d[PyInfo].imports for d in ctx.attr.deps], - ) - transitive_sources = depset( - direct = [out], - transitive = [dep[PyInfo].transitive_sources for dep in ctx.attr.deps if PyInfo in dep], - ) - runfiles = ctx.runfiles(files = [out]) - for d in ctx.attr.deps: - runfiles = runfiles.merge(d[DefaultInfo].default_runfiles) - - return [ - DefaultInfo( - files = depset(direct = [out]), - runfiles = runfiles, - ), - PyInfo( - has_py2_only_sources = has_py2_only_sources, - has_py3_only_sources = has_py3_only_sources, - imports = imports, - transitive_sources = transitive_sources, - uses_shared_libraries = True, # Docs say this is unused - ), - ] - -py_wheel_library = rule( - implementation = _py_wheel_library_impl, - attrs = { - "deps": attr.label_list( - doc = "A list of this wheel's Python library dependencies.", - providers = [DefaultInfo, PyInfo], - ), - "enable_implicit_namespace_pkgs": attr.bool( - default = True, - doc = """ -If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary -and py_test targets must specify either `legacy_create_init=False` or the global Bazel option -`--incompatible_default_to_explicit_init_py` to prevent `__init__.py` being automatically generated in every directory. -This option is required to support some packages which cannot handle the conversion to pkg-util style. - """, - ), - "patch_args": attr.string_list( - default = ["-p0"], - doc = - "The arguments given to the patch tool. Defaults to -p0, " + - "however -p1 will usually be needed for patches generated by " + - "git. If multiple -p arguments are specified, the last one will take effect.", - ), - "patch_tool": attr.string( - doc = "The patch(1) utility from the host to use. " + - "If set, overrides `patch_tool_target`. Please note that setting " + - "this means that builds are not completely hermetic.", - ), - "patch_tool_target": attr.label( - executable = True, - cfg = "exec", - doc = "The label of the patch(1) utility to use. " + - "Only used if `patch_tool` is not set.", - ), - "patches": attr.label_list( - allow_files = True, - default = [], - doc = - "A list of files that are to be applied as patches after " + - "extracting the archive. This will use the patch command line tool.", - ), - "python_version": attr.string( - doc = "The python version required for this wheel ('PY2' or 'PY3')", - values = ["PY2", "PY3", ""], - ), - "wheel": attr.label( - doc = "The wheel file.", - allow_single_file = [".whl"], - mandatory = True, - ), - "_tool": attr.label( - default = Label("//third_party/rules_pycross/pycross/private/tools:wheel_installer"), - cfg = "exec", - executable = True, - ), - }, -) From 369ca91fe346a7dac760a883d36352510eac8f1d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 28 May 2025 09:50:03 +0900 Subject: [PATCH 07/43] refactor(pypi): return a list from parse_requirements (#2931) The modeling of the data structures returned by the `parse_requirements` function was not optimal and this was because historically there was more logic in the `extension.bzl` and more things were decided there. With the recent refactors it is possible to have a harder to misuse data structure from the `parse_requirements`. For each `package` we will return a struct which will have a `srcs` field that will contain easy to consume values. With this in place we can do the fix that is outlined in the referenced issue. Work towards #2648 --- python/private/pypi/extension.bzl | 172 +++---- python/private/pypi/parse_requirements.bzl | 92 +++- python/private/pypi/pip_repository.bzl | 6 +- .../parse_requirements_tests.bzl | 485 ++++++++++-------- 4 files changed, 424 insertions(+), 331 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index d3a15dfc44..b79be6e038 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -202,8 +202,12 @@ def _create_whl_repos( logger = logger, ) - for whl_name, requirements in requirements_by_platform.items(): - group_name = whl_group_mapping.get(whl_name) + exposed_packages = {} + for whl in requirements_by_platform: + if whl.is_exposed: + exposed_packages[whl.name] = None + + group_name = whl_group_mapping.get(whl.name) group_deps = requirement_cycles.get(group_name, []) # Construct args separately so that the lock file can be smaller and does not include unused @@ -214,7 +218,7 @@ def _create_whl_repos( maybe_args = dict( # The following values are safe to omit if they have false like values add_libdir_to_library_search_path = pip_attr.add_libdir_to_library_search_path, - annotation = whl_modifications.get(whl_name), + annotation = whl_modifications.get(whl.name), download_only = pip_attr.download_only, enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs, environment = pip_attr.environment, @@ -226,7 +230,7 @@ def _create_whl_repos( python_interpreter_target = python_interpreter_target, whl_patches = { p: json.encode(args) - for p, args in whl_overrides.get(whl_name, {}).items() + for p, args in whl_overrides.get(whl.name, {}).items() }, ) if not enable_pipstar: @@ -245,119 +249,99 @@ def _create_whl_repos( if v != default }) - for requirement in requirements: - for repo_name, (args, config_setting) in _whl_repos( - requirement = requirement, + for src in whl.srcs: + repo = _whl_repo( + src = src, whl_library_args = whl_library_args, download_only = pip_attr.download_only, netrc = pip_attr.netrc, auth_patterns = pip_attr.auth_patterns, python_version = major_minor, - multiple_requirements_for_whl = len(requirements) > 1., + is_multiple_versions = whl.is_multiple_versions, enable_pipstar = enable_pipstar, - ).items(): - repo_name = "{}_{}".format(pip_name, repo_name) - if repo_name in whl_libraries: - fail("Attempting to creating a duplicate library {} for {}".format( - repo_name, - whl_name, - )) + ) - whl_libraries[repo_name] = args - whl_map.setdefault(whl_name, {})[config_setting] = repo_name + repo_name = "{}_{}".format(pip_name, repo.repo_name) + if repo_name in whl_libraries: + fail("Attempting to creating a duplicate library {} for {}".format( + repo_name, + whl.name, + )) + + whl_libraries[repo_name] = repo.args + whl_map.setdefault(whl.name, {})[repo.config_setting] = repo_name return struct( whl_map = whl_map, - exposed_packages = { - whl_name: None - for whl_name, requirements in requirements_by_platform.items() - if len([r for r in requirements if r.is_exposed]) > 0 - }, + exposed_packages = exposed_packages, extra_aliases = extra_aliases, whl_libraries = whl_libraries, ) -def _whl_repos(*, requirement, whl_library_args, download_only, netrc, auth_patterns, multiple_requirements_for_whl = False, python_version, enable_pipstar = False): - ret = {} - - dists = requirement.whls - if not download_only and requirement.sdist: - dists = dists + [requirement.sdist] - - for distribution in dists: - args = dict(whl_library_args) - if netrc: - args["netrc"] = netrc - if auth_patterns: - args["auth_patterns"] = auth_patterns - - if not distribution.filename.endswith(".whl"): - # pip is not used to download wheels and the python - # `whl_library` helpers are only extracting things, however - # for sdists, they will be built by `pip`, so we still - # need to pass the extra args there. - args["extra_pip_args"] = requirement.extra_pip_args - - # This is no-op because pip is not used to download the wheel. - args.pop("download_only", None) - - args["requirement"] = requirement.line - args["urls"] = [distribution.url] - args["sha256"] = distribution.sha256 - args["filename"] = distribution.filename - if not enable_pipstar: - args["experimental_target_platforms"] = [ - # Get rid of the version fot the target platforms because we are - # passing the interpreter any way. Ideally we should search of ways - # how to pass the target platforms through the hub repo. - p.partition("_")[2] - for p in requirement.target_platforms - ] - - # Pure python wheels or sdists may need to have a platform here - target_platforms = None - if distribution.filename.endswith(".whl") and not distribution.filename.endswith("-any.whl"): - pass - elif multiple_requirements_for_whl: - target_platforms = requirement.target_platforms - - repo_name = whl_repo_name( - distribution.filename, - distribution.sha256, - ) - ret[repo_name] = ( - args, - whl_config_setting( +def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, netrc, auth_patterns, python_version, enable_pipstar = False): + args = dict(whl_library_args) + args["requirement"] = src.requirement_line + is_whl = src.filename.endswith(".whl") + + if src.extra_pip_args and not is_whl: + # pip is not used to download wheels and the python + # `whl_library` helpers are only extracting things, however + # for sdists, they will be built by `pip`, so we still + # need to pass the extra args there, so only pop this for whls + args["extra_pip_args"] = src.extra_pip_args + + if not src.url or (not is_whl and download_only): + # Fallback to a pip-installed wheel + target_platforms = src.target_platforms if is_multiple_versions else [] + return struct( + repo_name = pypi_repo_name( + normalize_name(src.distribution), + *target_platforms + ), + args = args, + config_setting = whl_config_setting( version = python_version, - filename = distribution.filename, - target_platforms = target_platforms, + target_platforms = target_platforms or None, ), ) - if ret: - return ret - - # Fallback to a pip-installed wheel - args = dict(whl_library_args) # make a copy - args["requirement"] = requirement.line - if requirement.extra_pip_args: - args["extra_pip_args"] = requirement.extra_pip_args + # This is no-op because pip is not used to download the wheel. + args.pop("download_only", None) + + if netrc: + args["netrc"] = netrc + if auth_patterns: + args["auth_patterns"] = auth_patterns + + args["urls"] = [src.url] + args["sha256"] = src.sha256 + args["filename"] = src.filename + if not enable_pipstar: + args["experimental_target_platforms"] = [ + # Get rid of the version fot the target platforms because we are + # passing the interpreter any way. Ideally we should search of ways + # how to pass the target platforms through the hub repo. + p.partition("_")[2] + for p in src.target_platforms + ] + + # Pure python wheels or sdists may need to have a platform here + target_platforms = None + if is_whl and not src.filename.endswith("-any.whl"): + pass + elif is_multiple_versions: + target_platforms = src.target_platforms - target_platforms = requirement.target_platforms if multiple_requirements_for_whl else [] - repo_name = pypi_repo_name( - normalize_name(requirement.distribution), - *target_platforms - ) - ret[repo_name] = ( - args, - whl_config_setting( + return struct( + repo_name = whl_repo_name(src.filename, src.sha256), + args = args, + config_setting = whl_config_setting( version = python_version, - target_platforms = target_platforms or None, + filename = src.filename, + target_platforms = target_platforms, ), ) - return ret - def parse_modules( module_ctx, _fail = fail, diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index bdfac46ed6..bd2981efc0 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -179,49 +179,91 @@ def parse_requirements( }), ) - ret = {} - for whl_name, reqs in sorted(requirements_by_platform.items()): + ret = [] + for name, reqs in sorted(requirements_by_platform.items()): requirement_target_platforms = {} for r in reqs.values(): target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) for p in target_platforms: requirement_target_platforms[p] = None - is_exposed = len(requirement_target_platforms) == len(requirements) - if not is_exposed and logger: + item = struct( + # Return normalized names + name = normalize_name(name), + is_exposed = len(requirement_target_platforms) == len(requirements), + is_multiple_versions = len(reqs.values()) > 1, + srcs = _package_srcs( + name = name, + reqs = reqs, + index_urls = index_urls, + env_marker_target_platforms = env_marker_target_platforms, + extract_url_srcs = extract_url_srcs, + logger = logger, + ), + ) + ret.append(item) + if not item.is_exposed and logger: logger.debug(lambda: "Package '{}' will not be exposed because it is only present on a subset of platforms: {} out of {}".format( - whl_name, + name, sorted(requirement_target_platforms), sorted(requirements), )) - # Return normalized names - ret_requirements = ret.setdefault(normalize_name(whl_name), []) + if logger: + logger.debug(lambda: "Will configure whl repos: {}".format([w.name for w in ret])) - for r in sorted(reqs.values(), key = lambda r: r.requirement_line): - whls, sdist = _add_dists( - requirement = r, - index_urls = index_urls.get(whl_name), - logger = logger, - ) + return ret - target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) - ret_requirements.append( +def _package_srcs( + *, + name, + reqs, + index_urls, + logger, + env_marker_target_platforms, + extract_url_srcs): + """A function to return sources for a particular package.""" + srcs = [] + for r in sorted(reqs.values(), key = lambda r: r.requirement_line): + whls, sdist = _add_dists( + requirement = r, + index_urls = index_urls.get(name), + logger = logger, + ) + + target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) + target_platforms = sorted(target_platforms) + + all_dists = [] + whls + if sdist: + all_dists.append(sdist) + + if extract_url_srcs and all_dists: + req_line = r.srcs.requirement + else: + all_dists = [struct( + url = "", + filename = "", + sha256 = "", + yanked = False, + )] + req_line = r.srcs.requirement_line + + for dist in all_dists: + srcs.append( struct( - distribution = r.distribution, - line = r.srcs.requirement if extract_url_srcs and (whls or sdist) else r.srcs.requirement_line, - target_platforms = sorted(target_platforms), + distribution = name, extra_pip_args = r.extra_pip_args, - whls = whls, - sdist = sdist, - is_exposed = is_exposed, + requirement_line = req_line, + target_platforms = target_platforms, + filename = dist.filename, + sha256 = dist.sha256, + url = dist.url, + yanked = dist.yanked, ), ) - if logger: - logger.debug(lambda: "Will configure whl repos: {}".format(ret.keys())) - - return ret + return srcs def select_requirement(requirements, *, platform): """A simple function to get a requirement for a particular platform. diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index c8d23f471f..724fb6ddba 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -94,15 +94,15 @@ def _pip_repository_impl(rctx): selected_requirements = {} options = None repository_platform = host_platform(rctx) - for name, requirements in requirements_by_platform.items(): + for whl in requirements_by_platform: requirement = select_requirement( - requirements, + whl.srcs, platform = None if rctx.attr.download_only else repository_platform, ) if not requirement: continue options = options or requirement.extra_pip_args - selected_requirements[name] = requirement.line + selected_requirements[whl.name] = requirement.requirement_line bzl_packages = sorted(selected_requirements.keys()) diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 497e08361f..926a7e0c50 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -100,22 +100,28 @@ def _test_simple(env): "requirements_lock": ["linux_x86_64", "windows_x86_64"], }, ) - env.expect.that_dict(got).contains_exactly({ - "foo": [ - struct( - distribution = "foo", - extra_pip_args = [], - sdist = None, - is_exposed = True, - line = "foo[extra]==0.0.1 --hash=sha256:deadbeef", - target_platforms = [ - "linux_x86_64", - "windows_x86_64", - ], - whls = [], - ), - ], - }) + env.expect.that_collection(got).contains_exactly([ + struct( + name = "foo", + is_exposed = True, + is_multiple_versions = False, + srcs = [ + struct( + distribution = "foo", + extra_pip_args = [], + requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef", + target_platforms = [ + "linux_x86_64", + "windows_x86_64", + ], + url = "", + filename = "", + sha256 = "", + yanked = False, + ), + ], + ), + ]) _tests.append(_test_simple) @@ -127,24 +133,25 @@ def _test_direct_urls_integration(env): "requirements_direct": ["linux_x86_64"], }, ) - env.expect.that_dict(got).contains_exactly({ - "foo": [ - struct( - distribution = "foo", - extra_pip_args = [], - sdist = None, - is_exposed = True, - line = "foo[extra]", - target_platforms = ["linux_x86_64"], - whls = [struct( + env.expect.that_collection(got).contains_exactly([ + struct( + name = "foo", + is_exposed = True, + is_multiple_versions = False, + srcs = [ + struct( + distribution = "foo", + extra_pip_args = [], + requirement_line = "foo[extra]", + target_platforms = ["linux_x86_64"], url = "https://some-url/package.whl", filename = "package.whl", sha256 = "", yanked = False, - )], - ), - ], - }) + ), + ], + ), + ]) _tests.append(_test_direct_urls_integration) @@ -156,21 +163,27 @@ def _test_extra_pip_args(env): }, extra_pip_args = ["--trusted-host=example.org"], ) - env.expect.that_dict(got).contains_exactly({ - "foo": [ - struct( - distribution = "foo", - extra_pip_args = ["--index-url=example.org", "--trusted-host=example.org"], - sdist = None, - is_exposed = True, - line = "foo[extra]==0.0.1 --hash=sha256:deadbeef", - target_platforms = [ - "linux_x86_64", - ], - whls = [], - ), - ], - }) + env.expect.that_collection(got).contains_exactly([ + struct( + name = "foo", + is_exposed = True, + is_multiple_versions = False, + srcs = [ + struct( + distribution = "foo", + extra_pip_args = ["--index-url=example.org", "--trusted-host=example.org"], + requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef", + target_platforms = [ + "linux_x86_64", + ], + url = "", + filename = "", + sha256 = "", + yanked = False, + ), + ], + ), + ]) _tests.append(_test_extra_pip_args) @@ -181,19 +194,25 @@ def _test_dupe_requirements(env): "requirements_lock_dupe": ["linux_x86_64"], }, ) - env.expect.that_dict(got).contains_exactly({ - "foo": [ - struct( - distribution = "foo", - extra_pip_args = [], - sdist = None, - is_exposed = True, - line = "foo[extra,extra_2]==0.0.1 --hash=sha256:deadbeef", - target_platforms = ["linux_x86_64"], - whls = [], - ), - ], - }) + env.expect.that_collection(got).contains_exactly([ + struct( + name = "foo", + is_exposed = True, + is_multiple_versions = False, + srcs = [ + struct( + distribution = "foo", + extra_pip_args = [], + requirement_line = "foo[extra,extra_2]==0.0.1 --hash=sha256:deadbeef", + target_platforms = ["linux_x86_64"], + url = "", + filename = "", + sha256 = "", + yanked = False, + ), + ], + ), + ]) _tests.append(_test_dupe_requirements) @@ -206,44 +225,57 @@ def _test_multi_os(env): }, ) - env.expect.that_dict(got).contains_exactly({ - "bar": [ - struct( - distribution = "bar", - extra_pip_args = [], - line = "bar==0.0.1 --hash=sha256:deadb00f", - target_platforms = ["windows_x86_64"], - whls = [], - sdist = None, - is_exposed = False, - ), - ], - "foo": [ - struct( - distribution = "foo", - extra_pip_args = [], - line = "foo==0.0.3 --hash=sha256:deadbaaf", - target_platforms = ["linux_x86_64"], - whls = [], - sdist = None, - is_exposed = True, - ), - struct( - distribution = "foo", - extra_pip_args = [], - line = "foo[extra]==0.0.2 --hash=sha256:deadbeef", - target_platforms = ["windows_x86_64"], - whls = [], - sdist = None, - is_exposed = True, - ), - ], - }) + env.expect.that_collection(got).contains_exactly([ + struct( + name = "bar", + is_exposed = False, + is_multiple_versions = False, + srcs = [ + struct( + distribution = "bar", + extra_pip_args = [], + requirement_line = "bar==0.0.1 --hash=sha256:deadb00f", + target_platforms = ["windows_x86_64"], + url = "", + filename = "", + sha256 = "", + yanked = False, + ), + ], + ), + struct( + name = "foo", + is_exposed = True, + is_multiple_versions = True, + srcs = [ + struct( + distribution = "foo", + extra_pip_args = [], + requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf", + target_platforms = ["linux_x86_64"], + url = "", + filename = "", + sha256 = "", + yanked = False, + ), + struct( + distribution = "foo", + extra_pip_args = [], + requirement_line = "foo[extra]==0.0.2 --hash=sha256:deadbeef", + target_platforms = ["windows_x86_64"], + url = "", + filename = "", + sha256 = "", + yanked = False, + ), + ], + ), + ]) env.expect.that_str( select_requirement( - got["foo"], + got[1].srcs, platform = "windows_x86_64", - ).line, + ).requirement_line, ).equals("foo[extra]==0.0.2 --hash=sha256:deadbeef") _tests.append(_test_multi_os) @@ -257,39 +289,52 @@ def _test_multi_os_legacy(env): }, ) - env.expect.that_dict(got).contains_exactly({ - "bar": [ - struct( - distribution = "bar", - extra_pip_args = ["--platform=manylinux_2_17_x86_64", "--python-version=39", "--implementation=cp", "--abi=cp39"], - is_exposed = False, - sdist = None, - line = "bar==0.0.1 --hash=sha256:deadb00f", - target_platforms = ["cp39_linux_x86_64"], - whls = [], - ), - ], - "foo": [ - struct( - distribution = "foo", - extra_pip_args = ["--platform=manylinux_2_17_x86_64", "--python-version=39", "--implementation=cp", "--abi=cp39"], - is_exposed = True, - sdist = None, - line = "foo==0.0.1 --hash=sha256:deadbeef", - target_platforms = ["cp39_linux_x86_64"], - whls = [], - ), - struct( - distribution = "foo", - extra_pip_args = ["--platform=macosx_10_9_arm64", "--python-version=39", "--implementation=cp", "--abi=cp39"], - is_exposed = True, - sdist = None, - line = "foo==0.0.3 --hash=sha256:deadbaaf", - target_platforms = ["cp39_osx_aarch64"], - whls = [], - ), - ], - }) + env.expect.that_collection(got).contains_exactly([ + struct( + name = "bar", + is_exposed = False, + is_multiple_versions = False, + srcs = [ + struct( + distribution = "bar", + extra_pip_args = ["--platform=manylinux_2_17_x86_64", "--python-version=39", "--implementation=cp", "--abi=cp39"], + requirement_line = "bar==0.0.1 --hash=sha256:deadb00f", + target_platforms = ["cp39_linux_x86_64"], + url = "", + filename = "", + sha256 = "", + yanked = False, + ), + ], + ), + struct( + name = "foo", + is_exposed = True, + is_multiple_versions = True, + srcs = [ + struct( + distribution = "foo", + extra_pip_args = ["--platform=manylinux_2_17_x86_64", "--python-version=39", "--implementation=cp", "--abi=cp39"], + requirement_line = "foo==0.0.1 --hash=sha256:deadbeef", + target_platforms = ["cp39_linux_x86_64"], + url = "", + filename = "", + sha256 = "", + yanked = False, + ), + struct( + distribution = "foo", + extra_pip_args = ["--platform=macosx_10_9_arm64", "--python-version=39", "--implementation=cp", "--abi=cp39"], + requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf", + target_platforms = ["cp39_osx_aarch64"], + url = "", + filename = "", + sha256 = "", + yanked = False, + ), + ], + ), + ]) _tests.append(_test_multi_os_legacy) @@ -324,30 +369,42 @@ def _test_env_marker_resolution(env): }, evaluate_markers = _mock_eval_markers, ) - env.expect.that_dict(got).contains_exactly({ - "bar": [ - struct( - distribution = "bar", - extra_pip_args = [], - is_exposed = True, - sdist = None, - line = "bar==0.0.1 --hash=sha256:deadbeef", - target_platforms = ["cp311_linux_super_exotic", "cp311_windows_x86_64"], - whls = [], - ), - ], - "foo": [ - struct( - distribution = "foo", - extra_pip_args = [], - is_exposed = False, - sdist = None, - line = "foo[extra]==0.0.1 --hash=sha256:deadbeef", - target_platforms = ["cp311_windows_x86_64"], - whls = [], - ), - ], - }) + env.expect.that_collection(got).contains_exactly([ + struct( + name = "bar", + is_exposed = True, + is_multiple_versions = False, + srcs = [ + struct( + distribution = "bar", + extra_pip_args = [], + requirement_line = "bar==0.0.1 --hash=sha256:deadbeef", + target_platforms = ["cp311_linux_super_exotic", "cp311_windows_x86_64"], + url = "", + filename = "", + sha256 = "", + yanked = False, + ), + ], + ), + struct( + name = "foo", + is_exposed = False, + is_multiple_versions = False, + srcs = [ + struct( + distribution = "foo", + extra_pip_args = [], + requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef", + target_platforms = ["cp311_windows_x86_64"], + url = "", + filename = "", + sha256 = "", + yanked = False, + ), + ], + ), + ]) _tests.append(_test_env_marker_resolution) @@ -358,28 +415,35 @@ def _test_different_package_version(env): "requirements_different_package_version": ["linux_x86_64"], }, ) - env.expect.that_dict(got).contains_exactly({ - "foo": [ - struct( - distribution = "foo", - extra_pip_args = [], - is_exposed = True, - sdist = None, - line = "foo==0.0.1 --hash=sha256:deadb00f", - target_platforms = ["linux_x86_64"], - whls = [], - ), - struct( - distribution = "foo", - extra_pip_args = [], - is_exposed = True, - sdist = None, - line = "foo==0.0.1+local --hash=sha256:deadbeef", - target_platforms = ["linux_x86_64"], - whls = [], - ), - ], - }) + env.expect.that_collection(got).contains_exactly([ + struct( + name = "foo", + is_exposed = True, + is_multiple_versions = True, + srcs = [ + struct( + distribution = "foo", + extra_pip_args = [], + requirement_line = "foo==0.0.1 --hash=sha256:deadb00f", + target_platforms = ["linux_x86_64"], + url = "", + filename = "", + sha256 = "", + yanked = False, + ), + struct( + distribution = "foo", + extra_pip_args = [], + requirement_line = "foo==0.0.1+local --hash=sha256:deadbeef", + target_platforms = ["linux_x86_64"], + url = "", + filename = "", + sha256 = "", + yanked = False, + ), + ], + ), + ]) _tests.append(_test_different_package_version) @@ -390,38 +454,35 @@ def _test_optional_hash(env): "requirements_optional_hash": ["linux_x86_64"], }, ) - env.expect.that_dict(got).contains_exactly({ - "foo": [ - struct( - distribution = "foo", - extra_pip_args = [], - sdist = None, - is_exposed = True, - line = "foo==0.0.4", - target_platforms = ["linux_x86_64"], - whls = [struct( + env.expect.that_collection(got).contains_exactly([ + struct( + name = "foo", + is_exposed = True, + is_multiple_versions = True, + srcs = [ + struct( + distribution = "foo", + extra_pip_args = [], + requirement_line = "foo==0.0.4", + target_platforms = ["linux_x86_64"], url = "https://example.org/foo-0.0.4.whl", filename = "foo-0.0.4.whl", sha256 = "", yanked = False, - )], - ), - struct( - distribution = "foo", - extra_pip_args = [], - sdist = None, - is_exposed = True, - line = "foo==0.0.5", - target_platforms = ["linux_x86_64"], - whls = [struct( + ), + struct( + distribution = "foo", + extra_pip_args = [], + requirement_line = "foo==0.0.5", + target_platforms = ["linux_x86_64"], url = "https://example.org/foo-0.0.5.whl", filename = "foo-0.0.5.whl", sha256 = "deadbeef", yanked = False, - )], - ), - ], - }) + ), + ], + ), + ]) _tests.append(_test_optional_hash) @@ -432,19 +493,25 @@ def _test_git_sources(env): "requirements_git": ["linux_x86_64"], }, ) - env.expect.that_dict(got).contains_exactly({ - "foo": [ - struct( - distribution = "foo", - extra_pip_args = [], - is_exposed = True, - sdist = None, - line = "foo @ git+https://github.com/org/foo.git@deadbeef", - target_platforms = ["linux_x86_64"], - whls = [], - ), - ], - }) + env.expect.that_collection(got).contains_exactly([ + struct( + name = "foo", + is_exposed = True, + is_multiple_versions = False, + srcs = [ + struct( + distribution = "foo", + extra_pip_args = [], + requirement_line = "foo @ git+https://github.com/org/foo.git@deadbeef", + target_platforms = ["linux_x86_64"], + url = "", + filename = "", + sha256 = "", + yanked = False, + ), + ], + ), + ]) _tests.append(_test_git_sources) From 3464c14c36e5d20a56e61952c5e06ef608aa0ed9 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 28 May 2025 22:53:50 +0900 Subject: [PATCH 08/43] fix: symlink root-level python files to the venv (#2908) As found in #2882 testing, packages like `typing-extensions` which have `.py` files at the root of the `site-packages` folder don't work and it seems that the comment about `rules_python` being too eager is only half-correct. Since `namespace_pkgs` are no longer there, we can just include all of the files and if there are collisions, they will be highlighted as build errors. Now the following works: ``` bazel build //docs --@rules_python//python/config_settings:venvs_site_packages=yes ``` Work towards #2156 --- .bazelrc | 4 +-- python/private/py_library.bzl | 19 +++++----- tests/modules/other/nspkg_single/BUILD.bazel | 10 ++++++ .../nspkg_single/site-packages/__init__.py | 1 + .../nspkg_single/site-packages/single_file.py | 5 +++ tests/venv_site_packages_libs/BUILD.bazel | 1 + tests/venv_site_packages_libs/bin.py | 1 + .../nspkg_alpha/BUILD.bazel | 2 +- .../venv_site_packages_pypi_test.py | 36 ------------------- 9 files changed, 32 insertions(+), 47 deletions(-) create mode 100644 tests/modules/other/nspkg_single/BUILD.bazel create mode 100644 tests/modules/other/nspkg_single/site-packages/__init__.py create mode 100644 tests/modules/other/nspkg_single/site-packages/single_file.py delete mode 100644 tests/venv_site_packages_libs/venv_site_packages_pypi_test.py diff --git a/.bazelrc b/.bazelrc index 4e6f2fa187..7e744fb67a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single test --test_output=errors diff --git a/python/private/py_library.bzl b/python/private/py_library.bzl index bf0c25439e..fd9dad9f20 100644 --- a/python/private/py_library.bzl +++ b/python/private/py_library.bzl @@ -253,6 +253,7 @@ def _get_site_packages_symlinks(ctx): repo_runfiles_dirname = None dirs_with_init = {} # dirname -> runfile path + site_packages_symlinks = [] for src in ctx.files.srcs: if src.extension not in PYTHON_FILE_EXTENSIONS: continue @@ -261,16 +262,19 @@ def _get_site_packages_symlinks(ctx): continue path = path.removeprefix(site_packages_root) dir_name, _, filename = path.rpartition("/") - if not dir_name: - # This would be e.g. `site-packages/__init__.py`, which isn't valid - # because it's not within a directory for an importable Python package. - # However, the pypi integration over-eagerly adds a pkgutil-style - # __init__.py file during the repo phase. Just ignore them for now. - continue - if filename.startswith("__init__."): + if dir_name and filename.startswith("__init__."): dirs_with_init[dir_name] = None repo_runfiles_dirname = runfiles_root_path(ctx, src.short_path).partition("/")[0] + elif not dir_name: + repo_runfiles_dirname = runfiles_root_path(ctx, src.short_path).partition("/")[0] + + # This would be files that do not have directories and we just need to add + # direct symlinks to them as is: + site_packages_symlinks.append(( + paths.join(repo_runfiles_dirname, site_packages_root, filename), + filename, + )) # Sort so that we encounter `foo` before `foo/bar`. This ensures we # see the top-most explicit package first. @@ -286,7 +290,6 @@ def _get_site_packages_symlinks(ctx): if not is_sub_package: first_level_explicit_packages.append(d) - site_packages_symlinks = [] for dirname in first_level_explicit_packages: site_packages_symlinks.append(( paths.join(repo_runfiles_dirname, site_packages_root, dirname), diff --git a/tests/modules/other/nspkg_single/BUILD.bazel b/tests/modules/other/nspkg_single/BUILD.bazel new file mode 100644 index 0000000000..08cb4f373e --- /dev/null +++ b/tests/modules/other/nspkg_single/BUILD.bazel @@ -0,0 +1,10 @@ +load("@rules_python//python:py_library.bzl", "py_library") + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "nspkg_single", + srcs = glob(["site-packages/**/*.py"]), + experimental_venvs_site_packages = "@rules_python//python/config_settings:venvs_site_packages", + imports = [package_name() + "/site-packages"], +) diff --git a/tests/modules/other/nspkg_single/site-packages/__init__.py b/tests/modules/other/nspkg_single/site-packages/__init__.py new file mode 100644 index 0000000000..bb26c87599 --- /dev/null +++ b/tests/modules/other/nspkg_single/site-packages/__init__.py @@ -0,0 +1 @@ +# empty, will not be added to the site-packages dir diff --git a/tests/modules/other/nspkg_single/site-packages/single_file.py b/tests/modules/other/nspkg_single/site-packages/single_file.py new file mode 100644 index 0000000000..f6d7dfd640 --- /dev/null +++ b/tests/modules/other/nspkg_single/site-packages/single_file.py @@ -0,0 +1,5 @@ +__all__ = [ + "SOMETHING", +] + +SOMETHING = "nothing" diff --git a/tests/venv_site_packages_libs/BUILD.bazel b/tests/venv_site_packages_libs/BUILD.bazel index 1f48331ff2..d5a4fe6750 100644 --- a/tests/venv_site_packages_libs/BUILD.bazel +++ b/tests/venv_site_packages_libs/BUILD.bazel @@ -13,5 +13,6 @@ py_reconfig_test( "//tests/venv_site_packages_libs/nspkg_beta", "@other//nspkg_delta", "@other//nspkg_gamma", + "@other//nspkg_single", ], ) diff --git a/tests/venv_site_packages_libs/bin.py b/tests/venv_site_packages_libs/bin.py index b944be69e3..58572a2a1e 100644 --- a/tests/venv_site_packages_libs/bin.py +++ b/tests/venv_site_packages_libs/bin.py @@ -26,6 +26,7 @@ def test_imported_from_venv(self): self.assert_imported_from_venv("nspkg.subnspkg.beta") self.assert_imported_from_venv("nspkg.subnspkg.gamma") self.assert_imported_from_venv("nspkg.subnspkg.delta") + self.assert_imported_from_venv("single_file") if __name__ == "__main__": diff --git a/tests/venv_site_packages_libs/nspkg_alpha/BUILD.bazel b/tests/venv_site_packages_libs/nspkg_alpha/BUILD.bazel index c40c3b4080..aec415f7a0 100644 --- a/tests/venv_site_packages_libs/nspkg_alpha/BUILD.bazel +++ b/tests/venv_site_packages_libs/nspkg_alpha/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_python//python:py_library.bzl", "py_library") +load("//python:py_library.bzl", "py_library") package(default_visibility = ["//visibility:public"]) diff --git a/tests/venv_site_packages_libs/venv_site_packages_pypi_test.py b/tests/venv_site_packages_libs/venv_site_packages_pypi_test.py deleted file mode 100644 index 519b258044..0000000000 --- a/tests/venv_site_packages_libs/venv_site_packages_pypi_test.py +++ /dev/null @@ -1,36 +0,0 @@ -import os -import sys -import unittest - - -class VenvSitePackagesLibraryTest(unittest.TestCase): - def test_imported_from_venv(self): - self.assertNotEqual(sys.prefix, sys.base_prefix, "Not running under a venv") - venv = sys.prefix - - from nspkg.subnspkg import alpha - - self.assertEqual(alpha.whoami, "alpha") - self.assertEqual(alpha.__name__, "nspkg.subnspkg.alpha") - - self.assertTrue( - alpha.__file__.startswith(sys.prefix), - f"\nalpha was imported, not from within the venv.\n" - + f"venv : {venv}\n" - + f"actual: {alpha.__file__}", - ) - - from nspkg.subnspkg import beta - - self.assertEqual(beta.whoami, "beta") - self.assertEqual(beta.__name__, "nspkg.subnspkg.beta") - self.assertTrue( - beta.__file__.startswith(sys.prefix), - f"\nbeta was imported, not from within the venv.\n" - + f"venv : {venv}\n" - + f"actual: {beta.__file__}", - ) - - -if __name__ == "__main__": - unittest.main() From 0d203a95d9ba6ec3365119fc709dc9eb3885f6d7 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 29 May 2025 11:27:28 +0900 Subject: [PATCH 09/43] docs: split PyPI docs up and add more (#2935) Summary: - Split the PyPI docs per topic. - Move everything to its own folder. - Separate the `bzlmod` and `WORKSPACE` documentation. Some of the features are only available in `bzlmod` and since `bzlmod` is the future having that as the default makes things a little easier. - Fix a few warnings. Fixes #2810. --- CONTRIBUTING.md | 2 +- MODULE.bazel | 1 + docs/BUILD.bazel | 3 + docs/conf.py | 4 + docs/getting-started.md | 10 +- docs/index.md | 3 +- docs/pip.md | 4 - docs/pypi-dependencies.md | 519 ------------------ docs/pypi/circular-dependencies.md | 82 +++ docs/pypi/download-workspace.md | 107 ++++ docs/pypi/download.md | 302 ++++++++++ docs/pypi/index.md | 27 + docs/pypi/lock.md | 46 ++ docs/pypi/patch.md | 10 + docs/pypi/use.md | 133 +++++ docs/requirements.txt | 1 - python/private/pypi/BUILD.bazel | 1 + python/private/pypi/pkg_aliases.bzl | 5 +- python/private/pypi/simpleapi_download.bzl | 1 + python/private/pypi/whl_config_setting.bzl | 8 +- sphinxdocs/inventories/bazel_inventory.txt | 4 + .../simpleapi_download_tests.bzl | 5 + 22 files changed, 740 insertions(+), 538 deletions(-) delete mode 100644 docs/pip.md delete mode 100644 docs/pypi-dependencies.md create mode 100644 docs/pypi/circular-dependencies.md create mode 100644 docs/pypi/download-workspace.md create mode 100644 docs/pypi/download.md create mode 100644 docs/pypi/index.md create mode 100644 docs/pypi/lock.md create mode 100644 docs/pypi/patch.md create mode 100644 docs/pypi/use.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b087119dc6..324801cfc3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,7 @@ to the actual rules_python project and begin the code review process. ## Developer guide For more more details, guidance, and tips for working with the code base, -see [DEVELOPING.md](DEVELOPING.md) +see [docs/devguide.md](./devguide) ## Formatting diff --git a/MODULE.bazel b/MODULE.bazel index fa24ed04ba..d3a95350e5 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -134,6 +134,7 @@ dev_pip.parse( download_only = True, experimental_index_url = "https://pypi.org/simple", hub_name = "dev_pip", + parallel_download = False, python_version = "3.11", requirements_lock = "//docs:requirements.txt", ) diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index b3e5f52022..852c4d4fa6 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -120,7 +120,10 @@ sphinx_stardocs( "//python/private:rule_builders_bzl", "//python/private/api:py_common_api_bzl", "//python/private/pypi:config_settings_bzl", + "//python/private/pypi:env_marker_info_bzl", "//python/private/pypi:pkg_aliases_bzl", + "//python/private/pypi:whl_config_setting_bzl", + "//python/private/pypi:whl_library_bzl", "//python/uv:lock_bzl", "//python/uv:uv_bzl", "//python/uv:uv_toolchain_bzl", diff --git a/docs/conf.py b/docs/conf.py index 96bbdb50ab..1d9f526b93 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -91,6 +91,8 @@ "api/sphinxdocs/private/sphinx_docs_library": "/api/sphinxdocs/sphinxdocs/private/sphinx_docs_library.html", "api/sphinxdocs/sphinx_docs_library": "/api/sphinxdocs/sphinxdocs/sphinx_docs_library.html", "api/sphinxdocs/inventories/index": "/api/sphinxdocs/sphinxdocs/inventories/index.html", + "pip.html": "pypi/index.html", + "pypi-dependencies.html": "pypi/index.html", } # Adapted from the template code: @@ -139,7 +141,9 @@ # --- Extlinks configuration extlinks = { + "gh-issue": (f"https://github.com/bazel-contrib/rules_python/issues/%s", "#%s issue"), "gh-path": (f"https://github.com/bazel-contrib/rules_python/tree/main/%s", "%s"), + "gh-pr": (f"https://github.com/bazel-contrib/rules_python/pulls/%s", "#%s PR"), } # --- MyST configuration diff --git a/docs/getting-started.md b/docs/getting-started.md index 60d5d5e0be..7e7b88aa8a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -8,13 +8,13 @@ It assumes you have a `requirements.txt` file with your PyPI dependencies. For more details information about configuring `rules_python`, see: * [Configuring the runtime](configuring-toolchains) -* [Configuring third party dependencies (pip/pypi)](pypi-dependencies) +* [Configuring third party dependencies (pip/pypi)](./pypi/index) * [API docs](api/index) -## Using bzlmod +## Including dependencies -The first step to using rules_python with bzlmod is to add the dependency to -your MODULE.bazel file: +The first step to using `rules_python` is to add the dependency to +your `MODULE.bazel` file: ```starlark # Update the version "0.0.0" to the release found here: @@ -30,7 +30,7 @@ pip.parse( use_repo(pip, "pypi") ``` -## Using a WORKSPACE file +### Using a WORKSPACE file Using WORKSPACE is deprecated, but still supported, and a bit more involved than using Bzlmod. Here is a simplified setup to download the prebuilt runtimes. diff --git a/docs/index.md b/docs/index.md index 4983a6a029..82023f3ad8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -95,9 +95,8 @@ See {gh-path}`Bzlmod support ` for any behaviour differences :hidden: self getting-started -pypi-dependencies +pypi/index Toolchains -pip coverage precompiling gazelle diff --git a/docs/pip.md b/docs/pip.md deleted file mode 100644 index 43d8fc4978..0000000000 --- a/docs/pip.md +++ /dev/null @@ -1,4 +0,0 @@ -(pip-integration)= -# Pip Integration - -See [PyPI dependencies](./pypi-dependencies). diff --git a/docs/pypi-dependencies.md b/docs/pypi-dependencies.md deleted file mode 100644 index b3ae7fe594..0000000000 --- a/docs/pypi-dependencies.md +++ /dev/null @@ -1,519 +0,0 @@ -:::{default-domain} bzl -::: - -# Using dependencies from PyPI - -Using PyPI packages (aka "pip install") involves two main steps. - -1. [Generating requirements file](#generating-requirements-file) -2. [Installing third party packages](#installing-third-party-packages) -3. [Using third party packages as dependencies](#using-third-party-packages) - -{#generating-requirements-file} -## Generating requirements file - -Generally, when working on a Python project, you'll have some dependencies that themselves have other dependencies. You might also specify dependency bounds instead of specific versions. So you'll need to generate a full list of all transitive dependencies and pinned versions for every dependency. - -Typically, you'd have your dependencies specified in `pyproject.toml` or `requirements.in` and generate the full pinned list of dependencies in `requirements_lock.txt`, which you can manage with the `compile_pip_requirements` Bazel rule: - -```starlark -load("@rules_python//python:pip.bzl", "compile_pip_requirements") - -compile_pip_requirements( - name = "requirements", - src = "requirements.in", - requirements_txt = "requirements_lock.txt", -) -``` - -This rule generates two targets: -- `bazel run [name].update` will regenerate the `requirements_txt` file -- `bazel test [name]_test` will test that the `requirements_txt` file is up to date - -For more documentation, see the API docs under {obj}`@rules_python//python:pip.bzl`. - -Once you generate this fully specified list of requirements, you can install the requirements with the instructions in [Installing third party packages](#installing-third-party-packages). - -:::{warning} -If you're specifying dependencies in `pyproject.toml`, make sure to include the `[build-system]` configuration, with pinned dependencies. `compile_pip_requirements` will use the build system specified to read your project's metadata, and you might see non-hermetic behavior if you don't pin the build system. - -Not specifying `[build-system]` at all will result in using a default `[build-system]` configuration, which uses unpinned versions ([ref](https://peps.python.org/pep-0518/#build-system-table)). -::: - -{#installing-third-party-packages} -## Installing third party packages - -### Using bzlmod - -To add pip dependencies to your `MODULE.bazel` file, use the `pip.parse` -extension, and call it to create the central external repo and individual wheel -external repos. Include in the `MODULE.bazel` the toolchain extension as shown -in the first bzlmod example above. - -```starlark -pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") -pip.parse( - hub_name = "my_deps", - python_version = "3.11", - requirements_lock = "//:requirements_lock_3_11.txt", -) -use_repo(pip, "my_deps") -``` -For more documentation, see the bzlmod examples under the {gh-path}`examples` folder or the documentation -for the {obj}`@rules_python//python/extensions:pip.bzl` extension. - -```{note} -We are using a host-platform compatible toolchain by default to setup pip dependencies. -During the setup phase, we create some symlinks, which may be inefficient on Windows -by default. In that case use the following `.bazelrc` options to improve performance if -you have admin privileges: - - startup --windows_enable_symlinks - -This will enable symlinks on Windows and help with bootstrap performance of setting up the -hermetic host python interpreter on this platform. Linux and OSX users should see no -difference. -``` - -### Using a WORKSPACE file - -To add pip dependencies to your `WORKSPACE`, load the `pip_parse` function and -call it to create the central external repo and individual wheel external repos. - -```starlark -load("@rules_python//python:pip.bzl", "pip_parse") - -# Create a central repo that knows about the dependencies needed from -# requirements_lock.txt. -pip_parse( - name = "my_deps", - requirements_lock = "//path/to:requirements_lock.txt", -) -# Load the starlark macro, which will define your dependencies. -load("@my_deps//:requirements.bzl", "install_deps") -# Call it to define repos for your requirements. -install_deps() -``` - -(vendoring-requirements)= -#### Vendoring the requirements.bzl file - -In some cases you may not want to generate the requirements.bzl file as a repository rule -while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module -such as a ruleset, you may want to include the requirements.bzl file rather than make your users -install the WORKSPACE setup to generate it. -See https://github.com/bazel-contrib/rules_python/issues/608 - -This is the same workflow as Gazelle, which creates `go_repository` rules with -[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos) - -To do this, use the "write to source file" pattern documented in -https://blog.aspect.dev/bazel-can-write-to-the-source-folder -to put a copy of the generated requirements.bzl into your project. -Then load the requirements.bzl file directly rather than from the generated repository. -See the example in rules_python/examples/pip_parse_vendored. - -(per-os-arch-requirements)= -### Requirements for a specific OS/Architecture - -In some cases you may need to use different requirements files for different OS, Arch combinations. This is enabled via the `requirements_by_platform` attribute in `pip.parse` extension and the `pip_parse` repository rule. The keys of the dictionary are labels to the file and the values are a list of comma separated target (os, arch) tuples. - -For example: -```starlark - # ... - requirements_by_platform = { - "requirements_linux_x86_64.txt": "linux_x86_64", - "requirements_osx.txt": "osx_*", - "requirements_linux_exotic.txt": "linux_exotic", - "requirements_some_platforms.txt": "linux_aarch64,windows_*", - }, - # For the list of standard platforms that the rules_python has toolchains for, default to - # the following requirements file. - requirements_lock = "requirements_lock.txt", -``` - -In case of duplicate platforms, `rules_python` will raise an error as there has -to be unambiguous mapping of the requirement files to the (os, arch) tuples. - -An alternative way is to use per-OS requirement attributes. -```starlark - # ... - requirements_windows = "requirements_windows.txt", - requirements_darwin = "requirements_darwin.txt", - # For the remaining platforms (which is basically only linux OS), use this file. - requirements_lock = "requirements_lock.txt", -) -``` - -### pip rules - -Note that since `pip_parse` and `pip.parse` are executed at evaluation time, -Bazel has no information about the Python toolchain and cannot enforce that the -interpreter used to invoke `pip` matches the interpreter used to run -`py_binary` targets. By default, `pip_parse` uses the system command -`"python3"`. To override this, pass in the `python_interpreter` attribute or -`python_interpreter_target` attribute to `pip_parse`. The `pip.parse` `bzlmod` extension -by default uses the hermetic python toolchain for the host platform. - -You can have multiple `pip_parse`s in the same workspace, or use the pip -extension multiple times when using bzlmod. This configuration will create -multiple external repos that have no relation to one another and may result in -downloading the same wheels numerous times. - -As with any repository rule, if you would like to ensure that `pip_parse` is -re-executed to pick up a non-hermetic change to your environment (e.g., updating -your system `python` interpreter), you can force it to re-execute by running -`bazel sync --only [pip_parse name]`. - -{#using-third-party-packages} -## Using third party packages as dependencies - -Each extracted wheel repo contains a `py_library` target representing -the wheel's contents. There are two ways to access this library. The -first uses the `requirement()` function defined in the central -repo's `//:requirements.bzl` file. This function maps a pip package -name to a label: - -```starlark -load("@my_deps//:requirements.bzl", "requirement") - -py_library( - name = "mylib", - srcs = ["mylib.py"], - deps = [ - ":myotherlib", - requirement("some_pip_dep"), - requirement("another_pip_dep"), - ] -) -``` - -The reason `requirement()` exists is to insulate from -changes to the underlying repository and label strings. However, those -labels have become directly used, so aren't able to easily change regardless. - -On the other hand, using `requirement()` has several drawbacks; see -[this issue][requirements-drawbacks] for an enumeration. If you don't -want to use `requirement()`, you can use the library -labels directly instead. For `pip_parse`, the labels are of the following form: - -```starlark -@{name}//{package} -``` - -Here `name` is the `name` attribute that was passed to `pip_parse` and -`package` is the pip package name with characters that are illegal in -Bazel label names (e.g. `-`, `.`) replaced with `_`. If you need to -update `name` from "old" to "new", then you can run the following -buildozer command: - -```shell -buildozer 'substitute deps @old//([^/]+) @new//${1}' //...:* -``` - -[requirements-drawbacks]: https://github.com/bazel-contrib/rules_python/issues/414 - -### Entry points - -If you would like to access [entry points][whl_ep], see the `py_console_script_binary` rule documentation, -which can help you create a `py_binary` target for a particular console script exposed by a package. - -[whl_ep]: https://packaging.python.org/specifications/entry-points/ - -### 'Extras' dependencies - -Any 'extras' specified in the requirements lock file will be automatically added -as transitive dependencies of the package. In the example above, you'd just put -`requirement("useful_dep")` or `@pypi//useful_dep`. - -### Consuming Wheel Dists Directly - -If you need to depend on the wheel dists themselves, for instance, to pass them -to some other packaging tool, you can get a handle to them with the -`whl_requirement` macro. For example: - -```starlark -load("@pypi//:requirements.bzl", "whl_requirement") - -filegroup( - name = "whl_files", - data = [ - # This is equivalent to "@pypi//boto3:whl" - whl_requirement("boto3"), - ] -) -``` - -### Creating a filegroup of files within a whl - -The rule {obj}`whl_filegroup` exists as an easy way to extract the necessary files -from a whl file without the need to modify the `BUILD.bazel` contents of the -whl repositories generated via `pip_repository`. Use it similarly to the `filegroup` -above. See the API docs for more information. - -(advance-topics)= -## Advanced topics - -(circular-deps)= -### Circular dependencies - -Sometimes PyPi packages contain dependency cycles -- for instance a particular -version `sphinx` (this is no longer the case in the latest version as of -2024-06-02) depends on `sphinxcontrib-serializinghtml`. When using them as -`requirement()`s, ala - -``` -py_binary( - name = "doctool", - ... - deps = [ - requirement("sphinx"), - ], -) -``` - -Bazel will protest because it doesn't support cycles in the build graph -- - -``` -ERROR: .../external/pypi_sphinxcontrib_serializinghtml/BUILD.bazel:44:6: in alias rule @pypi_sphinxcontrib_serializinghtml//:pkg: cycle in dependency graph: - //:doctool (...) - @pypi//sphinxcontrib_serializinghtml:pkg (...) -.-> @pypi_sphinxcontrib_serializinghtml//:pkg (...) -| @pypi_sphinxcontrib_serializinghtml//:_pkg (...) -| @pypi_sphinx//:pkg (...) -| @pypi_sphinx//:_pkg (...) -`-- @pypi_sphinxcontrib_serializinghtml//:pkg (...) -``` - -The `experimental_requirement_cycles` argument allows you to work around these -issues by specifying groups of packages which form cycles. `pip_parse` will -transparently fix the cycles for you and provide the cyclic dependencies -simultaneously. - -```starlark -pip_parse( - ... - experimental_requirement_cycles = { - "sphinx": [ - "sphinx", - "sphinxcontrib-serializinghtml", - ] - }, -) -``` - -`pip_parse` supports fixing multiple cycles simultaneously, however cycles must -be distinct. `apache-airflow` for instance has dependency cycles with a number -of its optional dependencies, which means those optional dependencies must all -be a part of the `airflow` cycle. For instance -- - -```starlark -pip_parse( - ... - experimental_requirement_cycles = { - "airflow": [ - "apache-airflow", - "apache-airflow-providers-common-sql", - "apache-airflow-providers-postgres", - "apache-airflow-providers-sqlite", - ] - } -) -``` - -Alternatively, one could resolve the cycle by removing one leg of it. - -For example while `apache-airflow-providers-sqlite` is "baked into" the Airflow -package, `apache-airflow-providers-postgres` is not and is an optional feature. -Rather than listing `apache-airflow[postgres]` in your `requirements.txt` which -would expose a cycle via the extra, one could either _manually_ depend on -`apache-airflow` and `apache-airflow-providers-postgres` separately as -requirements. Bazel rules which need only `apache-airflow` can take it as a -dependency, and rules which explicitly want to mix in -`apache-airflow-providers-postgres` now can. - -Alternatively, one could use `rules_python`'s patching features to remove one -leg of the dependency manually. For instance by making -`apache-airflow-providers-postgres` not explicitly depend on `apache-airflow` or -perhaps `apache-airflow-providers-common-sql`. - - -### Multi-platform support - -Multi-platform support of cross-building the wheels can be done in two ways - either -using {bzl:attr}`experimental_index_url` for the {bzl:obj}`pip.parse` bzlmod tag class -or by using the {bzl:attr}`pip.parse.download_only` setting. In this section we -are going to outline quickly how one can use the latter option. - -Let's say you have 2 requirements files: -``` -# requirements.linux_x86_64.txt ---platform=manylinux_2_17_x86_64 ---python-version=39 ---implementation=cp ---abi=cp39 - -foo==0.0.1 --hash=sha256:deadbeef -bar==0.0.1 --hash=sha256:deadb00f -``` - -``` -# requirements.osx_aarch64.txt contents ---platform=macosx_10_9_arm64 ---python-version=39 ---implementation=cp ---abi=cp39 - -foo==0.0.3 --hash=sha256:deadbaaf -``` - -With these 2 files your {bzl:obj}`pip.parse` could look like: -``` -pip.parse( - hub_name = "pip", - python_version = "3.9", - # Tell `pip` to ignore sdists - download_only = True, - requirements_by_platform = { - "requirements.linux_x86_64.txt": "linux_x86_64", - "requirements.osx_aarch64.txt": "osx_aarch64", - }, -) -``` - -With this, the `pip.parse` will create a hub repository that is going to -support only two platforms - `cp39_osx_aarch64` and `cp39_linux_x86_64` and it -will only use `wheels` and ignore any sdists that it may find on the PyPI -compatible indexes. - -```{note} -This is only supported on `bzlmd`. -``` - - - -(bazel-downloader)= -### Bazel downloader and multi-platform wheel hub repository. - -The `bzlmod` `pip.parse` call supports pulling information from `PyPI` (or a -compatible mirror) and it will ensure that the [bazel -downloader][bazel_downloader] is used for downloading the wheels. This allows -the users to use the [credential helper](#credential-helper) to authenticate -with the mirror and it also ensures that the distribution downloads are cached. -It also avoids using `pip` altogether and results in much faster dependency -fetching. - -This can be enabled by `experimental_index_url` and related flags as shown in -the {gh-path}`examples/bzlmod/MODULE.bazel` example. - -When using this feature during the `pip` extension evaluation you will see the accessed indexes similar to below: -```console -Loading: 0 packages loaded - currently loading: docs/ - Fetching module extension pip in @@//python/extensions:pip.bzl; starting - Fetching https://pypi.org/simple/twine/ -``` - -This does not mean that `rules_python` is fetching the wheels eagerly, but it -rather means that it is calling the PyPI server to get the Simple API response -to get the list of all available source and wheel distributions. Once it has -got all of the available distributions, it will select the right ones depending -on the `sha256` values in your `requirements_lock.txt` file. If `sha256` hashes -are not present in the requirements file, we will fallback to matching by version -specified in the lock file. The compatible distribution URLs will be then -written to the `MODULE.bazel.lock` file. Currently users wishing to use the -lock file with `rules_python` with this feature have to set an environment -variable `RULES_PYTHON_OS_ARCH_LOCK_FILE=0` which will become default in the -next release. - -Fetching the distribution information from the PyPI allows `rules_python` to -know which `whl` should be used on which target platform and it will determine -that by parsing the `whl` filename based on [PEP600], [PEP656] standards. This -allows the user to configure the behaviour by using the following publicly -available flags: -* {obj}`--@rules_python//python/config_settings:py_linux_libc` for selecting the Linux libc variant. -* {obj}`--@rules_python//python/config_settings:pip_whl` for selecting `whl` distribution preference. -* {obj}`--@rules_python//python/config_settings:pip_whl_osx_arch` for selecting MacOS wheel preference. -* {obj}`--@rules_python//python/config_settings:pip_whl_glibc_version` for selecting the GLIBC version compatibility. -* {obj}`--@rules_python//python/config_settings:pip_whl_muslc_version` for selecting the musl version compatibility. -* {obj}`--@rules_python//python/config_settings:pip_whl_osx_version` for selecting MacOS version compatibility. - -[bazel_downloader]: https://bazel.build/rules/lib/builtins/repository_ctx#download -[pep600]: https://peps.python.org/pep-0600/ -[pep656]: https://peps.python.org/pep-0656/ - -(credential-helper)= -### Credential Helper - -The "use Bazel downloader for python wheels" experimental feature includes support for the Bazel -[Credential Helper][cred-helper-design]. - -Your python artifact registry may provide a credential helper for you. Refer to your index's docs -to see if one is provided. - -See the [Credential Helper Spec][cred-helper-spec] for details. - -[cred-helper-design]: https://github.com/bazelbuild/proposals/blob/main/designs/2022-06-07-bazel-credential-helpers.md -[cred-helper-spec]: https://github.com/EngFlow/credential-helper-spec/blob/main/spec.md - - -#### Basic Example: - -The simplest form of a credential helper is a bash script that accepts an arg and spits out JSON to -stdout. For a service like Google Artifact Registry that uses ['Basic' HTTP Auth][rfc7617] and does -not provide a credential helper that conforms to the [spec][cred-helper-spec], the script might -look like: - -```bash -#!/bin/bash -# cred_helper.sh -ARG=$1 # but we don't do anything with it as it's always "get" - -# formatting is optional -echo '{' -echo ' "headers": {' -echo ' "Authorization": ["Basic dGVzdDoxMjPCow=="]' -echo ' }' -echo '}' -``` - -Configure Bazel to use this credential helper for your python index `example.com`: - -``` -# .bazelrc -build --credential_helper=example.com=/full/path/to/cred_helper.sh -``` - -Bazel will call this file like `cred_helper.sh get` and use the returned JSON to inject headers -into whatever HTTP(S) request it performs against `example.com`. - -[rfc7617]: https://datatracker.ietf.org/doc/html/rfc7617 - - diff --git a/docs/pypi/circular-dependencies.md b/docs/pypi/circular-dependencies.md new file mode 100644 index 0000000000..d22f5b36a7 --- /dev/null +++ b/docs/pypi/circular-dependencies.md @@ -0,0 +1,82 @@ +:::{default-domain} bzl +::: + +# Circular dependencies + +Sometimes PyPi packages contain dependency cycles -- for instance a particular +version `sphinx` (this is no longer the case in the latest version as of +2024-06-02) depends on `sphinxcontrib-serializinghtml`. When using them as +`requirement()`s, ala + +```starlark +py_binary( + name = "doctool", + ... + deps = [ + requirement("sphinx"), + ], +) +``` + +Bazel will protest because it doesn't support cycles in the build graph -- + +``` +ERROR: .../external/pypi_sphinxcontrib_serializinghtml/BUILD.bazel:44:6: in alias rule @pypi_sphinxcontrib_serializinghtml//:pkg: cycle in dependency graph: + //:doctool (...) + @pypi//sphinxcontrib_serializinghtml:pkg (...) +.-> @pypi_sphinxcontrib_serializinghtml//:pkg (...) +| @pypi_sphinxcontrib_serializinghtml//:_pkg (...) +| @pypi_sphinx//:pkg (...) +| @pypi_sphinx//:_pkg (...) +`-- @pypi_sphinxcontrib_serializinghtml//:pkg (...) +``` + +The `experimental_requirement_cycles` attribute allows you to work around these +issues by specifying groups of packages which form cycles. `pip_parse` will +transparently fix the cycles for you and provide the cyclic dependencies +simultaneously. + +```starlark + ... + experimental_requirement_cycles = { + "sphinx": [ + "sphinx", + "sphinxcontrib-serializinghtml", + ] + }, +) +``` + +`pip_parse` supports fixing multiple cycles simultaneously, however cycles must +be distinct. `apache-airflow` for instance has dependency cycles with a number +of its optional dependencies, which means those optional dependencies must all +be a part of the `airflow` cycle. For instance -- + +```starlark + ... + experimental_requirement_cycles = { + "airflow": [ + "apache-airflow", + "apache-airflow-providers-common-sql", + "apache-airflow-providers-postgres", + "apache-airflow-providers-sqlite", + ] + } +) +``` + +Alternatively, one could resolve the cycle by removing one leg of it. + +For example while `apache-airflow-providers-sqlite` is "baked into" the Airflow +package, `apache-airflow-providers-postgres` is not and is an optional feature. +Rather than listing `apache-airflow[postgres]` in your `requirements.txt` which +would expose a cycle via the extra, one could either _manually_ depend on +`apache-airflow` and `apache-airflow-providers-postgres` separately as +requirements. Bazel rules which need only `apache-airflow` can take it as a +dependency, and rules which explicitly want to mix in +`apache-airflow-providers-postgres` now can. + +Alternatively, one could use `rules_python`'s patching features to remove one +leg of the dependency manually. For instance by making +`apache-airflow-providers-postgres` not explicitly depend on `apache-airflow` or +perhaps `apache-airflow-providers-common-sql`. diff --git a/docs/pypi/download-workspace.md b/docs/pypi/download-workspace.md new file mode 100644 index 0000000000..48710095a4 --- /dev/null +++ b/docs/pypi/download-workspace.md @@ -0,0 +1,107 @@ +:::{default-domain} bzl +::: + +# Download (WORKSPACE) + +This documentation page covers how to download the PyPI dependencies in the legacy `WORKSPACE` setup. + +To add pip dependencies to your `WORKSPACE`, load the `pip_parse` function and +call it to create the central external repo and individual wheel external repos. + +```starlark +load("@rules_python//python:pip.bzl", "pip_parse") + +# Create a central repo that knows about the dependencies needed from +# requirements_lock.txt. +pip_parse( + name = "my_deps", + requirements_lock = "//path/to:requirements_lock.txt", +) + +# Load the starlark macro, which will define your dependencies. +load("@my_deps//:requirements.bzl", "install_deps") + +# Call it to define repos for your requirements. +install_deps() +``` + +## Interpreter selection + +Note that pip parse runs before the Bazel before decides which Python toolchain to use, it cannot +enforce that the interpreter used to invoke `pip` matches the interpreter used to run `py_binary` +targets. By default, `pip_parse` uses the system command `"python3"`. To override this, pass in the +{attr}`pip_parse.python_interpreter` attribute or {attr}`pip_parse.python_interpreter_target`. + +You can have multiple `pip_parse`s in the same workspace. This configuration will create multiple +external repos that have no relation to one another and may result in downloading the same wheels +numerous times. + +As with any repository rule, if you would like to ensure that `pip_parse` is +re-executed to pick up a non-hermetic change to your environment (e.g., updating +your system `python` interpreter), you can force it to re-execute by running +`bazel sync --only [pip_parse name]`. + +(per-os-arch-requirements)= +## Requirements for a specific OS/Architecture + +In some cases you may need to use different requirements files for different OS, Arch combinations. +This is enabled via the {attr}`pip_parse.requirements_by_platform` attribute. The keys of the +dictionary are labels to the file and the values are a list of comma separated target (os, arch) +tuples. + +For example: +```starlark + # ... + requirements_by_platform = { + "requirements_linux_x86_64.txt": "linux_x86_64", + "requirements_osx.txt": "osx_*", + "requirements_linux_exotic.txt": "linux_exotic", + "requirements_some_platforms.txt": "linux_aarch64,windows_*", + }, + # For the list of standard platforms that the rules_python has toolchains for, default to + # the following requirements file. + requirements_lock = "requirements_lock.txt", +``` + +In case of duplicate platforms, `rules_python` will raise an error as there has +to be unambiguous mapping of the requirement files to the (os, arch) tuples. + +An alternative way is to use per-OS requirement attributes. +```starlark + # ... + requirements_windows = "requirements_windows.txt", + requirements_darwin = "requirements_darwin.txt", + # For the remaining platforms (which is basically only linux OS), use this file. + requirements_lock = "requirements_lock.txt", +) +``` + +:::{note} +If you are using a universal lock file but want to restrict the list of platforms that +the lock file will be evaluated against, consider using the aforementioned +`requirements_by_platform` attribute and listing the platforms explicitly. +::: + +(vendoring-requirements)= +## Vendoring the requirements.bzl file + +:::{note} +For `bzlmod`, refer to standard `bazel vendor` usage if you want to really vendor it, otherwise +just use the `pip` extension as you would normally. + +However, be aware that there are caveats when doing so. +::: + +In some cases you may not want to generate the requirements.bzl file as a repository rule +while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module +such as a ruleset, you may want to include the `requirements.bzl` file rather than make your users +install the `WORKSPACE` setup to generate it, see {gh-issue}`608`. + +This is the same workflow as Gazelle, which creates `go_repository` rules with +[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos) + +To do this, use the "write to source file" pattern documented in + +to put a copy of the generated `requirements.bzl` into your project. +Then load the requirements.bzl file directly rather than from the generated repository. +See the example in {gh-path}`examples/pip_parse_vendored`. diff --git a/docs/pypi/download.md b/docs/pypi/download.md new file mode 100644 index 0000000000..18d6699ab3 --- /dev/null +++ b/docs/pypi/download.md @@ -0,0 +1,302 @@ +:::{default-domain} bzl +::: + +# Download (bzlmod) + +:::{seealso} +For WORKSPACE instructions see [here](./download-workspace). +::: + +To add PyPI dependencies to your `MODULE.bazel` file, use the `pip.parse` +extension, and call it to create the central external repo and individual wheel +external repos. Include in the `MODULE.bazel` the toolchain extension as shown +in the first bzlmod example above. + +```starlark +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") + +pip.parse( + hub_name = "my_deps", + python_version = "3.13", + requirements_lock = "//:requirements_lock_3_11.txt", +) + +use_repo(pip, "my_deps") +``` + +For more documentation, see the bzlmod examples under the {gh-path}`examples` folder or the documentation +for the {obj}`@rules_python//python/extensions:pip.bzl` extension. + +:::note} +We are using a host-platform compatible toolchain by default to setup pip dependencies. +During the setup phase, we create some symlinks, which may be inefficient on Windows +by default. In that case use the following `.bazelrc` options to improve performance if +you have admin privileges: + + startup --windows_enable_symlinks + +This will enable symlinks on Windows and help with bootstrap performance of setting up the +hermetic host python interpreter on this platform. Linux and OSX users should see no +difference. +::: + +## Interpreter selection + +The {obj}`pip.parse` `bzlmod` extension by default uses the hermetic python toolchain for the host +platform, but you can customize the interpreter using {attr}`pip.parse.python_interpreter` and +{attr}`pip.parse.python_interpreter_target`. + +You can use the pip extension multiple times. This configuration will create +multiple external repos that have no relation to one another and may result in +downloading the same wheels numerous times. + +As with any repository rule or extension, if you would like to ensure that `pip_parse` is +re-executed to pick up a non-hermetic change to your environment (e.g., updating your system +`python` interpreter), you can force it to re-execute by running `bazel sync --only [pip_parse +name]`. + +(per-os-arch-requirements)= +## Requirements for a specific OS/Architecture + +In some cases you may need to use different requirements files for different OS, Arch combinations. +This is enabled via the `requirements_by_platform` attribute in `pip.parse` extension and the +{obj}`pip.parse` tag class. The keys of the dictionary are labels to the file and the values are a +list of comma separated target (os, arch) tuples. + +For example: +```starlark + # ... + requirements_by_platform = { + "requirements_linux_x86_64.txt": "linux_x86_64", + "requirements_osx.txt": "osx_*", + "requirements_linux_exotic.txt": "linux_exotic", + "requirements_some_platforms.txt": "linux_aarch64,windows_*", + }, + # For the list of standard platforms that the rules_python has toolchains for, default to + # the following requirements file. + requirements_lock = "requirements_lock.txt", +``` + +In case of duplicate platforms, `rules_python` will raise an error as there has +to be unambiguous mapping of the requirement files to the (os, arch) tuples. + +An alternative way is to use per-OS requirement attributes. +```starlark + # ... + requirements_windows = "requirements_windows.txt", + requirements_darwin = "requirements_darwin.txt", + # For the remaining platforms (which is basically only linux OS), use this file. + requirements_lock = "requirements_lock.txt", +) +``` + +:::{note} +If you are using a universal lock file but want to restrict the list of platforms that +the lock file will be evaluated against, consider using the aforementioned +`requirements_by_platform` attribute and listing the platforms explicitly. +::: + +## Multi-platform support + +Historically the {obj}`pip_parse` and {obj}`pip.parse` have been only downloading/building +Python dependencies for the host platform that the `bazel` commands are executed on. Over +the years people started needing support for building containers and usually that involves +fetching dependencies for a particular target platform that may be other than the host +platform. + +Multi-platform support of cross-building the wheels can be done in two ways: +1. using {attr}`experimental_index_url` for the {bzl:obj}`pip.parse` bzlmod tag class +2. using {attr}`pip.parse.download_only` setting. + +:::{warning} +This will not for sdists with C extensions, but pure Python sdists may still work using the first +approach. +::: + +### Using `download_only` attribute + +Let's say you have 2 requirements files: +``` +# requirements.linux_x86_64.txt +--platform=manylinux_2_17_x86_64 +--python-version=39 +--implementation=cp +--abi=cp39 + +foo==0.0.1 --hash=sha256:deadbeef +bar==0.0.1 --hash=sha256:deadb00f +``` + +``` +# requirements.osx_aarch64.txt contents +--platform=macosx_10_9_arm64 +--python-version=39 +--implementation=cp +--abi=cp39 + +foo==0.0.3 --hash=sha256:deadbaaf +``` + +With these 2 files your {bzl:obj}`pip.parse` could look like: +```starlark +pip.parse( + hub_name = "pip", + python_version = "3.9", + # Tell `pip` to ignore sdists + download_only = True, + requirements_by_platform = { + "requirements.linux_x86_64.txt": "linux_x86_64", + "requirements.osx_aarch64.txt": "osx_aarch64", + }, +) +``` + +With this, the `pip.parse` will create a hub repository that is going to +support only two platforms - `cp39_osx_aarch64` and `cp39_linux_x86_64` and it +will only use `wheels` and ignore any sdists that it may find on the PyPI +compatible indexes. + +:::{warning} +Because bazel is not aware what exactly is downloaded, the same wheel may be downloaded +multiple times. +::: + +:::{note} +This will only work for wheel-only setups, i.e. all of your dependencies need to have wheels +available on the PyPI index that you use. +::: + +### Customizing `Requires-Dist` resolution + +:::{note} +Currently this is disabled by default, but you can turn it on using +{envvar}`RULES_PYTHON_ENABLE_PIPSTAR` environment variable. +::: + +In order to understand what dependencies to pull for a particular package +`rules_python` parses the `whl` file [`METADATA`][metadata]. +Packages can express dependencies via `Requires-Dist` and they can add conditions using +"environment markers", which represent the Python version, OS, etc. + +While the PyPI integration provides reasonable defaults to support most +platforms and environment markers, the values it uses can be customized in case +more esoteric configurations are needed. + +To customize the values used, you need to do two things: +1. Define a target that returns {obj}`EnvMarkerInfo` +2. Set the {obj}`//python/config_settings:pip_env_marker_config` flag to + the target defined in (1). + +The keys and values should be compatible with the [PyPA dependency specifiers +specification](https://packaging.python.org/en/latest/specifications/dependency-specifiers/). +This is not strictly enforced, however, so you can return a subset of keys or +additional keys, which become available during dependency evaluation. + +[metadata]: https://packaging.python.org/en/latest/specifications/core-metadata/ + +(bazel-downloader)= +### Bazel downloader and multi-platform wheel hub repository. + +:::{warning} +This is currently still experimental and whilst it has been proven to work in quite a few +environments, the APIs are still being finalized and there may be changes to the APIs for this +feature without much notice. + +The issues that you can subscribe to for updates are: +* {gh-issue}`260` +* {gh-issue}`1357` +::: + +The {obj}`pip` extension supports pulling information from `PyPI` (or a compatible mirror) and it +will ensure that the [bazel downloader][bazel_downloader] is used for downloading the wheels. + +This provides the following benefits: +* Integration with the [credential_helper](#credential-helper) to authenticate with private + mirrors. +* Cache the downloaded wheels speeding up the consecutive re-initialization of the repositories. +* Reuse the same instance of the wheel for multiple target platforms. +* Allow using transitions and targeting free-threaded and musl platforms more easily. +* Avoids `pip` for wheel fetching and results in much faster dependency fetching. + +To enable the feature specify {attr}`pip.parse.experimental_index_url` as shown in +the {gh-path}`examples/bzlmod/MODULE.bazel` example. + +Similar to [uv](https://docs.astral.sh/uv/configuration/indexes/), one can override the +index that is used for a single package. By default we first search in the index specified by +{attr}`pip.parse.experimental_index_url`, then we iterate through the +{attr}`pip.parse.experimental_extra_index_urls` unless there are overrides specified via +{attr}`pip.parse.experimental_index_url_overrides`. + +When using this feature during the `pip` extension evaluation you will see the accessed indexes similar to below: +```console +Loading: 0 packages loaded + Fetching module extension @@//python/extensions:pip.bzl%pip; Fetch package lists from PyPI index + Fetching https://pypi.org/simple/jinja2/ + +``` + +This does not mean that `rules_python` is fetching the wheels eagerly, but it +rather means that it is calling the PyPI server to get the Simple API response +to get the list of all available source and wheel distributions. Once it has +got all of the available distributions, it will select the right ones depending +on the `sha256` values in your `requirements_lock.txt` file. If `sha256` hashes +are not present in the requirements file, we will fallback to matching by version +specified in the lock file. + +Fetching the distribution information from the PyPI allows `rules_python` to +know which `whl` should be used on which target platform and it will determine +that by parsing the `whl` filename based on [PEP600], [PEP656] standards. This +allows the user to configure the behaviour by using the following publicly +available flags: +* {obj}`--@rules_python//python/config_settings:py_linux_libc` for selecting the Linux libc variant. +* {obj}`--@rules_python//python/config_settings:pip_whl` for selecting `whl` distribution preference. +* {obj}`--@rules_python//python/config_settings:pip_whl_osx_arch` for selecting MacOS wheel preference. +* {obj}`--@rules_python//python/config_settings:pip_whl_glibc_version` for selecting the GLIBC version compatibility. +* {obj}`--@rules_python//python/config_settings:pip_whl_muslc_version` for selecting the musl version compatibility. +* {obj}`--@rules_python//python/config_settings:pip_whl_osx_version` for selecting MacOS version compatibility. + +[bazel_downloader]: https://bazel.build/rules/lib/builtins/repository_ctx#download +[pep600]: https://peps.python.org/pep-0600/ +[pep656]: https://peps.python.org/pep-0656/ + +(credential-helper)= +## Credential Helper + +The [Bazel downloader](#bazel-downloader) usage allows for the Bazel +[Credential Helper][cred-helper-design]. +Your python artifact registry may provide a credential helper for you. +Refer to your index's docs to see if one is provided. + +The simplest form of a credential helper is a bash script that accepts an arg and spits out JSON to +stdout. For a service like Google Artifact Registry that uses ['Basic' HTTP Auth][rfc7617] and does +not provide a credential helper that conforms to the [spec][cred-helper-spec], the script might +look like: + +```bash +#!/bin/bash +# cred_helper.sh +ARG=$1 # but we don't do anything with it as it's always "get" + +# formatting is optional +echo '{' +echo ' "headers": {' +echo ' "Authorization": ["Basic dGVzdDoxMjPCow=="]' +echo ' }' +echo '}' +``` + +Configure Bazel to use this credential helper for your python index `example.com`: + +``` +# .bazelrc +build --credential_helper=example.com=/full/path/to/cred_helper.sh +``` + +Bazel will call this file like `cred_helper.sh get` and use the returned JSON to inject headers +into whatever HTTP(S) request it performs against `example.com`. + +See the [Credential Helper Spec][cred-helper-spec] for more details. + +[rfc7617]: https://datatracker.ietf.org/doc/html/rfc7617 +[cred-helper-design]: https://github.com/bazelbuild/proposals/blob/main/designs/2022-06-07-bazel-credential-helpers.md +[cred-helper-spec]: https://github.com/EngFlow/credential-helper-spec/blob/main/spec.md diff --git a/docs/pypi/index.md b/docs/pypi/index.md new file mode 100644 index 0000000000..c300124398 --- /dev/null +++ b/docs/pypi/index.md @@ -0,0 +1,27 @@ +:::{default-domain} bzl +::: + +# Using PyPI + +Using PyPI packages (aka "pip install") involves the following main steps. + +1. [Generating requirements file](./lock) +2. Installing third party packages in [bzlmod](./download) or [WORKSPACE](./download-workspace). +3. [Using third party packages as dependencies](./use) + +With the advanced topics covered separately: +* Dealing with [circular dependencies](./circular-dependencies). + +```{toctree} +lock +download +download-workspace +use +``` + +## Advanced topics + +```{toctree} +circular-dependencies +patch +``` diff --git a/docs/pypi/lock.md b/docs/pypi/lock.md new file mode 100644 index 0000000000..c9376036fb --- /dev/null +++ b/docs/pypi/lock.md @@ -0,0 +1,46 @@ +:::{default-domain} bzl +::: + +# Lock + +:::{note} +Currently `rules_python` only supports `requirements.txt` format. +::: + +## requirements.txt + +### pip compile + +Generally, when working on a Python project, you'll have some dependencies that themselves have other dependencies. You might also specify dependency bounds instead of specific versions. So you'll need to generate a full list of all transitive dependencies and pinned versions for every dependency. + +Typically, you'd have your project dependencies specified in `pyproject.toml` or `requirements.in` and generate the full pinned list of dependencies in `requirements_lock.txt`, which you can manage with the {obj}`compile_pip_requirements`: + +```starlark +load("@rules_python//python:pip.bzl", "compile_pip_requirements") + +compile_pip_requirements( + name = "requirements", + src = "requirements.in", + requirements_txt = "requirements_lock.txt", +) +``` + +This rule generates two targets: +- `bazel run [name].update` will regenerate the `requirements_txt` file +- `bazel test [name]_test` will test that the `requirements_txt` file is up to date + +Once you generate this fully specified list of requirements, you can install the requirements ([bzlmod](./download)/[WORKSPACE](./download-workspace)). + +:::{warning} +If you're specifying dependencies in `pyproject.toml`, make sure to include the `[build-system]` configuration, with pinned dependencies. `compile_pip_requirements` will use the build system specified to read your project's metadata, and you might see non-hermetic behavior if you don't pin the build system. + +Not specifying `[build-system]` at all will result in using a default `[build-system]` configuration, which uses unpinned versions ([ref](https://peps.python.org/pep-0518/#build-system-table)). +::: + +### uv pip compile (bzlmod only) + +We also have experimental setup for the `uv pip compile` way of generating lock files. +This is well tested with the public PyPI index, but you may hit some rough edges with private +mirrors. + +For more documentation see {obj}`lock` documentation. diff --git a/docs/pypi/patch.md b/docs/pypi/patch.md new file mode 100644 index 0000000000..f341bd1091 --- /dev/null +++ b/docs/pypi/patch.md @@ -0,0 +1,10 @@ +:::{default-domain} bzl +::: + +# Patching wheels + +Sometimes the wheels have to be patched to: +* Workaround the lack of a standard `site-packages` layout ({gh-issue}`2156`) +* Include certain PRs of your choice on top of wheels and avoid building from sdist, + +You can patch the wheels by using the {attr}`pip.override.patches` attribute. diff --git a/docs/pypi/use.md b/docs/pypi/use.md new file mode 100644 index 0000000000..7a16b7d9e9 --- /dev/null +++ b/docs/pypi/use.md @@ -0,0 +1,133 @@ +:::{default-domain} bzl +::: + +# Use in BUILD.bazel files + +Once you have setup the dependencies, you are ready to start using them in your `BUILD.bazel` +files. If you haven't done so yet, set it up by following the following docs: +1. [WORKSPACE](./download-workspace) +1. [bzlmod](./download) + +To refer to targets in a hub repo `pypi`, you can do one of two things: +```starlark +py_library( + name = "my_lib", + deps = [ + "@pypi//numpy", + ], +) +``` + +Or use the `requirement` helper that needs to be loaded from the `hub` repo itself: +```starlark +load("@pypi//:requirements.bzl", "requirement") + +py_library( + deps = [ + requirement("numpy") + ], +) +``` + +Note, that the usage of the `requirement` helper is not advised and can be problematic. See the +[notes below](#requirement-helper). + +Note, that the hub repo contains the following targets for each package: +* `@pypi//numpy` which is a shorthand for `@pypi//numpy:numpy`. This is an {obj}`alias` to + `@pypi//numpy:pkg`. +* `@pypi//numpy:pkg` - the {obj}`py_library` target automatically generated by the repository + rules. +* `@pypi//numpy:data` - the {obj}`filegroup` that is for all of the extra files that are included + as data in the `pkg` target. +* `@pypi//numpy:dist_info` - the {obj}`filegroup` that is for all of the files in the `.distinfo` directory. +* `@pypi//numpy:whl` - the {obj}`filegroup` that is the `.whl` file itself which includes all of + the transitive dependencies via the {attr}`filegroup.data` attribute. + +## Entry points + +If you would like to access [entry points][whl_ep], see the `py_console_script_binary` rule documentation, +which can help you create a `py_binary` target for a particular console script exposed by a package. + +[whl_ep]: https://packaging.python.org/specifications/entry-points/ + +## 'Extras' dependencies + +Any 'extras' specified in the requirements lock file will be automatically added +as transitive dependencies of the package. In the example above, you'd just put +`requirement("useful_dep")` or `@pypi//useful_dep`. + +## Consuming Wheel Dists Directly + +If you need to depend on the wheel dists themselves, for instance, to pass them +to some other packaging tool, you can get a handle to them with the +`whl_requirement` macro. For example: + +```starlark +load("@pypi//:requirements.bzl", "whl_requirement") + +filegroup( + name = "whl_files", + data = [ + # This is equivalent to "@pypi//boto3:whl" + whl_requirement("boto3"), + ] +) +``` + +## Creating a filegroup of files within a whl + +The rule {obj}`whl_filegroup` exists as an easy way to extract the necessary files +from a whl file without the need to modify the `BUILD.bazel` contents of the +whl repositories generated via `pip_repository`. Use it similarly to the `filegroup` +above. See the API docs for more information. + +(requirement-helper)= +## A note about using the requirement helper + +Each extracted wheel repo contains a `py_library` target representing +the wheel's contents. There are two ways to access this library. The +first uses the `requirement()` function defined in the central +repo's `//:requirements.bzl` file. This function maps a pip package +name to a label: + +```starlark +load("@my_deps//:requirements.bzl", "requirement") + +py_library( + name = "mylib", + srcs = ["mylib.py"], + deps = [ + ":myotherlib", + requirement("some_pip_dep"), + requirement("another_pip_dep"), + ] +) +``` + +The reason `requirement()` exists is to insulate from +changes to the underlying repository and label strings. However, those +labels have become directly used, so aren't able to easily change regardless. + +On the other hand, using `requirement()` helper has several drawbacks: + +- It doesn't work with `buildifier` +- It doesn't work with `buildozer` +- It adds extra layer on top of normal mechanisms to refer to targets. +- It does not scale well as each type of target needs a new macro to be loaded and imported. + +If you don't want to use `requirement()`, you can use the library labels directly instead. For +`pip_parse`, the labels are of the following form: + +```starlark +@{name}//{package} +``` + +Here `name` is the `name` attribute that was passed to `pip_parse` and +`package` is the pip package name with characters that are illegal in +Bazel label names (e.g. `-`, `.`) replaced with `_`. If you need to +update `name` from "old" to "new", then you can run the following +`buildozer` command: + +```shell +buildozer 'substitute deps @old//([^/]+) @new//${1}' //...:* +``` diff --git a/docs/requirements.txt b/docs/requirements.txt index e4ec16fa5e..87c13aa8ba 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,5 @@ # This file was autogenerated by uv via the following command: # bazel run //docs:requirements.update ---index-url https://pypi.org/simple absl-py==2.2.2 \ --hash=sha256:bf25b2c2eed013ca456918c453d687eab4e8309fba81ee2f4c1a6aa2494175eb \ diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index e9036c3013..d89dc6c228 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -398,6 +398,7 @@ bzl_library( ":pep508_requirement_bzl", ":pypi_repo_utils_bzl", ":whl_metadata_bzl", + ":whl_target_platforms_bzl", "//python/private:auth_bzl", "//python/private:bzlmod_enabled_bzl", "//python/private:envsubst_bzl", diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index 28d70ff715..d71c37cb4b 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -237,9 +237,10 @@ def multiplatform_whl_aliases( Exposed only for unit tests. Args: - aliases: {type}`str | dict[whl_config_setting | str, str]`: The aliases + aliases: {type}`str | dict[struct | str, str]`: The aliases to process. Any aliases that have the filename set will be - converted to a dict of config settings to repo names. + converted to a dict of config settings to repo names. The + struct is created by {func}`whl_config_setting`. glibc_versions: {type}`list[tuple[int, int]]` list of versions that can be used in this hub repo. muslc_versions: {type}`list[tuple[int, int]]` list of versions that can be diff --git a/python/private/pypi/simpleapi_download.bzl b/python/private/pypi/simpleapi_download.bzl index e8d7d0941a..164d4e8dbd 100644 --- a/python/private/pypi/simpleapi_download.bzl +++ b/python/private/pypi/simpleapi_download.bzl @@ -83,6 +83,7 @@ def simpleapi_download( found_on_index = {} warn_overrides = False + ctx.report_progress("Fetch package lists from PyPI index") for i, index_url in enumerate(index_urls): if i != 0: # Warn the user about a potential fix for the overrides diff --git a/python/private/pypi/whl_config_setting.bzl b/python/private/pypi/whl_config_setting.bzl index 6e10eb4d27..3b81e4694f 100644 --- a/python/private/pypi/whl_config_setting.bzl +++ b/python/private/pypi/whl_config_setting.bzl @@ -21,14 +21,14 @@ def whl_config_setting(*, version = None, config_setting = None, filename = None aliases in a hub repository. Args: - version: optional(str), the version of the python toolchain that this + version: {type}`str | None`the version of the python toolchain that this whl alias is for. If not set, then non-version aware aliases will be constructed. This is mainly used for better error messages when there is no match found during a select. - config_setting: optional(Label or str), the config setting that we should use. Defaults + config_setting: {type}`str | Label | None` the config setting that we should use. Defaults to "//_config:is_python_{version}". - filename: optional(str), the distribution filename to derive the config_setting. - target_platforms: optional(list[str]), the list of target_platforms for this + filename: {type}`str | None` the distribution filename to derive the config_setting. + target_platforms: {type}`list[str] | None` the list of target_platforms for this distribution. Returns: diff --git a/sphinxdocs/inventories/bazel_inventory.txt b/sphinxdocs/inventories/bazel_inventory.txt index bbd200ddb5..e14ea76067 100644 --- a/sphinxdocs/inventories/bazel_inventory.txt +++ b/sphinxdocs/inventories/bazel_inventory.txt @@ -15,6 +15,7 @@ RBE bzl:obj 1 remote/rbe - RunEnvironmentInfo bzl:type 1 rules/lib/providers/RunEnvironmentInfo - Target bzl:type 1 rules/lib/builtins/Target - ToolchainInfo bzl:type 1 rules/lib/providers/ToolchainInfo.html - +alias bzl:rule 1 reference/be/general#alias - attr.bool bzl:type 1 rules/lib/toplevel/attr#bool - attr.int bzl:type 1 rules/lib/toplevel/attr#int - attr.int_list bzl:type 1 rules/lib/toplevel/attr#int_list - @@ -40,6 +41,7 @@ config.string_list bzl:function 1 rules/lib/toplevel/config#string_list - config.target bzl:function 1 rules/lib/toplevel/config#target - config_common.FeatureFlagInfo bzl:type 1 rules/lib/toplevel/config_common#FeatureFlagInfo - config_common.toolchain_type bzl:function 1 rules/lib/toplevel/config_common#toolchain_type - +config_setting bzl:rule 1 reference/be/general#config_setting - ctx bzl:type 1 rules/lib/builtins/repository_ctx - ctx.actions bzl:obj 1 rules/lib/builtins/ctx#actions - ctx.aspect_ids bzl:obj 1 rules/lib/builtins/ctx#aspect_ids - @@ -79,6 +81,8 @@ depset bzl:type 1 rules/lib/depset - dict bzl:type 1 rules/lib/dict - exec_compatible_with bzl:attr 1 reference/be/common-definitions#common.exec_compatible_with - exec_group bzl:function 1 rules/lib/globals/bzl#exec_group - +filegroup bzl:rule 1 reference/be/general#filegroup - +filegroup.data bzl:attr 1 reference/be/general#filegroup.data - int bzl:type 1 rules/lib/int - label bzl:type 1 concepts/labels - list bzl:type 1 rules/lib/list - diff --git a/tests/pypi/simpleapi_download/simpleapi_download_tests.bzl b/tests/pypi/simpleapi_download/simpleapi_download_tests.bzl index ce214d6e34..a96815c12c 100644 --- a/tests/pypi/simpleapi_download/simpleapi_download_tests.bzl +++ b/tests/pypi/simpleapi_download/simpleapi_download_tests.bzl @@ -43,6 +43,7 @@ def _test_simple(env): contents = simpleapi_download( ctx = struct( os = struct(environ = {}), + report_progress = lambda _: None, ), attr = struct( index_url_overrides = {}, @@ -95,6 +96,7 @@ def _test_fail(env): simpleapi_download( ctx = struct( os = struct(environ = {}), + report_progress = lambda _: None, ), attr = struct( index_url_overrides = {}, @@ -136,6 +138,7 @@ def _test_download_url(env): ctx = struct( os = struct(environ = {}), download = download, + report_progress = lambda _: None, read = lambda i: "contents of " + i, path = lambda i: "path/for/" + i, ), @@ -171,6 +174,7 @@ def _test_download_url_parallel(env): ctx = struct( os = struct(environ = {}), download = download, + report_progress = lambda _: None, read = lambda i: "contents of " + i, path = lambda i: "path/for/" + i, ), @@ -206,6 +210,7 @@ def _test_download_envsubst_url(env): ctx = struct( os = struct(environ = {"INDEX_URL": "https://example.com/main/simple/"}), download = download, + report_progress = lambda _: None, read = lambda i: "contents of " + i, path = lambda i: "path/for/" + i, ), From fd29d273e41180c56d691a67004ade742f7c7b2f Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 28 May 2025 23:37:23 -0700 Subject: [PATCH 10/43] refactor: change site_packages_symlinks to venv_symlinks (#2939) This generalizes the ability to populate the venv directory by adding and additional field, `kind`, which tells which directory of the venv to populate. A symbolic constant is used to indicate which directory so that users don't have to re-derive the platform and version specific paths that make up the venv directory names. This follows the design described by https://github.com/bazel-contrib/rules_python/issues/2156#issuecomment-2855580026 This also changes it to a depset of structs to make it more forward compatible. A provider is used because they're slightly more memory efficient than regular structs. Work towards https://github.com/bazel-contrib/rules_python/issues/2156 --- CHANGELOG.md | 4 +- python/features.bzl | 8 +- python/private/attributes.bzl | 2 +- python/private/common.bzl | 6 +- python/private/flags.bzl | 6 +- python/private/py_executable.bzl | 102 ++++++++++++++----------- python/private/py_info.bzl | 127 ++++++++++++++++++++++--------- python/private/py_library.bzl | 40 +++++----- 8 files changed, 187 insertions(+), 108 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9655b90487..4a6bdf0a96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,8 @@ END_UNRELEASED_TEMPLATE `_test` target is deprecated and will be removed in the next major release. ([#2794](https://github.com/bazel-contrib/rules_python/issues/2794) * (py_wheel) py_wheel always creates zip64-capable wheel zips +* (providers) (experimental) {obj}`PyInfo.venv_symlinks` replaces + `PyInfo.site_packages_symlinks` {#v0-0-0-fixed} ### Fixed @@ -203,7 +205,7 @@ END_UNRELEASED_TEMPLATE please check the {obj}`uv.configure` tag class. * Add support for riscv64 linux platform. * (toolchains) Add python 3.13.2 and 3.12.9 toolchains -* (providers) (experimental) {obj}`PyInfo.site_packages_symlinks` field added to +* (providers) (experimental) `PyInfo.site_packages_symlinks` field added to allow specifying links to create within the venv site packages (only applicable with {obj}`--bootstrap_impl=script`) ([#2156](https://github.com/bazelbuild/rules_python/issues/2156)). diff --git a/python/features.bzl b/python/features.bzl index 917bd3800c..b678a45241 100644 --- a/python/features.bzl +++ b/python/features.bzl @@ -31,11 +31,11 @@ def _features_typedef(): ::: :::: - ::::{field} py_info_site_packages_symlinks + ::::{field} py_info_venv_symlinks - True if the `PyInfo.site_packages_symlinks` field is available. + True if the `PyInfo.venv_symlinks` field is available. - :::{versionadded} 1.4.0 + :::{versionadded} VERSION_NEXT_FEATURE ::: :::: @@ -61,7 +61,7 @@ features = struct( TYPEDEF = _features_typedef, # keep sorted precompile = True, - py_info_site_packages_symlinks = True, + py_info_venv_symlinks = True, uses_builtin_rules = not config.enable_pystar, version = _VERSION_PRIVATE if "$Format" not in _VERSION_PRIVATE else "", ) diff --git a/python/private/attributes.bzl b/python/private/attributes.bzl index 98aba4eb23..ad8cba2e6c 100644 --- a/python/private/attributes.bzl +++ b/python/private/attributes.bzl @@ -260,7 +260,7 @@ The order of this list can matter because it affects the order that information from dependencies is merged in, which can be relevant depending on the ordering mode of depsets that are merged. -* {obj}`PyInfo.site_packages_symlinks` uses topological ordering. +* {obj}`PyInfo.venv_symlinks` uses topological ordering. See {obj}`PyInfo` for more information about the ordering of its depsets and how its fields are merged. diff --git a/python/private/common.bzl b/python/private/common.bzl index a58a9c00a4..e49dbad20c 100644 --- a/python/private/common.bzl +++ b/python/private/common.bzl @@ -378,7 +378,7 @@ def create_py_info( implicit_pyc_files, implicit_pyc_source_files, imports, - site_packages_symlinks = []): + venv_symlinks = []): """Create PyInfo provider. Args: @@ -396,7 +396,7 @@ def create_py_info( implicit_pyc_files: {type}`depset[File]` Implicitly generated pyc files that a binary can choose to include. imports: depset of strings; the import path values to propagate. - site_packages_symlinks: {type}`list[tuple[str, str]]` tuples of + venv_symlinks: {type}`list[tuple[str, str]]` tuples of `(runfiles_path, site_packages_path)` for symlinks to create in the consuming binary's venv site packages. @@ -406,7 +406,7 @@ def create_py_info( necessary for deprecated extra actions support). """ py_info = PyInfoBuilder.new() - py_info.site_packages_symlinks.add(site_packages_symlinks) + py_info.venv_symlinks.add(venv_symlinks) py_info.direct_original_sources.add(original_sources) py_info.direct_pyc_files.add(required_pyc_files) py_info.direct_pyi_files.add(ctx.files.pyi_srcs) diff --git a/python/private/flags.bzl b/python/private/flags.bzl index 40ce63b3b0..710402ba68 100644 --- a/python/private/flags.bzl +++ b/python/private/flags.bzl @@ -154,12 +154,12 @@ def _venvs_site_packages_is_enabled(ctx): flag_value = ctx.attr.experimental_venvs_site_packages[BuildSettingInfo].value return flag_value == VenvsSitePackages.YES -# Decides if libraries try to use a site-packages layout using site_packages_symlinks +# Decides if libraries try to use a site-packages layout using venv_symlinks # buildifier: disable=name-conventions VenvsSitePackages = FlagEnum( - # Use site_packages_symlinks + # Use venv_symlinks YES = "yes", - # Don't use site_packages_symlinks + # Don't use venv_symlinks NO = "no", is_enabled = _venvs_site_packages_is_enabled, ) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 24be8dd2ad..7c3e0cb757 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -54,7 +54,7 @@ load(":flags.bzl", "BootstrapImplFlag", "VenvsUseDeclareSymlinkFlag") load(":precompile.bzl", "maybe_precompile") load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") load(":py_executable_info.bzl", "PyExecutableInfo") -load(":py_info.bzl", "PyInfo") +load(":py_info.bzl", "PyInfo", "VenvSymlinkKind") load(":py_internal.bzl", "py_internal") load(":py_runtime_info.bzl", "DEFAULT_STUB_SHEBANG", "PyRuntimeInfo") load(":reexports.bzl", "BuiltinPyInfo", "BuiltinPyRuntimeInfo") @@ -543,6 +543,7 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): VenvsUseDeclareSymlinkFlag.get_value(ctx) == VenvsUseDeclareSymlinkFlag.YES ) recreate_venv_at_runtime = False + bin_dir = "{}/bin".format(venv) if not venvs_use_declare_symlink_enabled or not runtime.supports_build_time_venv: recreate_venv_at_runtime = True @@ -556,7 +557,7 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): # When the venv symlinks are disabled, the $venv/bin/python3 file isn't # needed or used at runtime. However, the zip code uses the interpreter # File object to figure out some paths. - interpreter = ctx.actions.declare_file("{}/bin/{}".format(venv, py_exe_basename)) + interpreter = ctx.actions.declare_file("{}/{}".format(bin_dir, py_exe_basename)) ctx.actions.write(interpreter, "actual:{}".format(interpreter_actual_path)) elif runtime.interpreter: @@ -568,7 +569,7 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): # declare_symlink() is required to ensure that the resulting file # in runfiles is always a symlink. An RBE implementation, for example, # may choose to write what symlink() points to instead. - interpreter = ctx.actions.declare_symlink("{}/bin/{}".format(venv, py_exe_basename)) + interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename)) interpreter_actual_path = runfiles_root_path(ctx, runtime.interpreter.short_path) rel_path = relative_path( @@ -581,7 +582,7 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): ctx.actions.symlink(output = interpreter, target_path = rel_path) else: py_exe_basename = paths.basename(runtime.interpreter_path) - interpreter = ctx.actions.declare_symlink("{}/bin/{}".format(venv, py_exe_basename)) + interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename)) ctx.actions.symlink(output = interpreter, target_path = runtime.interpreter_path) interpreter_actual_path = runtime.interpreter_path @@ -618,89 +619,104 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): }, computed_substitutions = computed_subs, ) - site_packages_symlinks = _create_site_packages_symlinks(ctx, site_packages) + + venv_dir_map = { + VenvSymlinkKind.BIN: bin_dir, + VenvSymlinkKind.LIB: site_packages, + } + venv_symlinks = _create_venv_symlinks(ctx, venv_dir_map) return struct( interpreter = interpreter, recreate_venv_at_runtime = recreate_venv_at_runtime, # Runfiles root relative path or absolute path interpreter_actual_path = interpreter_actual_path, - files_without_interpreter = [pyvenv_cfg, pth, site_init] + site_packages_symlinks, + files_without_interpreter = [pyvenv_cfg, pth, site_init] + venv_symlinks, # string; venv-relative path to the site-packages directory. venv_site_packages = venv_site_packages, ) -def _create_site_packages_symlinks(ctx, site_packages): - """Creates symlinks within site-packages. +def _create_venv_symlinks(ctx, venv_dir_map): + """Creates symlinks within the venv. Args: ctx: current rule ctx - site_packages: runfiles-root-relative path to the site-packages directory + venv_dir_map: mapping of VenvSymlinkKind constants to the + venv path. Returns: {type}`list[File]` list of the File symlink objects created. """ - # maps site-package symlink to the runfiles path it should point to + # maps venv-relative path to the runfiles path it should point to entries = depset( # NOTE: Topological ordering is used so that dependencies closer to the # binary have precedence in creating their symlinks. This allows the # binary a modicum of control over the result. order = "topological", transitive = [ - dep[PyInfo].site_packages_symlinks + dep[PyInfo].venv_symlinks for dep in ctx.attr.deps if PyInfo in dep ], ).to_list() + link_map = _build_link_map(entries) + venv_files = [] + for kind, kind_map in link_map.items(): + base = venv_dir_map[kind] + for venv_path, link_to in kind_map.items(): + venv_link = ctx.actions.declare_symlink(paths.join(base, venv_path)) + venv_link_rf_path = runfiles_root_path(ctx, venv_link.short_path) + rel_path = relative_path( + # dirname is necessary because a relative symlink is relative to + # the directory the symlink resides within. + from_ = paths.dirname(venv_link_rf_path), + to = link_to, + ) + ctx.actions.symlink(output = venv_link, target_path = rel_path) + venv_files.append(venv_link) - sp_files = [] - for sp_dir_path, link_to in link_map.items(): - sp_link = ctx.actions.declare_symlink(paths.join(site_packages, sp_dir_path)) - sp_link_rf_path = runfiles_root_path(ctx, sp_link.short_path) - rel_path = relative_path( - # dirname is necessary because a relative symlink is relative to - # the directory the symlink resides within. - from_ = paths.dirname(sp_link_rf_path), - to = link_to, - ) - ctx.actions.symlink(output = sp_link, target_path = rel_path) - sp_files.append(sp_link) - return sp_files + return venv_files def _build_link_map(entries): + # dict[str kind, dict[str rel_path, str link_to_path]] link_map = {} - for link_to_runfiles_path, site_packages_path in entries: - if site_packages_path in link_map: + for entry in entries: + kind = entry.kind + kind_map = link_map.setdefault(kind, {}) + if entry.venv_path in kind_map: # We ignore duplicates by design. The dependency closer to the # binary gets precedence due to the topological ordering. continue else: - link_map[site_packages_path] = link_to_runfiles_path + kind_map[entry.venv_path] = entry.link_to_path # An empty link_to value means to not create the site package symlink. # Because of the topological ordering, this allows binaries to remove # entries by having an earlier dependency produce empty link_to values. - for sp_dir_path, link_to in link_map.items(): - if not link_to: - link_map.pop(sp_dir_path) + for kind, kind_map in link_map.items(): + for dir_path, link_to in kind_map.items(): + if not link_to: + kind_map.pop(dir_path) - # Remove entries that would be a child path of a created symlink. - # Earlier entries have precedence to match how exact matches are handled. + # dict[str kind, dict[str rel_path, str link_to_path]] keep_link_map = {} - for _ in range(len(link_map)): - if not link_map: - break - dirname, value = link_map.popitem() - keep_link_map[dirname] = value - - prefix = dirname + "/" # Add slash to prevent /X matching /XY - for maybe_suffix in link_map.keys(): - maybe_suffix += "/" # Add slash to prevent /X matching /XY - if maybe_suffix.startswith(prefix) or prefix.startswith(maybe_suffix): - link_map.pop(maybe_suffix) + # Remove entries that would be a child path of a created symlink. + # Earlier entries have precedence to match how exact matches are handled. + for kind, kind_map in link_map.items(): + keep_kind_map = keep_link_map.setdefault(kind, {}) + for _ in range(len(kind_map)): + if not kind_map: + break + dirname, value = kind_map.popitem() + keep_kind_map[dirname] = value + prefix = dirname + "/" # Add slash to prevent /X matching /XY + for maybe_suffix in kind_map.keys(): + maybe_suffix += "/" # Add slash to prevent /X matching /XY + if maybe_suffix.startswith(prefix) or prefix.startswith(maybe_suffix): + kind_map.pop(maybe_suffix) return keep_link_map def _map_each_identity(v): diff --git a/python/private/py_info.bzl b/python/private/py_info.bzl index d175eefb69..2a2f4554e3 100644 --- a/python/private/py_info.bzl +++ b/python/private/py_info.bzl @@ -18,6 +18,64 @@ load(":builders.bzl", "builders") load(":reexports.bzl", "BuiltinPyInfo") load(":util.bzl", "define_bazel_6_provider") +def _VenvSymlinkKind_typedef(): + """An enum of types of venv directories. + + :::{field} BIN + :type: object + + Indicates to create paths under the directory that has binaries + within the venv. + ::: + + :::{field} LIB + :type: object + + Indicates to create paths under the venv's site-packages directory. + ::: + + :::{field} INCLUDE + :type: object + + Indicates to create paths under the venv's include directory. + ::: + """ + +# buildifier: disable=name-conventions +VenvSymlinkKind = struct( + TYPEDEF = _VenvSymlinkKind_typedef, + BIN = "BIN", + LIB = "LIB", + INCLUDE = "INCLUDE", +) + +# A provider is used for memory efficiency. +# buildifier: disable=name-conventions +VenvSymlinkEntry = provider( + doc = """ +An entry in `PyInfo.venv_symlinks` +""", + fields = { + "kind": """ +:type: str + +One of the {obj}`VenvSymlinkKind` values. It represents which directory within +the venv to create the path under. +""", + "link_to_path": """ +:type: str | None + +A runfiles-root relative path that `venv_path` will symlink to. If `None`, +it means to not create a symlink. +""", + "venv_path": """ +:type: str + +A path relative to the `kind` directory within the venv. +""", + }, +) + def _check_arg_type(name, required_type, value): """Check that a value is of an expected type.""" value_type = type(value) @@ -43,7 +101,7 @@ def _PyInfo_init( transitive_original_sources = depset(), direct_pyi_files = depset(), transitive_pyi_files = depset(), - site_packages_symlinks = depset()): + venv_symlinks = depset()): _check_arg_type("transitive_sources", "depset", transitive_sources) # Verify it's postorder compatible, but retain is original ordering. @@ -71,7 +129,6 @@ def _PyInfo_init( "has_py2_only_sources": has_py2_only_sources, "has_py3_only_sources": has_py2_only_sources, "imports": imports, - "site_packages_symlinks": site_packages_symlinks, "transitive_implicit_pyc_files": transitive_implicit_pyc_files, "transitive_implicit_pyc_source_files": transitive_implicit_pyc_source_files, "transitive_original_sources": transitive_original_sources, @@ -79,6 +136,7 @@ def _PyInfo_init( "transitive_pyi_files": transitive_pyi_files, "transitive_sources": transitive_sources, "uses_shared_libraries": uses_shared_libraries, + "venv_symlinks": venv_symlinks, } PyInfo, _unused_raw_py_info_ctor = define_bazel_6_provider( @@ -146,34 +204,6 @@ A depset of import path strings to be added to the `PYTHONPATH` of executable Python targets. These are accumulated from the transitive `deps`. The order of the depset is not guaranteed and may be changed in the future. It is recommended to use `default` order (the default). -""", - "site_packages_symlinks": """ -:type: depset[tuple[str | None, str]] - -A depset with `topological` ordering. - -Tuples of `(runfiles_path, site_packages_path)`. Where -* `runfiles_path` is a runfiles-root relative path. It is the path that - has the code to make importable. If `None` or empty string, then it means - to not create a site packages directory with the `site_packages_path` - name. -* `site_packages_path` is a path relative to the site-packages directory of - the venv for whatever creates the venv (typically py_binary). It makes - the code in `runfiles_path` available for import. Note that this - is created as a "raw" symlink (via `declare_symlink`). - -:::{include} /_includes/experimental_api.md -::: - -:::{tip} -The topological ordering means dependencies earlier and closer to the consumer -have precedence. This allows e.g. a binary to add dependencies that override -values from further way dependencies, such as forcing symlinks to point to -specific paths or preventing symlinks from being created. -::: - -:::{versionadded} 1.4.0 -::: """, "transitive_implicit_pyc_files": """ :type: depset[File] @@ -262,6 +292,35 @@ Whether any of this target's transitive `deps` has a shared library file (such as a `.so` file). This field is currently unused in Bazel and may go away in the future. +""", + "venv_symlinks": """ +:type: depset[VenvSymlinkEntry] + +A depset with `topological` ordering. + + +Tuples of `(runfiles_path, site_packages_path)`. Where +* `runfiles_path` is a runfiles-root relative path. It is the path that + has the code to make importable. If `None` or empty string, then it means + to not create a site packages directory with the `site_packages_path` + name. +* `site_packages_path` is a path relative to the site-packages directory of + the venv for whatever creates the venv (typically py_binary). It makes + the code in `runfiles_path` available for import. Note that this + is created as a "raw" symlink (via `declare_symlink`). + +:::{include} /_includes/experimental_api.md +::: + +:::{tip} +The topological ordering means dependencies earlier and closer to the consumer +have precedence. This allows e.g. a binary to add dependencies that override +values from further way dependencies, such as forcing symlinks to point to +specific paths or preventing symlinks from being created. +::: + +:::{versionadded} VERSION_NEXT_FEATURE +::: """, }, ) @@ -314,7 +373,7 @@ def _PyInfoBuilder_typedef(): :type: DepsetBuilder[File] ::: - :::{field} site_packages_symlinks + :::{field} venv_symlinks :type: DepsetBuilder[tuple[str | None, str]] NOTE: This depset has `topological` order @@ -358,7 +417,7 @@ def _PyInfoBuilder_new(): transitive_pyc_files = builders.DepsetBuilder(), transitive_pyi_files = builders.DepsetBuilder(), transitive_sources = builders.DepsetBuilder(), - site_packages_symlinks = builders.DepsetBuilder(order = "topological"), + venv_symlinks = builders.DepsetBuilder(order = "topological"), ) return self @@ -525,7 +584,7 @@ def _PyInfoBuilder_merge_all(self, transitive, *, direct = []): self.transitive_original_sources.add(info.transitive_original_sources) self.transitive_pyc_files.add(info.transitive_pyc_files) self.transitive_pyi_files.add(info.transitive_pyi_files) - self.site_packages_symlinks.add(info.site_packages_symlinks) + self.venv_symlinks.add(info.venv_symlinks) return self @@ -583,7 +642,7 @@ def _PyInfoBuilder_build(self): transitive_original_sources = self.transitive_original_sources.build(), transitive_pyc_files = self.transitive_pyc_files.build(), transitive_pyi_files = self.transitive_pyi_files.build(), - site_packages_symlinks = self.site_packages_symlinks.build(), + venv_symlinks = self.venv_symlinks.build(), ) else: kwargs = {} diff --git a/python/private/py_library.bzl b/python/private/py_library.bzl index fd9dad9f20..fabc880a8d 100644 --- a/python/private/py_library.bzl +++ b/python/private/py_library.bzl @@ -43,7 +43,7 @@ load( load(":flags.bzl", "AddSrcsToRunfilesFlag", "PrecompileFlag", "VenvsSitePackages") load(":precompile.bzl", "maybe_precompile") load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") -load(":py_info.bzl", "PyInfo") +load(":py_info.bzl", "PyInfo", "VenvSymlinkEntry", "VenvSymlinkKind") load(":py_internal.bzl", "py_internal") load(":reexports.bzl", "BuiltinPyInfo") load(":rule_builders.bzl", "ruleb") @@ -90,9 +90,9 @@ won't be understood as namespace packages; they'll be seen as regular packages. likely lead to conflicts with other targets that contribute to the namespace. :::{tip} -This attributes populates {obj}`PyInfo.site_packages_symlinks`, which is +This attributes populates {obj}`PyInfo.venv_symlinks`, which is a topologically ordered depset. This means dependencies closer and earlier -to a consumer have precedence. See {obj}`PyInfo.site_packages_symlinks` for +to a consumer have precedence. See {obj}`PyInfo.venv_symlinks` for more information. ::: @@ -155,9 +155,9 @@ def py_library_impl(ctx, *, semantics): runfiles = runfiles.build(ctx) imports = [] - site_packages_symlinks = [] + venv_symlinks = [] - imports, site_packages_symlinks = _get_imports_and_site_packages_symlinks(ctx, semantics) + imports, venv_symlinks = _get_imports_and_venv_symlinks(ctx, semantics) cc_info = semantics.get_cc_info_for_library(ctx) py_info, deps_transitive_sources, builtins_py_info = create_py_info( @@ -168,7 +168,7 @@ def py_library_impl(ctx, *, semantics): implicit_pyc_files = implicit_pyc_files, implicit_pyc_source_files = implicit_pyc_source_files, imports = imports, - site_packages_symlinks = site_packages_symlinks, + venv_symlinks = venv_symlinks, ) # TODO(b/253059598): Remove support for extra actions; https://github.com/bazelbuild/bazel/issues/16455 @@ -206,16 +206,16 @@ Source files are no longer added to the runfiles directly. ::: """ -def _get_imports_and_site_packages_symlinks(ctx, semantics): +def _get_imports_and_venv_symlinks(ctx, semantics): imports = depset() - site_packages_symlinks = depset() + venv_symlinks = depset() if VenvsSitePackages.is_enabled(ctx): - site_packages_symlinks = _get_site_packages_symlinks(ctx) + venv_symlinks = _get_venv_symlinks(ctx) else: imports = collect_imports(ctx, semantics) - return imports, site_packages_symlinks + return imports, venv_symlinks -def _get_site_packages_symlinks(ctx): +def _get_venv_symlinks(ctx): imports = ctx.attr.imports if len(imports) == 0: fail("When venvs_site_packages is enabled, exactly one `imports` " + @@ -253,7 +253,7 @@ def _get_site_packages_symlinks(ctx): repo_runfiles_dirname = None dirs_with_init = {} # dirname -> runfile path - site_packages_symlinks = [] + venv_symlinks = [] for src in ctx.files.srcs: if src.extension not in PYTHON_FILE_EXTENSIONS: continue @@ -271,9 +271,10 @@ def _get_site_packages_symlinks(ctx): # This would be files that do not have directories and we just need to add # direct symlinks to them as is: - site_packages_symlinks.append(( - paths.join(repo_runfiles_dirname, site_packages_root, filename), - filename, + venv_symlinks.append(VenvSymlinkEntry( + kind = VenvSymlinkKind.LIB, + link_to_path = paths.join(repo_runfiles_dirname, site_packages_root, filename), + venv_path = filename, )) # Sort so that we encounter `foo` before `foo/bar`. This ensures we @@ -291,11 +292,12 @@ def _get_site_packages_symlinks(ctx): first_level_explicit_packages.append(d) for dirname in first_level_explicit_packages: - site_packages_symlinks.append(( - paths.join(repo_runfiles_dirname, site_packages_root, dirname), - dirname, + venv_symlinks.append(VenvSymlinkEntry( + kind = VenvSymlinkKind.LIB, + link_to_path = paths.join(repo_runfiles_dirname, site_packages_root, dirname), + venv_path = dirname, )) - return site_packages_symlinks + return venv_symlinks def _repo_relative_short_path(short_path): # Convert `../+pypi+foo/some/file.py` to `some/file.py` From bbf3ab8956007f48fc012fb9316debffde8b0495 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 29 May 2025 06:21:27 -0700 Subject: [PATCH 11/43] docs: fix sphinxdocs mis-redirect (#2940) The redirect was going to a non-existent URL when viewed on the deployed docs. This was happening because the absolute paths `/api/whatever` don't exist in the deployed site -- it's actually `/en/latest/api/whatever`. This went unnoticed because it works locally (where there is no /en/latest prefix). To fix, use a relative url (relative urls are relative to the path that is redirected from) --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 1d9f526b93..8537d9996c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -87,7 +87,7 @@ "api/sphinxdocs/sphinx": "/api/sphinxdocs/sphinxdocs/sphinx.html", "api/sphinxdocs/sphinx_stardoc": "/api/sphinxdocs/sphinxdocs/sphinx_stardoc.html", "api/sphinxdocs/readthedocs": "/api/sphinxdocs/sphinxdocs/readthedocs.html", - "api/sphinxdocs/index": "/api/sphinxdocs/sphinxdocs/index.html", + "api/sphinxdocs/index": "sphinxdocs/index.html", "api/sphinxdocs/private/sphinx_docs_library": "/api/sphinxdocs/sphinxdocs/private/sphinx_docs_library.html", "api/sphinxdocs/sphinx_docs_library": "/api/sphinxdocs/sphinxdocs/sphinx_docs_library.html", "api/sphinxdocs/inventories/index": "/api/sphinxdocs/sphinxdocs/inventories/index.html", From d60cee2623bf6cedb4dbd9899eb99ac84432fb37 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 29 May 2025 08:43:56 -0700 Subject: [PATCH 12/43] feat: allow custom platform when overriding (#2880) This basically allows using any python-build-standalone archive and using it if custom flags are set. This is done through the `single_version_platform_override()` API, because such archives are inherently version and platform specific. Key changes: * The `platform` arg can be any value (mostly; it ends up in repo names) * Added `target_compatible_with` and `target_settings` args, which become the settings used on the generated toolchain() definition. The platform settings are version specific, i.e. the key `(python_version, platform)` is what maps to the TCW/TS values. If an existing platform is used, it'll override the defaults that normally come from the PLATFORMS global for the particular version. If a new platform is used, it creates a new platform entry with those settings. Along the way: * Added various docs about internal variables so they're easier to grok at a glance. Work towards https://github.com/bazel-contrib/rules_python/issues/2081 --- CHANGELOG.md | 4 + MODULE.bazel | 16 + docs/toolchains.md | 67 ++++ internal_dev_setup.bzl | 2 +- python/BUILD.bazel | 1 + python/private/BUILD.bazel | 6 + python/private/platform_info.bzl | 34 ++ python/private/py_repositories.bzl | 2 +- python/private/python.bzl | 311 +++++++++++++++--- python/private/python_repository.bzl | 3 +- python/private/pythons_hub.bzl | 30 +- python/private/repo_utils.bzl | 20 +- python/private/toolchains_repo.bzl | 131 ++++++-- python/versions.bzl | 56 +--- tests/bootstrap_impls/bin.py | 1 + tests/python/python_tests.bzl | 23 ++ tests/support/BUILD.bazel | 13 + tests/support/sh_py_run_test.bzl | 2 + tests/toolchains/BUILD.bazel | 13 + .../custom_platform_toolchain_test.py | 15 + 20 files changed, 609 insertions(+), 141 deletions(-) create mode 100644 python/private/platform_info.bzl create mode 100644 tests/toolchains/custom_platform_toolchain_test.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a6bdf0a96..a113c7411f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,6 +107,10 @@ END_UNRELEASED_TEMPLATE Set the `RULES_PYTHON_ENABLE_PIPSTAR=1` environment variable to enable it. * (utils) Add a way to run a REPL for any `rules_python` target that returns a `PyInfo` provider. +* (toolchains) Arbitrary python-build-standalone runtimes can be registered + and activated with custom flags. See the [Registering custom runtimes] + docs and {obj}`single_version_platform_override()` API docs for more + information. {#v0-0-0-removed} ### Removed diff --git a/MODULE.bazel b/MODULE.bazel index d3a95350e5..144e130c1b 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -125,6 +125,22 @@ dev_python.override( register_all_versions = True, ) +# For testing an arbitrary runtime triggered by a custom flag. +# See //tests/toolchains:custom_platform_toolchain_test +dev_python.single_version_platform_override( + platform = "linux-x86-install-only-stripped", + python_version = "3.13.1", + sha256 = "56817aa976e4886bec1677699c136cb01c1cdfe0495104c0d8ef546541864bbb", + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + target_settings = [ + "@@//tests/support:is_custom_runtime_linux-x86-install-only-stripped", + ], + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20250115/cpython-3.13.1+20250115-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz"], +) + dev_pip = use_extension( "//python/extensions:pip.bzl", "pip", diff --git a/docs/toolchains.md b/docs/toolchains.md index ada887c945..57d43d27f1 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -243,6 +243,73 @@ existing attributes: * Adding additional Python versions via {bzl:obj}`python.single_version_override` or {bzl:obj}`python.single_version_platform_override`. +### Registering custom runtimes + +Because the python-build-standalone project has _thousands_ of prebuilt runtimes +available, rules_python only includes popular runtimes in its built in +configurations. If you want to use a runtime that isn't already known to +rules_python then {obj}`single_version_platform_override()` can be used to do +so. In short, it allows specifying an arbitrary URL and using custom flags +to control when a runtime is used. + +In the example below, we register a particular python-build-standalone runtime +that is activated for Linux x86 builds when the custom flag +`--//:runtime=my-custom-runtime` is set. + +``` +# File: MODULE.bazel +bazel_dep(name = "bazel_skylib", version = "1.7.1.") +bazel_dep(name = "rules_python", version = "1.5.0") +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.single_version_platform_override( + platform = "my-platform", + python_version = "3.13.3", + sha256 = "01d08b9bc8a96698b9d64c2fc26da4ecc4fa9e708ce0a34fb88f11ab7e552cbd", + os_name = "linux", + arch = "x86_64", + target_settings = [ + "@@//:runtime=my-custom-runtime", + ], + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3+20250409-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz"], +) +# File: //:BUILD.bazel +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") +string_flag( + name = "custom_runtime", + build_setting_default = "", +) +config_setting( + name = "is_custom_runtime_linux-x86-install-only-stripped", + flag_values = { + ":custom_runtime": "linux-x86-install-only-stripped", + }, +) +``` + +Notes: +- While any URL and archive can be used, it's assumed their content looks how + a python-build-standalone archive looks. +- A "version aware" toolchain is registered, which means the Python version flag + must also match (e.g. `--@rules_python//python/config_settings:python_version=3.13.3` + must be set -- see `minor_mapping` and `is_default` for controls and docs + about version matching and selection). +- The `target_compatible_with` attribute can be used to entirely specify the + arg of the same name the toolchain uses. +- The labels in `target_settings` must be absolute; `@@` refers to the main repo. +- The `target_settings` are `config_setting` targets, which means you can + customize how matching occurs. + +:::{seealso} +See {obj}`//python/config_settings` for flags rules_python already defines +that can be used with `target_settings`. Some particular ones of note are: +{flag}`--py_linux_libc` and {flag}`--py_freethreaded`, among others. +::: + +:::{versionadded} VERSION_NEXT_FEATURE +Added support for custom platform names, `target_compatible_with`, and +`target_settings` with `single_version_platform_override`. +::: + ### Using defined toolchains from WORKSPACE It is possible to use toolchains defined in `MODULE.bazel` in `WORKSPACE`. For example diff --git a/internal_dev_setup.bzl b/internal_dev_setup.bzl index 62a11ab1d4..c37c59a5da 100644 --- a/internal_dev_setup.bzl +++ b/internal_dev_setup.bzl @@ -42,7 +42,7 @@ def rules_python_internal_setup(): toolchain_platform_keys = {}, toolchain_python_versions = {}, toolchain_set_python_version_constraints = {}, - base_toolchain_repo_names = [], + host_compatible_repo_names = [], ) runtime_env_repo(name = "rules_python_runtime_env_tc_info") diff --git a/python/BUILD.bazel b/python/BUILD.bazel index 867c43478a..58cff5b99d 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -247,6 +247,7 @@ bzl_library( name = "versions_bzl", srcs = ["versions.bzl"], visibility = ["//:__subpackages__"], + deps = ["//python/private:platform_info_bzl"], ) # NOTE: Remember to add bzl_library targets to //tests:bzl_libraries diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index ce22421300..b319919305 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -241,11 +241,17 @@ bzl_library( ], ) +bzl_library( + name = "platform_info_bzl", + srcs = ["platform_info.bzl"], +) + bzl_library( name = "python_bzl", srcs = ["python.bzl"], deps = [ ":full_version_bzl", + ":platform_info_bzl", ":python_register_toolchains_bzl", ":pythons_hub_bzl", ":repo_utils_bzl", diff --git a/python/private/platform_info.bzl b/python/private/platform_info.bzl new file mode 100644 index 0000000000..3f7dc00165 --- /dev/null +++ b/python/private/platform_info.bzl @@ -0,0 +1,34 @@ +"""Helper to define a struct used to define platform metadata.""" + +def platform_info( + *, + compatible_with = [], + flag_values = {}, + target_settings = [], + os_name, + arch): + """Creates a struct of platform metadata. + + This is just a helper to ensure structs are created the same and + the meaning/values are documented. + + Args: + compatible_with: list[str], where the values are string labels. These + are the target_compatible_with values to use with the toolchain + flag_values: dict[str|Label, Any] of config_setting.flag_values + compatible values. DEPRECATED -- use target_settings instead + target_settings: list[str], where the values are string labels. These + are the target_settings values to use with the toolchain. + os_name: str, the os name; must match the name used in `@platfroms//os` + arch: str, the cpu name; must match the name used in `@platforms//cpu` + + Returns: + A struct with attributes and values matching the args. + """ + return struct( + compatible_with = compatible_with, + flag_values = flag_values, + target_settings = target_settings, + os_name = os_name, + arch = arch, + ) diff --git a/python/private/py_repositories.bzl b/python/private/py_repositories.bzl index b5bd93b7c1..10bc06630b 100644 --- a/python/private/py_repositories.bzl +++ b/python/private/py_repositories.bzl @@ -47,7 +47,7 @@ def py_repositories(): toolchain_platform_keys = {}, toolchain_python_versions = {}, toolchain_set_python_version_constraints = {}, - base_toolchain_repo_names = [], + host_compatible_repo_names = [], ) http_archive( name = "bazel_skylib", diff --git a/python/private/python.bzl b/python/private/python.bzl index a7e257601f..8e23668879 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -18,29 +18,44 @@ load("@bazel_features//:features.bzl", "bazel_features") load("//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", "PLATFORMS", "TOOL_VERSIONS") load(":auth.bzl", "AUTH_ATTRS") load(":full_version.bzl", "full_version") +load(":platform_info.bzl", "platform_info") load(":python_register_toolchains.bzl", "python_register_toolchains") load(":pythons_hub.bzl", "hub_repo") load(":repo_utils.bzl", "repo_utils") -load(":toolchains_repo.bzl", "host_compatible_python_repo", "multi_toolchain_aliases", "sorted_host_platforms") +load( + ":toolchains_repo.bzl", + "host_compatible_python_repo", + "multi_toolchain_aliases", + "sorted_host_platform_names", + "sorted_host_platforms", +) load(":util.bzl", "IS_BAZEL_6_4_OR_HIGHER") load(":version.bzl", "version") -def parse_modules(*, module_ctx, _fail = fail): +def parse_modules(*, module_ctx, logger, _fail = fail): """Parse the modules and return a struct for registrations. Args: module_ctx: {type}`module_ctx` module context. + logger: {type}`repo_utils.logger` A logger to use. _fail: {type}`function` the failure function, mainly for testing. Returns: A struct with the following attributes: - * `toolchains`: The list of toolchains to register. The last - element is special and is treated as the default toolchain. + * `toolchains`: {type}`list[ToolchainConfig]` The list of toolchains to + register. The last element is special and is treated as the default + toolchain. * `config`: Various toolchain config, see `_get_toolchain_config`. * `debug_info`: {type}`None | dict` extra information to be passed to the debug repo. * `platforms`: {type}`dict[str, platform_info]` of the base set of platforms toolchains should be created for, if possible. + + ToolchainConfig struct: + * python_version: str, full python version string + * name: str, the base toolchain name, e.g., "python_3_10", no + platform suffix. + * register_coverage_tool: bool """ if module_ctx.os.environ.get("RULES_PYTHON_BZLMOD_DEBUG", "0") == "1": debug_info = { @@ -64,8 +79,6 @@ def parse_modules(*, module_ctx, _fail = fail): ignore_root_user_error = None - logger = repo_utils.logger(module_ctx, "python") - # if the root module does not register any toolchain then the # ignore_root_user_error takes its default value: True if not module_ctx.modules[0].tags.toolchain: @@ -265,19 +278,37 @@ def parse_modules(*, module_ctx, _fail = fail): ) def _python_impl(module_ctx): - py = parse_modules(module_ctx = module_ctx) + logger = repo_utils.logger(module_ctx, "python") + py = parse_modules(module_ctx = module_ctx, logger = logger) + + # Host compatible runtime repos + # dict[str version, struct] where struct has: + # * full_python_version: str + # * platform: platform_info struct + # * platform_name: str platform name + # * impl_repo_name: str repo name of the runtime's python_repository() repo + all_host_compatible_impls = {} + + # Host compatible repos that still need to be created because, when + # creating the actual runtime repo, there wasn't a host-compatible + # variant defined for it. + # dict[str reponame, struct] where struct has: + # * compatible_version: str, e.g. 3.10 or 3.10.1. The version the host + # repo should be compatible with + # * full_python_version: str, e.g. 3.10.1, the full python version of + # the toolchain that still needs a host repo created. + needed_host_repos = {} # list of structs; see inline struct call within the loop below. toolchain_impls = [] - # list[str] of the base names of toolchain repos - base_toolchain_repo_names = [] + # list[str] of the repo names for host compatible repos + all_host_compatible_repo_names = [] # Create the underlying python_repository repos that contain the # python runtimes and their toolchain implementation definitions. for i, toolchain_info in enumerate(py.toolchains): is_last = (i + 1) == len(py.toolchains) - base_toolchain_repo_names.append(toolchain_info.name) # Ensure that we pass the full version here. full_python_version = full_version( @@ -298,6 +329,8 @@ def _python_impl(module_ctx): _internal_bzlmod_toolchain_call = True, **kwargs ) + if not register_result.impl_repos: + continue host_platforms = {} for repo_name, (platform_name, platform_info) in register_result.impl_repos.items(): @@ -318,27 +351,81 @@ def _python_impl(module_ctx): set_python_version_constraint = is_last, )) if _is_compatible_with_host(module_ctx, platform_info): - host_platforms[platform_name] = platform_info + host_compat_entry = struct( + full_python_version = full_python_version, + platform = platform_info, + platform_name = platform_name, + impl_repo_name = repo_name, + ) + host_platforms[platform_name] = host_compat_entry + all_host_compatible_impls.setdefault(full_python_version, []).append( + host_compat_entry, + ) + parsed_version = version.parse(full_python_version) + all_host_compatible_impls.setdefault( + "{}.{}".format(*parsed_version.release[0:2]), + [], + ).append(host_compat_entry) + + host_repo_name = toolchain_info.name + "_host" + if host_platforms: + all_host_compatible_repo_names.append(host_repo_name) + host_platforms = sorted_host_platforms(host_platforms) + entries = host_platforms.values() + host_compatible_python_repo( + name = host_repo_name, + base_name = host_repo_name, + # NOTE: Order matters. The first found to be compatible is + # (usually) used. + platforms = host_platforms.keys(), + os_names = {str(i): e.platform.os_name for i, e in enumerate(entries)}, + arch_names = {str(i): e.platform.arch for i, e in enumerate(entries)}, + python_versions = {str(i): e.full_python_version for i, e in enumerate(entries)}, + impl_repo_names = {str(i): e.impl_repo_name for i, e in enumerate(entries)}, + ) + else: + needed_host_repos[host_repo_name] = struct( + compatible_version = toolchain_info.python_version, + full_python_version = full_python_version, + ) + + if needed_host_repos: + for key, entries in all_host_compatible_impls.items(): + all_host_compatible_impls[key] = sorted( + entries, + reverse = True, + key = lambda e: version.key(version.parse(e.full_python_version)), + ) - host_platforms = sorted_host_platforms(host_platforms) + for host_repo_name, info in needed_host_repos.items(): + choices = [] + if info.compatible_version not in all_host_compatible_impls: + logger.warn("No host compatible runtime found compatible with version {}".format(info.compatible_version)) + continue + + choices = all_host_compatible_impls[info.compatible_version] + platform_keys = [ + # We have to prepend the offset because the same platform + # name might occur across different versions + "{}_{}".format(i, entry.platform_name) + for i, entry in enumerate(choices) + ] + platform_keys = sorted_host_platform_names(platform_keys) + + all_host_compatible_repo_names.append(host_repo_name) host_compatible_python_repo( - name = toolchain_info.name + "_host", - # NOTE: Order matters. The first found to be compatible is (usually) used. - platforms = host_platforms.keys(), - os_names = { - str(i): platform_info.os_name - for i, platform_info in enumerate(host_platforms.values()) - }, - arch_names = { - str(i): platform_info.arch - for i, platform_info in enumerate(host_platforms.values()) + name = host_repo_name, + base_name = host_repo_name, + platforms = platform_keys, + impl_repo_names = { + str(i): entry.impl_repo_name + for i, entry in enumerate(choices) }, - python_version = full_python_version, + os_names = {str(i): entry.platform.os_name for i, entry in enumerate(choices)}, + arch_names = {str(i): entry.platform.arch for i, entry in enumerate(choices)}, + python_versions = {str(i): entry.full_python_version for i, entry in enumerate(choices)}, ) - # List of the base names ("python_3_10") for the toolchain repos - base_toolchain_repo_names = [] - # list[str] The infix to use for the resulting toolchain() `name` arg. toolchain_names = [] @@ -399,7 +486,7 @@ def _python_impl(module_ctx): toolchain_platform_keys = toolchain_platform_keys, toolchain_python_versions = toolchain_python_versions, toolchain_set_python_version_constraints = toolchain_set_python_version_constraints, - base_toolchain_repo_names = [t.name for t in py.toolchains], + host_compatible_repo_names = sorted(all_host_compatible_repo_names), default_python_version = py.default_python_version, minor_mapping = py.config.minor_mapping, python_versions = list(py.config.default["tool_versions"].keys()), @@ -583,9 +670,56 @@ def _process_single_version_platform_overrides(*, tag, _fail = fail, default): available_versions[tag.python_version].setdefault("sha256", {})[tag.platform] = tag.sha256 if tag.strip_prefix: available_versions[tag.python_version].setdefault("strip_prefix", {})[tag.platform] = tag.strip_prefix + if tag.urls: available_versions[tag.python_version].setdefault("url", {})[tag.platform] = tag.urls + # If platform is customized, or doesn't exist, (re)define one. + if ((tag.target_compatible_with or tag.target_settings or tag.os_name or tag.arch) or + tag.platform not in default["platforms"]): + os_name = tag.os_name + arch = tag.arch + + if not tag.target_compatible_with: + target_compatible_with = [] + if os_name: + target_compatible_with.append("@platforms//os:{}".format( + repo_utils.get_platforms_os_name(os_name), + )) + if arch: + target_compatible_with.append("@platforms//cpu:{}".format( + repo_utils.get_platforms_cpu_name(arch), + )) + else: + target_compatible_with = tag.target_compatible_with + + # For lack of a better option, give a bogus value. It only affects + # if the runtime is considered host-compatible. + if not os_name: + os_name = "UNKNOWN_CUSTOM_OS" + if not arch: + arch = "UNKNOWN_CUSTOM_ARCH" + + # Move the override earlier in the ordering -- the platform key ordering + # becomes the toolchain ordering within the version. This allows the + # override to have a superset of constraints from a regular runtimes + # (e.g. same platform, but with a custom flag required). + override_first = { + tag.platform: platform_info( + compatible_with = target_compatible_with, + target_settings = tag.target_settings, + os_name = os_name, + arch = arch, + ), + } + for key, value in default["platforms"].items(): + # Don't replace our override with the old value + if key in override_first: + continue + override_first[key] = value + + default["platforms"] = override_first + def _process_global_overrides(*, tag, default, _fail = fail): if tag.available_python_versions: available_versions = default["tool_versions"] @@ -664,22 +798,29 @@ def _get_toolchain_config(*, modules, _fail = fail): """ # Items that can be overridden - available_versions = { - version: { - # Use a dicts straight away so that we could do URL overrides for a - # single version. - "sha256": dict(item["sha256"]), - "strip_prefix": { - platform: item["strip_prefix"] - for platform in item["sha256"] - } if type(item["strip_prefix"]) == type("") else item["strip_prefix"], - "url": { - platform: [item["url"]] - for platform in item["sha256"] - } if type(item["url"]) == type("") else item["url"], - } - for version, item in TOOL_VERSIONS.items() - } + available_versions = {} + for py_version, item in TOOL_VERSIONS.items(): + available_versions[py_version] = {} + available_versions[py_version]["sha256"] = dict(item["sha256"]) + platforms = item["sha256"].keys() + + strip_prefix = item["strip_prefix"] + if type(strip_prefix) == type(""): + available_versions[py_version]["strip_prefix"] = { + platform: strip_prefix + for platform in platforms + } + else: + available_versions[py_version]["strip_prefix"] = dict(strip_prefix) + url = item["url"] + if type(url) == type(""): + available_versions[py_version]["url"] = { + platform: url + for platform in platforms + } + else: + available_versions[py_version]["url"] = dict(url) + default = { "base_url": DEFAULT_RELEASE_BASE_URL, "platforms": dict(PLATFORMS), # Copy so it's mutable. @@ -1084,10 +1225,48 @@ configuration, please use {obj}`single_version_override`. ::: """, attrs = { + "arch": attr.string( + doc = """ +The arch (cpu) the runtime is compatible with. + +If not set, then the runtime cannot be used as a `python_X_Y_host` runtime. + +If set, the `os_name`, `target_compatible_with` and `target_settings` attributes +should also be set. + +The values should be one of the values in `@platforms//cpu` + +:::{seealso} +Docs for [Registering custom runtimes] +::: + +:::{{versionadded}} VERSION_NEXT_FEATURE +::: +""", + ), "coverage_tool": attr.label( doc = """\ The coverage tool to be used for a particular Python interpreter. This can override `rules_python` defaults. +""", + ), + "os_name": attr.string( + doc = """ +The host OS the runtime is compatible with. + +If not set, then the runtime cannot be used as a `python_X_Y_host` runtime. + +If set, the `os_name`, `target_compatible_with` and `target_settings` attributes +should also be set. + +The values should be one of the values in `@platforms//os` + +:::{seealso} +Docs for [Registering custom runtimes] +::: + +:::{{versionadded}} VERSION_NEXT_FEATURE +::: """, ), "patch_strip": attr.int( @@ -1101,8 +1280,20 @@ The coverage tool to be used for a particular Python interpreter. This can overr ), "platform": attr.string( mandatory = True, - values = PLATFORMS.keys(), - doc = "The platform to override the values for, must be one of:\n{}.".format("\n".join(sorted(["* `{}`".format(p) for p in PLATFORMS]))), + doc = """ +The platform to override the values for, typically one of:\n +{platforms} + +Other values are allowed, in which case, `target_compatible_with`, +`target_settings`, `os_name`, and `arch` should be specified so the toolchain is +only used when appropriate. + +:::{{versionchanged}} VERSION_NEXT_FEATURE +Arbitrary platform strings allowed. +::: +""".format( + platforms = "\n".join(sorted(["* `{}`".format(p) for p in PLATFORMS])), + ), ), "python_version": attr.string( mandatory = True, @@ -1117,6 +1308,36 @@ The coverage tool to be used for a particular Python interpreter. This can overr doc = "The 'strip_prefix' for the archive, defaults to 'python'.", default = "python", ), + "target_compatible_with": attr.string_list( + doc = """ +The `target_compatible_with` values to use for the toolchain definition. + +If not set, then `os_name` and `arch` will be used to populate it. + +If set, `target_settings`, `os_name`, and `arch` should also be set. + +:::{seealso} +Docs for [Registering custom runtimes] +::: + +:::{{versionadded}} VERSION_NEXT_FEATURE +::: +""", + ), + "target_settings": attr.string_list( + doc = """ +The `target_setings` values to use for the toolchain definition. + +If set, `target_compatible_with`, `os_name`, and `arch` should also be set. + +:::{seealso} +Docs for [Registering custom runtimes] +::: + +:::{{versionadded}} VERSION_NEXT_FEATURE +::: +""", + ), "urls": attr.string_list( mandatory = False, doc = "The URL template to fetch releases for this Python version. If the URL template results in a relative fragment, default base URL is going to be used. Occurrences of `{python_version}`, `{platform}` and `{build}` will be interpolated based on the contents in the override and the known {attr}`platform` values.", diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl index fd86b415cc..cb0731e6eb 100644 --- a/python/private/python_repository.bzl +++ b/python/private/python_repository.bzl @@ -15,7 +15,7 @@ """This file contains repository rules and macros to support toolchain registration. """ -load("//python:versions.bzl", "FREETHREADED", "INSTALL_ONLY", "PLATFORMS") +load("//python:versions.bzl", "FREETHREADED", "INSTALL_ONLY") load(":auth.bzl", "get_auth") load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":text_util.bzl", "render") @@ -327,7 +327,6 @@ function defaults (e.g. `single_version_override` for `MODULE.bazel` files. "platform": attr.string( doc = "The platform name for the Python interpreter tarball.", mandatory = True, - values = PLATFORMS.keys(), ), "python_version": attr.string( doc = "The Python version.", diff --git a/python/private/pythons_hub.bzl b/python/private/pythons_hub.bzl index 53351cacb9..cc25b4ba1d 100644 --- a/python/private/pythons_hub.bzl +++ b/python/private/pythons_hub.bzl @@ -84,13 +84,7 @@ def _hub_build_file_content(rctx): ) _interpreters_bzl_template = """ -INTERPRETER_LABELS = {{ -{interpreter_labels} -}} -""" - -_line_for_hub_template = """\ - "{name}_host": Label("@{name}_host//:python"), +INTERPRETER_LABELS = {labels} """ _versions_bzl_template = """ @@ -110,15 +104,16 @@ def _hub_repo_impl(rctx): # Create a dict that is later used to create # a symlink to a interpreter. - interpreter_labels = "".join([ - _line_for_hub_template.format(name = name) - for name in rctx.attr.base_toolchain_repo_names - ]) - rctx.file( "interpreters.bzl", _interpreters_bzl_template.format( - interpreter_labels = interpreter_labels, + labels = render.dict( + { + name: 'Label("@{}//:python")'.format(name) + for name in rctx.attr.host_compatible_repo_names + }, + value_repr = str, + ), ), executable = False, ) @@ -144,15 +139,14 @@ This rule also writes out the various toolchains for the different Python versio """, implementation = _hub_repo_impl, attrs = { - "base_toolchain_repo_names": attr.string_list( - doc = "The base repo name for toolchains ('python_3_10', no " + - "platform suffix)", - mandatory = True, - ), "default_python_version": attr.string( doc = "Default Python version for the build in `X.Y` or `X.Y.Z` format.", mandatory = True, ), + "host_compatible_repo_names": attr.string_list( + doc = "Names of `host_compatible_python_repo` repos.", + mandatory = True, + ), "minor_mapping": attr.string_dict( doc = "The minor mapping of the `X.Y` to `X.Y.Z` format that is used in config settings.", mandatory = True, diff --git a/python/private/repo_utils.bzl b/python/private/repo_utils.bzl index eee56ec86c..32a5b70e15 100644 --- a/python/private/repo_utils.bzl +++ b/python/private/repo_utils.bzl @@ -31,13 +31,15 @@ def _is_repo_debug_enabled(mrctx): """ return _getenv(mrctx, REPO_DEBUG_ENV_VAR) == "1" -def _logger(mrctx, name = None): +def _logger(mrctx = None, name = None, verbosity_level = None): """Creates a logger instance for printing messages. Args: mrctx: repository_ctx or module_ctx object. If the attribute `_rule_name` is present, it will be included in log messages. name: name for the logger. Optional for repository_ctx usage. + verbosity_level: {type}`int | None` verbosity level. If not set, + taken from `mrctx` Returns: A struct with attributes logging: trace, debug, info, warn, fail. @@ -46,13 +48,14 @@ def _logger(mrctx, name = None): the logger injected into the function work as expected by terminating on the given line. """ - if _is_repo_debug_enabled(mrctx): - verbosity_level = "DEBUG" - else: - verbosity_level = "WARN" + if verbosity_level == None: + if _is_repo_debug_enabled(mrctx): + verbosity_level = "DEBUG" + else: + verbosity_level = "WARN" - env_var_verbosity = _getenv(mrctx, REPO_VERBOSITY_ENV_VAR) - verbosity_level = env_var_verbosity or verbosity_level + env_var_verbosity = _getenv(mrctx, REPO_VERBOSITY_ENV_VAR) + verbosity_level = env_var_verbosity or verbosity_level verbosity = { "DEBUG": 2, @@ -376,7 +379,7 @@ def _get_platforms_os_name(mrctx): """Return the name in @platforms//os for the host os. Args: - mrctx: module_ctx or repository_ctx. + mrctx: {type}`module_ctx | repository_ctx` Returns: `str`. The target name. @@ -405,6 +408,7 @@ def _get_platforms_cpu_name(mrctx): `str`. The target name. """ arch = mrctx.os.arch.lower() + if arch in ["i386", "i486", "i586", "i686", "i786", "x86"]: return "x86_32" if arch in ["amd64", "x86_64", "x64"]: diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index 2476889583..93bbb52108 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -309,11 +309,11 @@ actions.""", environ = [REPO_DEBUG_ENV_VAR], ) -def _host_compatible_python_repo(rctx): +def _host_compatible_python_repo_impl(rctx): rctx.file("BUILD.bazel", _HOST_TOOLCHAIN_BUILD_CONTENT) os_name = repo_utils.get_platforms_os_name(rctx) - host_platform = _get_host_platform( + impl_repo_name = _get_host_impl_repo_name( rctx = rctx, logger = repo_utils.logger(rctx), python_version = rctx.attr.python_version, @@ -321,10 +321,11 @@ def _host_compatible_python_repo(rctx): cpu_name = repo_utils.get_platforms_cpu_name(rctx), platforms = rctx.attr.platforms, ) - repo = "@@{py_repository}_{host_platform}".format( - py_repository = rctx.attr.name[:-len("_host")], - host_platform = host_platform, - ) + + # Bzlmod quirk: A repository rule can't, in its **implemention function**, + # resolve an apparent repo name referring to a repo created by the same + # bzlmod extension. To work around this, we use a canonical label. + repo = "@@{}".format(impl_repo_name) rctx.report_progress("Symlinking interpreter files to the target platform") host_python_repo = rctx.path(Label("{repo}//:BUILD.bazel".format(repo = repo))) @@ -380,26 +381,76 @@ def _host_compatible_python_repo(rctx): # NOTE: The term "toolchain" is a misnomer for this rule. This doesn't define # a repo with toolchains or toolchain implementations. host_compatible_python_repo = repository_rule( - _host_compatible_python_repo, + implementation = _host_compatible_python_repo_impl, doc = """\ Creates a repository with a shorter name meant to be used in the repository_ctx, which needs to have `symlinks` for the interpreter. This is separate from the toolchain_aliases repo because referencing the `python` interpreter target from this repo causes an eager fetch of the toolchain for the host platform. - """, + +This repo has two ways in which is it called: + +1. Workspace. The `platforms` attribute is set, which are keys into the + PLATFORMS global. It assumes `name` + is a + valid repo name which it can use as the backing repo. + +2. Bzlmod. All platform and backing repo information is passed in via the + arch_names, impl_repo_names, os_names, python_versions attributes. +""", attrs = { "arch_names": attr.string_dict( doc = """ -If set, overrides the platform metadata. Keyed by index in `platforms` +Arch (cpu) names. Only set in bzlmod. Keyed by index in `platforms` +""", + ), + "base_name": attr.string( + doc = """ +The name arg, but without bzlmod canonicalization applied. Only set in bzlmod. +""", + ), + "impl_repo_names": attr.string_dict( + doc = """ +The names of backing runtime repos. Only set in bzlmod. The names must be repos +in the same extension as creates the host repo. Keyed by index in `platforms`. """, ), "os_names": attr.string_dict( doc = """ -If set, overrides the platform metadata. Keyed by index in `platforms` +If set, overrides the platform metadata. Only set in bzlmod. Keyed by +index in `platforms` +""", + ), + "platforms": attr.string_list( + mandatory = True, + doc = """ +Platform names (workspace) or platform name-like keys (bzlmod) + +NOTE: The order of this list matters. The first platform that is compatible +with the host will be selected; this can be customized by using the +`RULES_PYTHON_REPO_TOOLCHAIN_*` env vars. + +The values passed vary depending on workspace vs bzlmod. + +Workspace: the values are keys into the `PLATFORMS` dict and are the suffix +to append to `name` to point to the backing repo name. + +Bzlmod: The values are arbitrary keys to create the platform map from the +other attributes (os_name, arch_names, et al). +""", + ), + "python_version": attr.string( + doc = """ +Full python version, Major.Minor.Micro. + +Only set in workspace calls. +""", + ), + "python_versions": attr.string_dict( + doc = """ +If set, the Python version for the corresponding selected platform. Values in +Major.Minor.Micro format. Keyed by index in `platforms`. """, ), - "platforms": attr.string_list(mandatory = True), - "python_version": attr.string(mandatory = True), "_rule_name": attr.string(default = "host_compatible_python_repo"), "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")), }, @@ -435,8 +486,8 @@ multi_toolchain_aliases = repository_rule( }, ) -def sorted_host_platforms(platform_map): - """Sort the keys in the platform map to give correct precedence. +def sorted_host_platform_names(platform_names): + """Sort platform names to give correct precedence. The order of keys in the platform mapping matters for the host toolchain selection. When multiple runtimes are compatible with the host, we take the @@ -453,11 +504,10 @@ def sorted_host_platforms(platform_map): is an innocous looking formatter disable directive. Args: - platform_map: a mapping of platforms and their metadata. + platform_names: a list of platform names Returns: - dict; the same values, but with the keys inserted in the desired - order so that iteration happens in the desired order. + list[str] the same values, but in the desired order. """ def platform_keyer(name): @@ -467,13 +517,26 @@ def sorted_host_platforms(platform_map): 1 if FREETHREADED in name else 0, ) - sorted_platform_keys = sorted(platform_map.keys(), key = platform_keyer) + return sorted(platform_names, key = platform_keyer) + +def sorted_host_platforms(platform_map): + """Sort the keys in the platform map to give correct precedence. + + See sorted_host_platform_names for explanation. + + Args: + platform_map: a mapping of platforms and their metadata. + + Returns: + dict; the same values, but with the keys inserted in the desired + order so that iteration happens in the desired order. + """ return { key: platform_map[key] - for key in sorted_platform_keys + for key in sorted_host_platform_names(platform_map.keys()) } -def _get_host_platform(*, rctx, logger, python_version, os_name, cpu_name, platforms): +def _get_host_impl_repo_name(*, rctx, logger, python_version, os_name, cpu_name, platforms): """Gets the host platform. Args: @@ -488,24 +551,40 @@ def _get_host_platform(*, rctx, logger, python_version, os_name, cpu_name, platf """ if rctx.attr.os_names: platform_map = {} + base_name = rctx.attr.base_name + if not base_name: + fail("The `base_name` attribute must be set under bzlmod") for i, platform_name in enumerate(platforms): key = str(i) + impl_repo_name = rctx.attr.impl_repo_names[key] + impl_repo_name = rctx.name.replace(base_name, impl_repo_name) platform_map[platform_name] = struct( os_name = rctx.attr.os_names[key], arch = rctx.attr.arch_names[key], + python_version = rctx.attr.python_versions[key], + impl_repo_name = impl_repo_name, ) else: - platform_map = sorted_host_platforms(PLATFORMS) + base_name = rctx.name.removesuffix("_host") + platform_map = {} + for platform_name, info in sorted_host_platforms(PLATFORMS).items(): + platform_map[platform_name] = struct( + os_name = info.os_name, + arch = info.arch, + python_version = python_version, + impl_repo_name = "{}_{}".format(base_name, platform_name), + ) candidates = [] for platform in platforms: meta = platform_map[platform] if meta.os_name == os_name and meta.arch == cpu_name: - candidates.append(platform) + candidates.append((platform, meta)) if len(candidates) == 1: - return candidates[0] + platform_name, meta = candidates[0] + return meta.impl_repo_name if candidates: env_var = "RULES_PYTHON_REPO_TOOLCHAIN_{}_{}_{}".format( @@ -525,7 +604,11 @@ def _get_host_platform(*, rctx, logger, python_version, os_name, cpu_name, platf candidates = [preference] if candidates: - return candidates[0] + platform_name, meta = candidates[0] + suffix = meta.impl_repo_name + if not suffix: + suffix = platform_name + return suffix return logger.fail("Could not find a compatible 'host' python for '{os_name}', '{cpu_name}' from the loaded platforms: {platforms}".format( os_name = os_name, diff --git a/python/versions.bzl b/python/versions.bzl index 166cc98851..e712a2e126 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -15,6 +15,8 @@ """The Python versions we use for the toolchains. """ +load("//python/private:platform_info.bzl", "platform_info") + # Values present in the @platforms//os package MACOS_NAME = "osx" LINUX_NAME = "linux" @@ -684,42 +686,12 @@ MINOR_MAPPING = { "3.13": "3.13.2", } -def _platform_info( - *, - compatible_with = [], - flag_values = {}, - target_settings = [], - os_name, - arch): - """Creates a struct of platform metadata. - - Args: - compatible_with: list[str], where the values are string labels. These - are the target_compatible_with values to use with the toolchain - flag_values: dict[str|Label, Any] of config_setting.flag_values - compatible values. DEPRECATED -- use target_settings instead - target_settings: list[str], where the values are string labels. These - are the target_settings values to use with the toolchain. - os_name: str, the os name; must match the name used in `@platfroms//os` - arch: str, the cpu name; must match the name used in `@platforms//cpu` - - Returns: - A struct with attributes and values matching the args. - """ - return struct( - compatible_with = compatible_with, - flag_values = flag_values, - target_settings = target_settings, - os_name = os_name, - arch = arch, - ) - def _generate_platforms(): is_libc_glibc = str(Label("//python/config_settings:_is_py_linux_libc_glibc")) is_libc_musl = str(Label("//python/config_settings:_is_py_linux_libc_musl")) platforms = { - "aarch64-apple-darwin": _platform_info( + "aarch64-apple-darwin": platform_info( compatible_with = [ "@platforms//os:macos", "@platforms//cpu:aarch64", @@ -727,7 +699,7 @@ def _generate_platforms(): os_name = MACOS_NAME, arch = "aarch64", ), - "aarch64-unknown-linux-gnu": _platform_info( + "aarch64-unknown-linux-gnu": platform_info( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:aarch64", @@ -738,7 +710,7 @@ def _generate_platforms(): os_name = LINUX_NAME, arch = "aarch64", ), - "armv7-unknown-linux-gnu": _platform_info( + "armv7-unknown-linux-gnu": platform_info( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:armv7", @@ -749,7 +721,7 @@ def _generate_platforms(): os_name = LINUX_NAME, arch = "arm", ), - "i386-unknown-linux-gnu": _platform_info( + "i386-unknown-linux-gnu": platform_info( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:i386", @@ -760,7 +732,7 @@ def _generate_platforms(): os_name = LINUX_NAME, arch = "x86_32", ), - "ppc64le-unknown-linux-gnu": _platform_info( + "ppc64le-unknown-linux-gnu": platform_info( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:ppc", @@ -771,7 +743,7 @@ def _generate_platforms(): os_name = LINUX_NAME, arch = "ppc", ), - "riscv64-unknown-linux-gnu": _platform_info( + "riscv64-unknown-linux-gnu": platform_info( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:riscv64", @@ -782,7 +754,7 @@ def _generate_platforms(): os_name = LINUX_NAME, arch = "riscv64", ), - "s390x-unknown-linux-gnu": _platform_info( + "s390x-unknown-linux-gnu": platform_info( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:s390x", @@ -793,7 +765,7 @@ def _generate_platforms(): os_name = LINUX_NAME, arch = "s390x", ), - "x86_64-apple-darwin": _platform_info( + "x86_64-apple-darwin": platform_info( compatible_with = [ "@platforms//os:macos", "@platforms//cpu:x86_64", @@ -801,7 +773,7 @@ def _generate_platforms(): os_name = MACOS_NAME, arch = "x86_64", ), - "x86_64-pc-windows-msvc": _platform_info( + "x86_64-pc-windows-msvc": platform_info( compatible_with = [ "@platforms//os:windows", "@platforms//cpu:x86_64", @@ -809,7 +781,7 @@ def _generate_platforms(): os_name = WINDOWS_NAME, arch = "x86_64", ), - "x86_64-unknown-linux-gnu": _platform_info( + "x86_64-unknown-linux-gnu": platform_info( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:x86_64", @@ -820,7 +792,7 @@ def _generate_platforms(): os_name = LINUX_NAME, arch = "x86_64", ), - "x86_64-unknown-linux-musl": _platform_info( + "x86_64-unknown-linux-musl": platform_info( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:x86_64", @@ -836,7 +808,7 @@ def _generate_platforms(): is_freethreaded_yes = str(Label("//python/config_settings:_is_py_freethreaded_yes")) is_freethreaded_no = str(Label("//python/config_settings:_is_py_freethreaded_no")) return { - p + suffix: _platform_info( + p + suffix: platform_info( compatible_with = v.compatible_with, target_settings = [ freethreadedness, diff --git a/tests/bootstrap_impls/bin.py b/tests/bootstrap_impls/bin.py index 1176107384..3d467dcf29 100644 --- a/tests/bootstrap_impls/bin.py +++ b/tests/bootstrap_impls/bin.py @@ -23,3 +23,4 @@ print("sys.flags.safe_path:", sys.flags.safe_path) print("file:", __file__) print("sys.executable:", sys.executable) +print("sys._base_executable:", sys._base_executable) diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index 19be1c478e..116afa76ad 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -17,6 +17,7 @@ load("@pythons_hub//:versions.bzl", "MINOR_MAPPING") load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private:python.bzl", "parse_modules") # buildifier: disable=bzl-visibility +load("//python/private:repo_utils.bzl", "repo_utils") # buildifier: disable=bzl-visibility _tests = [] @@ -131,6 +132,10 @@ def _single_version_platform_override( python_version = python_version, patch_strip = patch_strip, patches = patches, + target_compatible_with = [], + target_settings = [], + os_name = "", + arch = "", ) def _test_default(env): @@ -138,6 +143,7 @@ def _test_default(env): module_ctx = _mock_mctx( _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) # The value there should be consistent in bzlmod with the automatically @@ -168,6 +174,7 @@ def _test_default_some_module(env): module_ctx = _mock_mctx( _mod(name = "rules_python", toolchain = [_toolchain("3.11")], is_root = False), ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) env.expect.that_str(py.default_python_version).equals("3.11") @@ -186,6 +193,7 @@ def _test_default_with_patch_version(env): module_ctx = _mock_mctx( _mod(name = "rules_python", toolchain = [_toolchain("3.11.2")]), ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) env.expect.that_str(py.default_python_version).equals("3.11.2") @@ -207,6 +215,7 @@ def _test_default_non_rules_python(env): # does not make any calls to the extension. _mod(name = "rules_python", toolchain = [_toolchain("3.11")], is_root = False), ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) env.expect.that_str(py.default_python_version).equals("3.11") @@ -228,6 +237,7 @@ def _test_default_non_rules_python_ignore_root_user_error(env): ), _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) env.expect.that_bool(py.config.default["ignore_root_user_error"]).equals(False) @@ -257,6 +267,7 @@ def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): _mod(name = "some_module", toolchain = [_toolchain("3.12", ignore_root_user_error = False)]), _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) env.expect.that_str(py.default_python_version).equals("3.13") @@ -302,6 +313,7 @@ def _test_toolchain_ordering(env): ), _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) got_versions = [ t.python_version @@ -347,6 +359,7 @@ def _test_default_from_defaults(env): is_root = True, ), ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) env.expect.that_str(py.default_python_version).equals("3.11") @@ -374,6 +387,7 @@ def _test_default_from_defaults_env(env): ), environ = {"PYENV_VERSION": "3.12"}, ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) env.expect.that_str(py.default_python_version).equals("3.12") @@ -401,6 +415,7 @@ def _test_default_from_defaults_file(env): ), mocked_files = {"@@//:.python-version": "3.12\n"}, ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) env.expect.that_str(py.default_python_version).equals("3.12") @@ -427,6 +442,7 @@ def _test_first_occurance_of_the_toolchain_wins(env): "RULES_PYTHON_BZLMOD_DEBUG": "1", }, ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) env.expect.that_str(py.default_python_version).equals("3.12") @@ -472,6 +488,7 @@ def _test_auth_overrides(env): ), _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) env.expect.that_dict(py.config.default).contains_at_least({ @@ -541,6 +558,7 @@ def _test_add_new_version(env): ], ), ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) env.expect.that_str(py.default_python_version).equals("3.13") @@ -609,6 +627,7 @@ def _test_register_all_versions(env): ], ), ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) env.expect.that_str(py.default_python_version).equals("3.13") @@ -685,6 +704,7 @@ def _test_add_patches(env): ], ), ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) env.expect.that_str(py.default_python_version).equals("3.13") @@ -731,6 +751,7 @@ def _test_fail_two_overrides(env): ), ), _fail = errors.append, + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) env.expect.that_collection(errors).contains_exactly([ "Only a single 'python.override' can be present", @@ -758,6 +779,7 @@ def _test_single_version_override_errors(env): ), ), _fail = errors.append, + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) env.expect.that_collection(errors).contains_exactly([test.want_error]) @@ -795,6 +817,7 @@ def _test_single_version_platform_override_errors(env): ), ), _fail = lambda *a: errors.append(" ".join(a)), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), ) env.expect.that_collection(errors).contains_exactly([test.want_error]) diff --git a/tests/support/BUILD.bazel b/tests/support/BUILD.bazel index 9fb5cd0760..303dbafbdf 100644 --- a/tests/support/BUILD.bazel +++ b/tests/support/BUILD.bazel @@ -18,6 +18,7 @@ # to force them to resolve in the proper context. # ==================== +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") load(":sh_py_run_test.bzl", "current_build_settings") package( @@ -90,3 +91,15 @@ platform( current_build_settings( name = "current_build_settings", ) + +string_flag( + name = "custom_runtime", + build_setting_default = "", +) + +config_setting( + name = "is_custom_runtime_linux-x86-install-only-stripped", + flag_values = { + ":custom_runtime": "linux-x86-install-only-stripped", + }, +) diff --git a/tests/support/sh_py_run_test.bzl b/tests/support/sh_py_run_test.bzl index 04a2883fde..69141fe8a4 100644 --- a/tests/support/sh_py_run_test.bzl +++ b/tests/support/sh_py_run_test.bzl @@ -42,6 +42,7 @@ def _perform_transition_impl(input_settings, attr, base_impl): # value into the output settings _RECONFIG_ATTR_SETTING_MAP = { "bootstrap_impl": "//python/config_settings:bootstrap_impl", + "custom_runtime": "//tests/support:custom_runtime", "extra_toolchains": "//command_line_option:extra_toolchains", "python_src": "//python/bin:python_src", "venvs_site_packages": "//python/config_settings:venvs_site_packages", @@ -58,6 +59,7 @@ _RECONFIG_INHERITED_OUTPUTS = [v for v in _RECONFIG_OUTPUTS if v in _RECONFIG_IN _RECONFIG_ATTRS = { "bootstrap_impl": attrb.String(), "build_python_zip": attrb.String(default = "auto"), + "custom_runtime": attrb.String(), "extra_toolchains": attrb.StringList( doc = """ Value for the --extra_toolchains flag. diff --git a/tests/toolchains/BUILD.bazel b/tests/toolchains/BUILD.bazel index c55dc92a7d..f346651d46 100644 --- a/tests/toolchains/BUILD.bazel +++ b/tests/toolchains/BUILD.bazel @@ -12,8 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility +load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") load(":defs.bzl", "define_toolchain_tests") define_toolchain_tests( name = "toolchain_tests", ) + +py_reconfig_test( + name = "custom_platform_toolchain_test", + srcs = ["custom_platform_toolchain_test.py"], + custom_runtime = "linux-x86-install-only-stripped", + python_version = "3.13.1", + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ] if BZLMOD_ENABLED else ["@platforms//:incompatible"], +) diff --git a/tests/toolchains/custom_platform_toolchain_test.py b/tests/toolchains/custom_platform_toolchain_test.py new file mode 100644 index 0000000000..d6c083a6a2 --- /dev/null +++ b/tests/toolchains/custom_platform_toolchain_test.py @@ -0,0 +1,15 @@ +import sys +import unittest + + +class VerifyCustomPlatformToolchainTest(unittest.TestCase): + + def test_custom_platform_interpreter_used(self): + # We expect the repo name, and thus path, to have the + # platform name in it. + self.assertIn("linux-x86-install-only-stripped", sys._base_executable) + print(sys._base_executable) + + +if __name__ == "__main__": + unittest.main() From ce80db6a8640cc7552e4b5eada891cd19c4550f2 Mon Sep 17 00:00:00 2001 From: Vihang Mehta Date: Thu, 29 May 2025 18:16:03 -0700 Subject: [PATCH 13/43] feat: Support constraints in pip_compile (#2916) This adds in support to pass in a constraints file to pip-compile. This is extremly useful when you want to uprade an indirect/intermediate dependency to pull in security fixes but don't want to add said dependency to the requirements.in file. --------- Signed-off-by: Vihang Mehta Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> --- CHANGELOG.md | 3 +++ examples/pip_parse/BUILD.bazel | 4 ++++ examples/pip_parse/constraints_certifi.txt | 1 + examples/pip_parse/constraints_urllib3.txt | 1 + examples/pip_parse/requirements_lock.txt | 20 ++++++++++++-------- examples/pip_parse/requirements_windows.txt | 20 ++++++++++++-------- python/private/pypi/pip_compile.bzl | 6 +++++- 7 files changed, 38 insertions(+), 17 deletions(-) create mode 100644 examples/pip_parse/constraints_certifi.txt create mode 100644 examples/pip_parse/constraints_urllib3.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index a113c7411f..355f1fe9ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -111,6 +111,9 @@ END_UNRELEASED_TEMPLATE and activated with custom flags. See the [Registering custom runtimes] docs and {obj}`single_version_platform_override()` API docs for more information. +* (rules) Added support for a using constraints files with `compile_pip_requirements`. + Useful when an intermediate dependency needs to be upgraded to pull in + security patches. {#v0-0-0-removed} ### Removed diff --git a/examples/pip_parse/BUILD.bazel b/examples/pip_parse/BUILD.bazel index 8bdbd94b2c..6ed8d26286 100644 --- a/examples/pip_parse/BUILD.bazel +++ b/examples/pip_parse/BUILD.bazel @@ -57,6 +57,10 @@ py_console_script_binary( compile_pip_requirements( name = "requirements", src = "requirements.in", + constraints = [ + "constraints_certifi.txt", + "constraints_urllib3.txt", + ], requirements_txt = "requirements_lock.txt", requirements_windows = "requirements_windows.txt", ) diff --git a/examples/pip_parse/constraints_certifi.txt b/examples/pip_parse/constraints_certifi.txt new file mode 100644 index 0000000000..7dc4eac259 --- /dev/null +++ b/examples/pip_parse/constraints_certifi.txt @@ -0,0 +1 @@ +certifi>=2025.1.31 \ No newline at end of file diff --git a/examples/pip_parse/constraints_urllib3.txt b/examples/pip_parse/constraints_urllib3.txt new file mode 100644 index 0000000000..3818262552 --- /dev/null +++ b/examples/pip_parse/constraints_urllib3.txt @@ -0,0 +1 @@ +urllib3>1.26.18 diff --git a/examples/pip_parse/requirements_lock.txt b/examples/pip_parse/requirements_lock.txt index aeac61eff9..dc34b45a45 100644 --- a/examples/pip_parse/requirements_lock.txt +++ b/examples/pip_parse/requirements_lock.txt @@ -12,10 +12,12 @@ babel==2.13.1 \ --hash=sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900 \ --hash=sha256:7077a4984b02b6727ac10f1f7294484f737443d7e2e66c5e4380e41a3ae0b4ed # via sphinx -certifi==2024.7.4 \ - --hash=sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b \ - --hash=sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90 - # via requests +certifi==2025.4.26 \ + --hash=sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6 \ + --hash=sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3 + # via + # -c ./constraints_certifi.txt + # requests chardet==4.0.0 \ --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 @@ -218,10 +220,12 @@ sphinxcontrib-serializinghtml==1.1.9 \ # via # -r requirements.in # sphinx -urllib3==1.26.18 \ - --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \ - --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0 - # via requests +urllib3==1.26.20 \ + --hash=sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e \ + --hash=sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32 + # via + # -c ./constraints_urllib3.txt + # requests yamllint==1.28.0 \ --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b diff --git a/examples/pip_parse/requirements_windows.txt b/examples/pip_parse/requirements_windows.txt index 61a6682047..78c1a45690 100644 --- a/examples/pip_parse/requirements_windows.txt +++ b/examples/pip_parse/requirements_windows.txt @@ -12,10 +12,12 @@ babel==2.13.1 \ --hash=sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900 \ --hash=sha256:7077a4984b02b6727ac10f1f7294484f737443d7e2e66c5e4380e41a3ae0b4ed # via sphinx -certifi==2024.7.4 \ - --hash=sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b \ - --hash=sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90 - # via requests +certifi==2025.4.26 \ + --hash=sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6 \ + --hash=sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3 + # via + # -c ./constraints_certifi.txt + # requests chardet==4.0.0 \ --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 @@ -222,10 +224,12 @@ sphinxcontrib-serializinghtml==1.1.9 \ # via # -r requirements.in # sphinx -urllib3==1.26.18 \ - --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \ - --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0 - # via requests +urllib3==1.26.20 \ + --hash=sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e \ + --hash=sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32 + # via + # -c ./constraints_urllib3.txt + # requests yamllint==1.28.0 \ --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl index 9782d3ce21..c9899503d6 100644 --- a/python/private/pypi/pip_compile.bzl +++ b/python/private/pypi/pip_compile.bzl @@ -38,6 +38,7 @@ def pip_compile( requirements_windows = None, visibility = ["//visibility:private"], tags = None, + constraints = [], **kwargs): """Generates targets for managing pip dependencies with pip-compile. @@ -77,6 +78,7 @@ def pip_compile( requirements_windows: File of windows specific resolve output to check validate if requirement.in has changes. tags: tagging attribute common to all build rules, passed to both the _test and .update rules. visibility: passed to both the _test and .update rules. + constraints: a list of files containing constraints to pass to pip-compile with `--constraint`. **kwargs: other bazel attributes passed to the "_test" rule. """ if len([x for x in [srcs, src, requirements_in] if x != None]) > 1: @@ -100,7 +102,7 @@ def pip_compile( visibility = visibility, ) - data = [name, requirements_txt] + srcs + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] + data = [name, requirements_txt] + srcs + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] + constraints # Use the Label constructor so this is expanded in the context of the file # where it appears, which is to say, in @rules_python @@ -122,6 +124,8 @@ def pip_compile( args.append("--requirements-darwin={}".format(loc.format(requirements_darwin))) if requirements_windows: args.append("--requirements-windows={}".format(loc.format(requirements_windows))) + for constraint in constraints: + args.append("--constraint=$(location {})".format(constraint)) args.extend(extra_args) deps = [ From af9e959538f34878ca0ccccd97d51dc7b3ffdadd Mon Sep 17 00:00:00 2001 From: rbeasley-avgo Date: Fri, 30 May 2025 05:25:03 -0400 Subject: [PATCH 14/43] fix(pypi): allow pip_compile to work with read-only sources (#2712) The validating `py_test` generated by `compile_pip_requirements` chokes when the source `requirements.txt` is stored read-only, such as when managed by the Perforce Helix Core SCM. Though `dependency_resolver` makes a temporary copy of this file, it does so w/ `shutil.copy` which preserves the original read-only file mode. To address this, this commit replaces `shutil.copy` with a `shutil.copyfileobj` such that the temporary file is created w/ permissions according to the user's umask. Resolves (#2608). --------- Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> --- CHANGELOG.md | 2 ++ .../pypi/dependency_resolver/dependency_resolver.py | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 355f1fe9ef..0a2dc413ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,8 @@ END_UNRELEASED_TEMPLATE also retrieved from the URL as opposed to only the `--hash` parameter. Fixes [#2363](https://github.com/bazel-contrib/rules_python/issues/2363). * (pypi) `whl_library` now infers file names from its `urls` attribute correctly. +* (pypi) When running under `bazel test`, be sure that temporary `requirements` file + remains writable. * (py_test, py_binary) Allow external files to be used for main {#v0-0-0-added} diff --git a/python/private/pypi/dependency_resolver/dependency_resolver.py b/python/private/pypi/dependency_resolver/dependency_resolver.py index ada0763558..a42821c458 100644 --- a/python/private/pypi/dependency_resolver/dependency_resolver.py +++ b/python/private/pypi/dependency_resolver/dependency_resolver.py @@ -151,9 +151,16 @@ def main( requirements_out = os.path.join( os.environ["TEST_TMPDIR"], os.path.basename(requirements_file) + ".out" ) + # Why this uses shutil.copyfileobj: + # # Those two files won't necessarily be on the same filesystem, so we can't use os.replace # or shutil.copyfile, as they will fail with OSError: [Errno 18] Invalid cross-device link. - shutil.copy(resolved_requirements_file, requirements_out) + # + # Further, shutil.copy preserves the source file's mode, and so if + # our source file is read-only (the default under Perforce Helix), + # this scratch file will also be read-only, defeating its purpose. + with open(resolved_requirements_file, "rb") as fsrc, open(requirements_out, "wb") as fdst: + shutil.copyfileobj(fsrc, fdst) update_command = ( os.getenv("CUSTOM_COMPILE_COMMAND") or f"bazel run {target_label_prefix}.update" From 02198f622ee1b496111bef6b880ea35e0d24b600 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 31 May 2025 15:52:54 +0900 Subject: [PATCH 15/43] feat(uv): handle credential helpers and .netrc (#2872) This allows one to download the uv binaries from private mirrors. The plumbing of the auth attrs allows us to correctly use the `~/.netrc` or the credential helper for downloading from mirrors that require authentication. Testing notes: * When I tested this, it seems that the dist manifest json may not work with private mirrors, but I think it is fine for users in such cases to define the `uv` srcs using the `urls` attribute. Work towards #1975. --- CHANGELOG.md | 2 ++ python/uv/private/BUILD.bazel | 2 ++ python/uv/private/uv.bzl | 35 +++++++++++++++++++++++------ python/uv/private/uv_repository.bzl | 5 ++++- tests/uv/uv/uv_tests.bzl | 23 ++++++++++++++++++- 5 files changed, 58 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a2dc413ae..f82df5aad0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,6 +109,8 @@ END_UNRELEASED_TEMPLATE Set the `RULES_PYTHON_ENABLE_PIPSTAR=1` environment variable to enable it. * (utils) Add a way to run a REPL for any `rules_python` target that returns a `PyInfo` provider. +* (uv) Handle `.netrc` and `auth_patterns` auth when downloading `uv`. Work towards + [#1975](https://github.com/bazel-contrib/rules_python/issues/1975). * (toolchains) Arbitrary python-build-standalone runtimes can be registered and activated with custom flags. See the [Registering custom runtimes] docs and {obj}`single_version_platform_override()` API docs for more diff --git a/python/uv/private/BUILD.bazel b/python/uv/private/BUILD.bazel index 587ad9a0f9..a07d8591ad 100644 --- a/python/uv/private/BUILD.bazel +++ b/python/uv/private/BUILD.bazel @@ -62,6 +62,7 @@ bzl_library( ":toolchain_types_bzl", ":uv_repository_bzl", ":uv_toolchains_repo_bzl", + "//python/private:auth_bzl", ], ) @@ -69,6 +70,7 @@ bzl_library( name = "uv_repository_bzl", srcs = ["uv_repository.bzl"], visibility = ["//python/uv:__subpackages__"], + deps = ["//python/private:auth_bzl"], ) bzl_library( diff --git a/python/uv/private/uv.bzl b/python/uv/private/uv.bzl index 09fb78322f..2cc2df1b21 100644 --- a/python/uv/private/uv.bzl +++ b/python/uv/private/uv.bzl @@ -18,6 +18,7 @@ EXPERIMENTAL: This is experimental and may be removed without notice A module extension for working with uv. """ +load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") load(":toolchain_types.bzl", "UV_TOOLCHAIN_TYPE") load(":uv_repository.bzl", "uv_repository") load(":uv_toolchains_repo.bzl", "uv_toolchains_repo") @@ -77,7 +78,7 @@ The version of uv to configure the sources for. If this is not specified it will last version used in the module or the default version set by `rules_python`. """, ), -} +} | AUTH_ATTRS default = tag_class( doc = """\ @@ -133,7 +134,7 @@ for a particular version. }, ) -def _configure(config, *, platform, compatible_with, target_settings, urls = [], sha256 = "", override = False, **values): +def _configure(config, *, platform, compatible_with, target_settings, auth_patterns, urls = [], sha256 = "", override = False, **values): """Set the value in the config if the value is provided""" for key, value in values.items(): if not value: @@ -144,6 +145,7 @@ def _configure(config, *, platform, compatible_with, target_settings, urls = [], config[key] = value + config.setdefault("auth_patterns", {}).update(auth_patterns) config.setdefault("platforms", {}) if not platform: if compatible_with or target_settings or urls: @@ -173,7 +175,8 @@ def process_modules( hub_name = "uv", uv_repository = uv_repository, toolchain_type = str(UV_TOOLCHAIN_TYPE), - hub_repo = uv_toolchains_repo): + hub_repo = uv_toolchains_repo, + get_auth = get_auth): """Parse the modules to get the config for 'uv' toolchains. Args: @@ -182,6 +185,7 @@ def process_modules( uv_repository: the rule to create a uv_repository override. toolchain_type: the toolchain type to use here. hub_repo: the hub repo factory function to use. + get_auth: the auth function to use. Returns: the result of the hub_repo. Mainly used for tests. @@ -216,6 +220,8 @@ def process_modules( compatible_with = tag.compatible_with, target_settings = tag.target_settings, override = mod.is_root, + netrc = tag.netrc, + auth_patterns = tag.auth_patterns, ) for key in [ @@ -271,6 +277,8 @@ def process_modules( sha256 = tag.sha256, urls = tag.urls, override = mod.is_root, + netrc = tag.netrc, + auth_patterns = tag.auth_patterns, ) if not versions: @@ -301,6 +309,11 @@ def process_modules( for platform, src in config.get("urls", {}).items() if src.urls } + auth = { + "auth_patterns": config.get("auth_patterns"), + "netrc": config.get("netrc"), + } + auth = {k: v for k, v in auth.items() if v} # Or fallback to fetching them from GH manifest file # Example file: https://github.com/astral-sh/uv/releases/download/0.6.3/dist-manifest.json @@ -313,6 +326,8 @@ def process_modules( ), manifest_filename = config["manifest_filename"], platforms = sorted(platforms), + get_auth = get_auth, + **auth ) for platform_name, platform in platforms.items(): @@ -327,6 +342,7 @@ def process_modules( platform = platform_name, urls = urls[platform_name].urls, sha256 = urls[platform_name].sha256, + **auth ) toolchain_names.append(toolchain_name) @@ -363,7 +379,7 @@ def _overlap(first_collection, second_collection): return False -def _get_tool_urls_from_dist_manifest(module_ctx, *, base_url, manifest_filename, platforms): +def _get_tool_urls_from_dist_manifest(module_ctx, *, base_url, manifest_filename, platforms, get_auth = get_auth, **auth_attrs): """Download the results about remote tool sources. This relies on the tools using the cargo packaging to infer the actual @@ -431,10 +447,13 @@ def _get_tool_urls_from_dist_manifest(module_ctx, *, base_url, manifest_filename "aarch64-apple-darwin" ] """ + auth_attr = struct(**auth_attrs) dist_manifest = module_ctx.path(manifest_filename) + urls = [base_url + "/" + manifest_filename] result = module_ctx.download( - base_url + "/" + manifest_filename, + url = urls, output = dist_manifest, + auth = get_auth(module_ctx, urls, ctx_attr = auth_attr), ) if not result.success: fail(result) @@ -454,11 +473,13 @@ def _get_tool_urls_from_dist_manifest(module_ctx, *, base_url, manifest_filename checksum_fname = checksum["name"] checksum_path = module_ctx.path(checksum_fname) + urls = ["{}/{}".format(base_url, checksum_fname)] downloads[checksum_path] = struct( download = module_ctx.download( - "{}/{}".format(base_url, checksum_fname), + url = urls, output = checksum_path, block = False, + auth = get_auth(module_ctx, urls, ctx_attr = auth_attr), ), archive_fname = fname, platforms = checksum["target_triples"], @@ -473,7 +494,7 @@ def _get_tool_urls_from_dist_manifest(module_ctx, *, base_url, manifest_filename sha256, _, checksummed_fname = module_ctx.read(checksum_path).partition(" ") checksummed_fname = checksummed_fname.strip(" *\n") - if archive_fname != checksummed_fname: + if checksummed_fname and archive_fname != checksummed_fname: fail("The checksum is for a different file, expected '{}' but got '{}'".format( archive_fname, checksummed_fname, diff --git a/python/uv/private/uv_repository.bzl b/python/uv/private/uv_repository.bzl index ba7d2a766c..fed4f576d3 100644 --- a/python/uv/private/uv_repository.bzl +++ b/python/uv/private/uv_repository.bzl @@ -18,6 +18,8 @@ EXPERIMENTAL: This is experimental and may be removed without notice Create repositories for uv toolchain dependencies """ +load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") + UV_BUILD_TMPL = """\ # Generated by repositories.bzl load("@rules_python//python/uv:uv_toolchain.bzl", "uv_toolchain") @@ -43,6 +45,7 @@ def _uv_repo_impl(repository_ctx): url = repository_ctx.attr.urls, sha256 = repository_ctx.attr.sha256, stripPrefix = strip_prefix, + auth = get_auth(repository_ctx, repository_ctx.attr.urls), ) binary = "uv.exe" if is_windows else "uv" @@ -70,5 +73,5 @@ uv_repository = repository_rule( "sha256": attr.string(mandatory = False), "urls": attr.string_list(mandatory = True), "version": attr.string(mandatory = True), - }, + } | AUTH_ATTRS, ) diff --git a/tests/uv/uv/uv_tests.bzl b/tests/uv/uv/uv_tests.bzl index bf0deefa88..b464dab55c 100644 --- a/tests/uv/uv/uv_tests.bzl +++ b/tests/uv/uv/uv_tests.bzl @@ -100,7 +100,7 @@ def _mod(*, name = None, default = [], configure = [], is_root = True): ) def _process_modules(env, **kwargs): - result = process_modules(hub_repo = struct, **kwargs) + result = process_modules(hub_repo = struct, get_auth = lambda *_, **__: None, **kwargs) return env.expect.that_struct( struct( @@ -124,6 +124,8 @@ def _default( platform = None, target_settings = None, version = None, + netrc = None, + auth_patterns = None, **kwargs): return struct( base_url = base_url, @@ -132,6 +134,8 @@ def _default( platform = platform, target_settings = [] + (target_settings or []), # ensure that the type is correct version = version, + netrc = netrc, + auth_patterns = {} | (auth_patterns or {}), # ensure that the type is correct **kwargs ) @@ -377,6 +381,11 @@ def _test_complex_configuring(env): platform = "linux", compatible_with = ["@platforms//os:linux"], ), + _configure( + version = "1.0.4", + netrc = "~/.my_netrc", + auth_patterns = {"foo": "bar"}, + ), # use auth ], ), ), @@ -388,18 +397,21 @@ def _test_complex_configuring(env): "1_0_1_osx", "1_0_2_osx", "1_0_3_linux", + "1_0_4_osx", ]) uv.implementations().contains_exactly({ "1_0_0_osx": "@uv_1_0_0_osx//:uv_toolchain", "1_0_1_osx": "@uv_1_0_1_osx//:uv_toolchain", "1_0_2_osx": "@uv_1_0_2_osx//:uv_toolchain", "1_0_3_linux": "@uv_1_0_3_linux//:uv_toolchain", + "1_0_4_osx": "@uv_1_0_4_osx//:uv_toolchain", }) uv.compatible_with().contains_exactly({ "1_0_0_osx": ["@platforms//os:os"], "1_0_1_osx": ["@platforms//os:os"], "1_0_2_osx": ["@platforms//os:different"], "1_0_3_linux": ["@platforms//os:linux"], + "1_0_4_osx": ["@platforms//os:os"], }) uv.target_settings().contains_exactly({}) env.expect.that_collection(calls).contains_exactly([ @@ -431,6 +443,15 @@ def _test_complex_configuring(env): "urls": ["https://example.org/1.0.3/linux"], "version": "1.0.3", }, + { + "auth_patterns": {"foo": "bar"}, + "name": "uv_1_0_4_osx", + "netrc": "~/.my_netrc", + "platform": "osx", + "sha256": "deadb00f", + "urls": ["https://example.org/1.0.4/osx"], + "version": "1.0.4", + }, ]) _tests.append(_test_complex_configuring) From 948fcec44edbe12f4edf94db098c761570a72763 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 3 Jun 2025 00:44:57 +0900 Subject: [PATCH 16/43] fix(pypi): correctly aggregate the requirements files (#2932) This implements the actual fix where we are aggregating the whls and sdists correctly from multiple different requirements lines. Fixes #2648. Closes #2658. --- CHANGELOG.md | 2 + python/private/pypi/parse_requirements.bzl | 18 +++- .../parse_requirements_tests.bzl | 84 ++++++++++++++++++- 3 files changed, 97 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f82df5aad0..c9668c507f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,6 +96,8 @@ END_UNRELEASED_TEMPLATE * (pypi) When running under `bazel test`, be sure that temporary `requirements` file remains writable. * (py_test, py_binary) Allow external files to be used for main +* (pypi) Correctly aggregate the sources when the hashes specified in the lockfile differ + by platform even though the same version is used. Fixes [#2648](https://github.com/bazel-contrib/rules_python/issues/2648). {#v0-0-0-added} ### Added diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index bd2981efc0..e4a8b90acb 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -223,7 +223,7 @@ def _package_srcs( env_marker_target_platforms, extract_url_srcs): """A function to return sources for a particular package.""" - srcs = [] + srcs = {} for r in sorted(reqs.values(), key = lambda r: r.requirement_line): whls, sdist = _add_dists( requirement = r, @@ -249,21 +249,31 @@ def _package_srcs( )] req_line = r.srcs.requirement_line + extra_pip_args = tuple(r.extra_pip_args) for dist in all_dists: - srcs.append( + key = ( + dist.filename, + req_line, + extra_pip_args, + ) + entry = srcs.setdefault( + key, struct( distribution = name, extra_pip_args = r.extra_pip_args, requirement_line = req_line, - target_platforms = target_platforms, + target_platforms = [], filename = dist.filename, sha256 = dist.sha256, url = dist.url, yanked = dist.yanked, ), ) + for p in target_platforms: + if p not in entry.target_platforms: + entry.target_platforms.append(p) - return srcs + return srcs.values() def select_requirement(requirements, *, platform): """A simple function to get a requirement for a particular platform. diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 926a7e0c50..82fdd0a051 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -38,7 +38,7 @@ foo[extra]==0.0.1 \ foo @ git+https://github.com/org/foo.git@deadbeef """, "requirements_linux": """\ -foo==0.0.3 --hash=sha256:deadbaaf +foo==0.0.3 --hash=sha256:deadbaaf --hash=sha256:5d15t """, # download_only = True "requirements_linux_download_only": """\ @@ -67,7 +67,7 @@ foo==0.0.4 @ https://example.org/foo-0.0.4.whl foo==0.0.5 @ https://example.org/foo-0.0.5.whl --hash=sha256:deadbeef """, "requirements_osx": """\ -foo==0.0.3 --hash=sha256:deadbaaf +foo==0.0.3 --hash=sha256:deadbaaf --hash=sha256:deadb11f --hash=sha256:5d15t """, "requirements_osx_download_only": """\ --platform=macosx_10_9_arm64 @@ -251,7 +251,7 @@ def _test_multi_os(env): struct( distribution = "foo", extra_pip_args = [], - requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf", + requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf --hash=sha256:5d15t", target_platforms = ["linux_x86_64"], url = "", filename = "", @@ -515,6 +515,84 @@ def _test_git_sources(env): _tests.append(_test_git_sources) +def _test_overlapping_shas_with_index_results(env): + got = parse_requirements( + ctx = _mock_ctx(), + requirements_by_platform = { + "requirements_linux": ["cp39_linux_x86_64"], + "requirements_osx": ["cp39_osx_x86_64"], + }, + get_index_urls = lambda _, __: { + "foo": struct( + sdists = { + "5d15t": struct( + url = "sdist", + sha256 = "5d15t", + filename = "foo-0.0.1.tar.gz", + yanked = False, + ), + }, + whls = { + "deadb11f": struct( + url = "super2", + sha256 = "deadb11f", + filename = "foo-0.0.1-py3-none-macosx_14_0_x86_64.whl", + yanked = False, + ), + "deadbaaf": struct( + url = "super2", + sha256 = "deadbaaf", + filename = "foo-0.0.1-py3-none-any.whl", + yanked = False, + ), + }, + ), + }, + ) + + env.expect.that_collection(got).contains_exactly([ + struct( + name = "foo", + is_exposed = True, + # TODO @aignas 2025-05-25: how do we rename this? + is_multiple_versions = True, + srcs = [ + struct( + distribution = "foo", + extra_pip_args = [], + filename = "foo-0.0.1-py3-none-any.whl", + requirement_line = "foo==0.0.3", + sha256 = "deadbaaf", + target_platforms = ["cp39_linux_x86_64", "cp39_osx_x86_64"], + url = "super2", + yanked = False, + ), + struct( + distribution = "foo", + extra_pip_args = [], + filename = "foo-0.0.1.tar.gz", + requirement_line = "foo==0.0.3", + sha256 = "5d15t", + target_platforms = ["cp39_linux_x86_64", "cp39_osx_x86_64"], + url = "sdist", + yanked = False, + ), + struct( + distribution = "foo", + extra_pip_args = [], + filename = "foo-0.0.1-py3-none-macosx_14_0_x86_64.whl", + requirement_line = "foo==0.0.3", + sha256 = "deadb11f", + target_platforms = ["cp39_osx_x86_64"], + url = "super2", + yanked = False, + ), + ], + ), + ]) + +_tests.append(_test_overlapping_shas_with_index_results) + def parse_requirements_test_suite(name): """Create the test suite. From 9429ae6446935059e79047654d3fe53d60aadc31 Mon Sep 17 00:00:00 2001 From: Mike Toldov Date: Tue, 3 Jun 2025 09:13:19 +0200 Subject: [PATCH 17/43] fix(pypi): inherit proxy env variables in compile_pip_requirements test (#2941) Bazel does not pass environment variables implicitly (even running test outside of sandbox). This forces compile_pip_requirements test to fail with timeout when attempting to run it behind the proxy. Also changes test_command in dependency_resolver string helper to use dot instead of underscore following deprecation notice --- CHANGELOG.md | 1 + .../pypi/dependency_resolver/dependency_resolver.py | 2 +- python/private/pypi/pip_compile.bzl | 8 +++++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9668c507f..e48e3d4f3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,7 @@ END_UNRELEASED_TEMPLATE * (py_test, py_binary) Allow external files to be used for main * (pypi) Correctly aggregate the sources when the hashes specified in the lockfile differ by platform even though the same version is used. Fixes [#2648](https://github.com/bazel-contrib/rules_python/issues/2648). +* (pypi) `compile_pip_requirements` test rule works behind the proxy {#v0-0-0-added} ### Added diff --git a/python/private/pypi/dependency_resolver/dependency_resolver.py b/python/private/pypi/dependency_resolver/dependency_resolver.py index a42821c458..f3a339f929 100644 --- a/python/private/pypi/dependency_resolver/dependency_resolver.py +++ b/python/private/pypi/dependency_resolver/dependency_resolver.py @@ -165,7 +165,7 @@ def main( update_command = ( os.getenv("CUSTOM_COMPILE_COMMAND") or f"bazel run {target_label_prefix}.update" ) - test_command = f"bazel test {target_label_prefix}_test" + test_command = f"bazel test {target_label_prefix}.test" os.environ["CUSTOM_COMPILE_COMMAND"] = update_command os.environ["PIP_CONFIG_FILE"] = os.getenv("PIP_CONFIG_FILE") or os.devnull diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl index c9899503d6..78b681b4ad 100644 --- a/python/private/pypi/pip_compile.bzl +++ b/python/private/pypi/pip_compile.bzl @@ -45,7 +45,6 @@ def pip_compile( By default this rules generates a filegroup named "[name]" which can be included in the data of some other compile_pip_requirements rule that references these requirements (e.g. with `-r ../other/requirements.txt`). - It also generates two targets for running pip-compile: - validate with `bazel test [name].test` @@ -160,6 +159,12 @@ def pip_compile( } env = kwargs.pop("env", {}) + env_inherit = kwargs.pop("env_inherit", []) + proxy_variables = ["https_proxy", "http_proxy", "no_proxy", "HTTPS_PROXY", "HTTP_PROXY", "NO_PROXY"] + + for var in proxy_variables: + if var not in env_inherit: + env_inherit.append(var) py_binary( name = name + ".update", @@ -182,6 +187,7 @@ def pip_compile( "@@platforms//os:windows": {"USERPROFILE": "Z:\\FakeSetuptoolsHomeDirectoryHack"}, "//conditions:default": {}, }) | env, + env_inherit = env_inherit, # kwargs could contain test-specific attributes like size **dict(attrs, **kwargs) ) From 049866442fee7bb54fcb1a09e920953a0666e4b3 Mon Sep 17 00:00:00 2001 From: Kayce Basques Date: Thu, 5 Jun 2025 14:29:07 -0700 Subject: [PATCH 18/43] feat: add persistent worker for sphinxdocs (#2938) This implements a simple, serialized persistent worker for Sphinxdocs with several optimizations. It is enabled by default. * The worker computes what inputs have changed, allowing Sphinx to only rebuild what is necessary. * Doctrees are written to a separate directory so they are retained between builds. * The worker tells Sphinx to write output to an internal directory, then copies it to the expected Bazel output directory afterwards. This allows Sphinx to only write output files that need to be updated. This works by having the worker compute what files have changed and having a Sphinx extension use the `get-env-outdated` event to tell Sphinx which files have changed. The extension is based on https://pwrev.dev/294057, but re-implemented to be in-memory as part of the worker instead of a separate extension projects must configure. For rules_python's doc building, this reduces incremental building from about 8 seconds to about 0.8 seconds. From what I can tell, about half the time is spent generating doctrees, and the other half generating the output files. Worker mode is enabled by default and can be disabled on the target or by adjusting the Bazel flags controlling execution strategy. Docs added to explain how. Because `--doctree-dir` is now always specified and outside the output dir, non-worker invocations can benefit, too, if run without sandboxing. Docs added to explain how to do this. Along the way: * Remove `--write-all` and `--fresh-env` from run args. This lets direct invocations benefit from the normal caching Sphinx does. * Change the args formatting to `--foo=bar` so they are a single element; just a bit nicer to see when debugging. Work towards https://github.com/bazel-contrib/rules_python/issues/2878, https://github.com/bazel-contrib/rules_python/issues/2879 --------- Co-authored-by: Kayce Basques Co-authored-by: Richard Levasseur --- sphinxdocs/docs/index.md | 23 +++ sphinxdocs/private/sphinx.bzl | 55 +++++-- sphinxdocs/private/sphinx_build.py | 231 ++++++++++++++++++++++++++- sphinxdocs/tests/sphinx_docs/doc1.md | 3 + sphinxdocs/tests/sphinx_docs/doc2.md | 3 + 5 files changed, 302 insertions(+), 13 deletions(-) create mode 100644 sphinxdocs/tests/sphinx_docs/doc1.md create mode 100644 sphinxdocs/tests/sphinx_docs/doc2.md diff --git a/sphinxdocs/docs/index.md b/sphinxdocs/docs/index.md index bd6448ced9..2ea1146e1b 100644 --- a/sphinxdocs/docs/index.md +++ b/sphinxdocs/docs/index.md @@ -11,6 +11,29 @@ documentation. It comes with: While it is primarily oriented towards docgen for Starlark code, the core of it is agnostic as to what is being documented. +### Optimization + +Normally, Sphinx keeps various cache files to improve incremental building. +Unfortunately, programs performing their own caching don't interact well +with Bazel's model of precisely declaring and strictly enforcing what are +inputs, what are outputs, and what files are available when running a program. +The net effect is programs don't have a prior invocation's cache files +available. + +There are two mechanisms available to make some cache available to Sphinx under +Bazel: + +* Disable sandboxing, which allows some files from prior invocations to be + visible to subsequent invocations. This can be done multiple ways: + * Set `tags = ["no-sandbox"]` on the `sphinx_docs` target + * `--modify_execution_info=SphinxBuildDocs=+no-sandbox` (Bazel flag) + * `--strategy=SphinxBuildDocs=local` (Bazel flag) +* Use persistent workers (enabled by default) by setting + `allow_persistent_workers=True` on the `sphinx_docs` target. Note that other + Bazel flags can disable using workers even if an action supports it. Setting + `--strategy=SphinxBuildDocs=dynamic,worker,local,sandbox` should tell Bazel + to use workers if possible, otherwise fallback to non-worker invocations. + ```{toctree} :hidden: diff --git a/sphinxdocs/private/sphinx.bzl b/sphinxdocs/private/sphinx.bzl index 8d19d87052..ee6b994e2e 100644 --- a/sphinxdocs/private/sphinx.bzl +++ b/sphinxdocs/private/sphinx.bzl @@ -103,6 +103,7 @@ def sphinx_docs( strip_prefix = "", extra_opts = [], tools = [], + allow_persistent_workers = True, **kwargs): """Generate docs using Sphinx. @@ -142,6 +143,9 @@ def sphinx_docs( tools: {type}`list[label]` Additional tools that are used by Sphinx and its plugins. This just makes the tools available during Sphinx execution. To locate them, use {obj}`extra_opts` and `$(location)`. + allow_persistent_workers: {type}`bool` (experimental) If true, allow + using persistent workers for running Sphinx, if Bazel decides to do so. + This can improve incremental building of docs. **kwargs: {type}`dict` Common attributes to pass onto rules. """ add_tag(kwargs, "@rules_python//sphinxdocs:sphinx_docs") @@ -165,6 +169,7 @@ def sphinx_docs( source_tree = internal_name + "/_sources", extra_opts = extra_opts, tools = tools, + allow_persistent_workers = allow_persistent_workers, **kwargs ) @@ -209,6 +214,7 @@ def _sphinx_docs_impl(ctx): source_path = source_dir_path, output_prefix = paths.join(ctx.label.name, "_build"), inputs = inputs, + allow_persistent_workers = ctx.attr.allow_persistent_workers, ) outputs[format] = output_dir per_format_args[format] = args_env @@ -229,6 +235,10 @@ def _sphinx_docs_impl(ctx): _sphinx_docs = rule( implementation = _sphinx_docs_impl, attrs = { + "allow_persistent_workers": attr.bool( + doc = "(experimental) Whether to invoke Sphinx as a persistent worker.", + default = False, + ), "extra_opts": attr.string_list( doc = "Additional options to pass onto Sphinx. These are added after " + "other options, but before the source/output args.", @@ -254,16 +264,27 @@ _sphinx_docs = rule( }, ) -def _run_sphinx(ctx, format, source_path, inputs, output_prefix): +def _run_sphinx(ctx, format, source_path, inputs, output_prefix, allow_persistent_workers): output_dir = ctx.actions.declare_directory(paths.join(output_prefix, format)) run_args = [] # Copy of the args to forward along to debug runner args = ctx.actions.args() # Args passed to the action + # An args file is required for persistent workers, but we don't know if + # the action will use worker mode or not (settings we can't see may + # force non-worker mode). For consistency, always use a params file. + args.use_param_file("@%s", use_always = True) + args.set_param_file_format("multiline") + + # NOTE: sphinx_build.py relies on the first two args being the srcdir and + # outputdir, in that order. + args.add(source_path) + args.add(output_dir.path) + args.add("--show-traceback") # Full tracebacks on error run_args.append("--show-traceback") - args.add("--builder", format) - run_args.extend(("--builder", format)) + args.add(format, format = "--builder=%s") + run_args.append("--builder={}".format(format)) if ctx.attr._quiet_flag[BuildSettingInfo].value: # Not added to run_args because run_args is for debugging @@ -271,11 +292,17 @@ def _run_sphinx(ctx, format, source_path, inputs, output_prefix): # Build in parallel, if possible # Don't add to run_args: parallel building breaks interactive debugging - args.add("--jobs", "auto") - args.add("--fresh-env") # Don't try to use cache files. Bazel can't make use of them. - run_args.append("--fresh-env") - args.add("--write-all") # Write all files; don't try to detect "changed" files - run_args.append("--write-all") + args.add("--jobs=auto") + + # Put the doctree dir outside of the output directory. + # This allows it to be reused between invocations when possible; Bazel + # clears the output directory every action invocation. + # * For workers, they can fully re-use it. + # * For non-workers, it can be reused when sandboxing is disabled via + # the `no-sandbox` tag or execution requirement. + # + # We also use a non-dot prefixed name so it shows up more visibly. + args.add(paths.join(output_dir.path + "_doctrees"), format = "--doctree-dir=%s") for opt in ctx.attr.extra_opts: expanded = ctx.expand_location(opt) @@ -287,9 +314,6 @@ def _run_sphinx(ctx, format, source_path, inputs, output_prefix): for define in extra_defines: run_args.extend(("--define", define)) - args.add(source_path) - args.add(output_dir.path) - env = dict([ v.split("=", 1) for v in ctx.attr._extra_env_flag[_FlagInfo].value @@ -299,6 +323,14 @@ def _run_sphinx(ctx, format, source_path, inputs, output_prefix): for tool in ctx.attr.tools: tools.append(tool[DefaultInfo].files_to_run) + # NOTE: Command line flags or RBE capabilities may override the execution + # requirements and disable workers. Thus, we can't assume that these + # exec requirements will actually be respected. + execution_requirements = {} + if allow_persistent_workers: + execution_requirements["supports-workers"] = "1" + execution_requirements["requires-worker-protocol"] = "json" + ctx.actions.run( executable = ctx.executable.sphinx, arguments = [args], @@ -308,6 +340,7 @@ def _run_sphinx(ctx, format, source_path, inputs, output_prefix): mnemonic = "SphinxBuildDocs", progress_message = "Sphinx building {} for %{{label}}".format(format), env = env, + execution_requirements = execution_requirements, ) return output_dir, struct(args = run_args, env = env) diff --git a/sphinxdocs/private/sphinx_build.py b/sphinxdocs/private/sphinx_build.py index 3b7b32eaf6..e9711042f6 100644 --- a/sphinxdocs/private/sphinx_build.py +++ b/sphinxdocs/private/sphinx_build.py @@ -1,8 +1,235 @@ +import contextlib +import io +import json +import logging import os -import pathlib +import shutil import sys +import traceback +import typing +import sphinx.application from sphinx.cmd.build import main +WorkRequest = object +WorkResponse = object + +logger = logging.getLogger("sphinxdocs_build") + +_WORKER_SPHINX_EXT_MODULE_NAME = "bazel_worker_sphinx_ext" + +# Config value name for getting the path to the request info file +_REQUEST_INFO_CONFIG_NAME = "bazel_worker_request_info_path" + + +class Worker: + + def __init__( + self, instream: "typing.TextIO", outstream: "typing.TextIO", exec_root: str + ): + # NOTE: Sphinx performs its own logging re-configuration, so any + # logging config we do isn't respected by Sphinx. Controlling where + # stdout and stderr goes are the main mechanisms. Recall that + # Bazel send worker stderr to the worker log file. + # outputBase=$(bazel info output_base) + # find $outputBase/bazel-workers/ -type f -printf '%T@ %p\n' | sort -n | tail -1 | awk '{print $2}' + logging.basicConfig(level=logging.WARN) + logger.info("Initializing worker") + + # The directory that paths are relative to. + self._exec_root = exec_root + # Where requests are read from. + self._instream = instream + # Where responses are written to. + self._outstream = outstream + + # dict[str srcdir, dict[str path, str digest]] + self._digests = {} + + # Internal output directories the worker gives to Sphinx that need + # to be cleaned up upon exit. + # set[str path] + self._worker_outdirs = set() + self._extension = BazelWorkerExtension() + + sys.modules[_WORKER_SPHINX_EXT_MODULE_NAME] = self._extension + sphinx.application.builtin_extensions += (_WORKER_SPHINX_EXT_MODULE_NAME,) + + def __enter__(self): + return self + + def __exit__(self): + for worker_outdir in self._worker_outdirs: + shutil.rmtree(worker_outdir, ignore_errors=True) + + def run(self) -> None: + logger.info("Worker started") + try: + while True: + request = None + try: + request = self._get_next_request() + if request is None: + logger.info("Empty request: exiting") + break + response = self._process_request(request) + if response: + self._send_response(response) + except Exception: + logger.exception("Unhandled error: request=%s", request) + output = ( + f"Unhandled error:\nRequest id: {request.get('id')}\n" + + traceback.format_exc() + ) + request_id = 0 if not request else request.get("requestId", 0) + self._send_response( + { + "exitCode": 3, + "output": output, + "requestId": request_id, + } + ) + finally: + logger.info("Worker shutting down") + + def _get_next_request(self) -> "object | None": + line = self._instream.readline() + if not line: + return None + return json.loads(line) + + def _send_response(self, response: "WorkResponse") -> None: + self._outstream.write(json.dumps(response) + "\n") + self._outstream.flush() + + def _prepare_sphinx(self, request): + sphinx_args = request["arguments"] + srcdir = sphinx_args[0] + + incoming_digests = {} + current_digests = self._digests.setdefault(srcdir, {}) + changed_paths = [] + request_info = {"exec_root": self._exec_root, "inputs": request["inputs"]} + for entry in request["inputs"]: + path = entry["path"] + digest = entry["digest"] + # Make the path srcdir-relative so Sphinx understands it. + path = path.removeprefix(srcdir + "/") + incoming_digests[path] = digest + + if path not in current_digests: + logger.info("path %s new", path) + changed_paths.append(path) + elif current_digests[path] != digest: + logger.info("path %s changed", path) + changed_paths.append(path) + + self._digests[srcdir] = incoming_digests + self._extension.changed_paths = changed_paths + request_info["changed_sources"] = changed_paths + + bazel_outdir = sphinx_args[1] + worker_outdir = bazel_outdir + ".worker-out.d" + self._worker_outdirs.add(worker_outdir) + sphinx_args[1] = worker_outdir + + request_info_path = os.path.join(srcdir, "_bazel_worker_request_info.json") + with open(request_info_path, "w") as fp: + json.dump(request_info, fp) + sphinx_args.append(f"--define={_REQUEST_INFO_CONFIG_NAME}={request_info_path}") + + return worker_outdir, bazel_outdir, sphinx_args + + @contextlib.contextmanager + def _redirect_streams(self): + out = io.StringIO() + orig_stdout = sys.stdout + try: + sys.stdout = out + yield out + finally: + sys.stdout = orig_stdout + + def _process_request(self, request: "WorkRequest") -> "WorkResponse | None": + logger.info("Request: %s", json.dumps(request, sort_keys=True, indent=2)) + if request.get("cancel"): + return None + + worker_outdir, bazel_outdir, sphinx_args = self._prepare_sphinx(request) + + # Prevent anything from going to stdout because it breaks the worker + # protocol. We have limited control over where Sphinx sends output. + with self._redirect_streams() as stdout: + logger.info("main args: %s", sphinx_args) + exit_code = main(sphinx_args) + + if exit_code: + raise Exception( + "Sphinx main() returned failure: " + + f" exit code: {exit_code}\n" + + "========== STDOUT START ==========\n" + + stdout.getvalue().rstrip("\n") + + "\n" + + "========== STDOUT END ==========\n" + ) + + # Copying is unfortunately necessary because Bazel doesn't know to + # implicily bring along what the symlinks point to. + shutil.copytree(worker_outdir, bazel_outdir, dirs_exist_ok=True) + + response = { + "requestId": request.get("requestId", 0), + "output": stdout.getvalue(), + "exitCode": 0, + } + return response + + +class BazelWorkerExtension: + """A Sphinx extension implemented as a class acting like a module.""" + + def __init__(self): + # Make it look like a Module object + self.__name__ = _WORKER_SPHINX_EXT_MODULE_NAME + # set[str] of src-dir relative path names + self.changed_paths = set() + + def setup(self, app): + app.add_config_value(_REQUEST_INFO_CONFIG_NAME, "", "") + app.connect("env-get-outdated", self._handle_env_get_outdated) + return {"parallel_read_safe": True, "parallel_write_safe": True} + + def _handle_env_get_outdated(self, app, env, added, changed, removed): + changed = { + # NOTE: path2doc returns None if it's not a doc path + env.path2doc(p) + for p in self.changed_paths + } + + logger.info("changed docs: %s", changed) + return changed + + +def _worker_main(stdin, stdout, exec_root): + with Worker(stdin, stdout, exec_root) as worker: + return worker.run() + + +def _non_worker_main(): + args = [] + for arg in sys.argv: + if arg.startswith("@"): + with open(arg.removeprefix("@")) as fp: + lines = [line.strip() for line in fp if line.strip()] + args.extend(lines) + else: + args.append(arg) + sys.argv[:] = args + return main() + + if __name__ == "__main__": - sys.exit(main()) + if "--persistent_worker" in sys.argv: + sys.exit(_worker_main(sys.stdin, sys.stdout, os.getcwd())) + else: + sys.exit(_non_worker_main()) diff --git a/sphinxdocs/tests/sphinx_docs/doc1.md b/sphinxdocs/tests/sphinx_docs/doc1.md new file mode 100644 index 0000000000..f6f70ba28c --- /dev/null +++ b/sphinxdocs/tests/sphinx_docs/doc1.md @@ -0,0 +1,3 @@ +# doc1 + +hello doc 1 diff --git a/sphinxdocs/tests/sphinx_docs/doc2.md b/sphinxdocs/tests/sphinx_docs/doc2.md new file mode 100644 index 0000000000..06eb76a596 --- /dev/null +++ b/sphinxdocs/tests/sphinx_docs/doc2.md @@ -0,0 +1,3 @@ +# doc 2 + +hello doc 3 From d98547e8ec6bdbf4f250dd01c2921c2d91dc6db6 Mon Sep 17 00:00:00 2001 From: Aaron Levy Date: Tue, 10 Jun 2025 01:20:22 -0700 Subject: [PATCH 19/43] fix: Updating setuptools to patch CVE-2025-47273 (#2955) Update setuptools to patch CVE-2025-47273 --- CHANGELOG.md | 1 + python/private/pypi/deps.bzl | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e48e3d4f3d..eeafc70bae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,7 @@ END_UNRELEASED_TEMPLATE * (py_wheel) py_wheel always creates zip64-capable wheel zips * (providers) (experimental) {obj}`PyInfo.venv_symlinks` replaces `PyInfo.site_packages_symlinks` +* (deps) Updating setuptools to patch CVE-2025-47273. {#v0-0-0-fixed} ### Fixed diff --git a/python/private/pypi/deps.bzl b/python/private/pypi/deps.bzl index 31a5201659..73b30c69ee 100644 --- a/python/private/pypi/deps.bzl +++ b/python/private/pypi/deps.bzl @@ -76,8 +76,8 @@ _RULE_DEPS = [ ), ( "pypi__setuptools", - "https://files.pythonhosted.org/packages/de/88/70c5767a0e43eb4451c2200f07d042a4bcd7639276003a9c54a68cfcc1f8/setuptools-70.0.0-py3-none-any.whl", - "54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4", + "https://files.pythonhosted.org/packages/90/99/158ad0609729111163fc1f674a5a42f2605371a4cf036d0441070e2f7455/setuptools-78.1.1-py3-none-any.whl", + "c3a9c4211ff4c309edb8b8c4f1cbfa7ae324c4ba9f91ff254e3d305b9fd54561", ), ( "pypi__tomli", From 013acd944643dfc939639cdc6e8b49ef685ed314 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 10 Jun 2025 19:58:08 +0900 Subject: [PATCH 20/43] feat: data and pyi files in the venv (#2936) This adds the remaining of the files into the venv and should get us reasonably close to handling 99% of the cases. The expected differences from this and a `venv` built by `uv` would be: * The `RECORD` files are excluded from the `venv`s for better cache hit rate in `bazel`. Topological ordering is removed because topo ordering doesn't provide the "closer target first" guarantees desired. For now, just use default ordering and document conflicts as undefined behavior. Internally, it continues to use first-wins (i.e. first in depset.to_list() order) semantics. Work towards #2156 --- .bazelrc | 4 +- MODULE.bazel | 6 + internal_dev_deps.bzl | 5 + python/private/BUILD.bazel | 2 + python/private/attributes.bzl | 2 +- python/private/common.bzl | 7 +- python/private/py_executable.bzl | 65 +++++---- python/private/py_info.bzl | 38 ++--- python/private/py_library.bzl | 137 ++++++++++++------ tests/modules/another_module/BUILD.bazel | 5 + tests/modules/another_module/MODULE.bazel | 1 + .../another_module/another_module_data.txt | 1 + tests/modules/other/MODULE.bazel | 2 + tests/modules/other/simple_v1/BUILD.bazel | 14 ++ .../simple-1.0.0.dist-info/METADATA | 1 + .../site-packages/simple/__init__.py | 1 + .../site-packages/simple_v1_extras/data.txt | 0 tests/modules/other/simple_v2/BUILD.bazel | 15 ++ .../simple-2.0.0.dist-info/METADATA | 1 + .../simple-2.0.0.dist-info/licenses/LICENSE | 1 + .../site-packages/simple.libs/data.so | 2 + .../site-packages/simple/__init__.py | 1 + .../site-packages/simple/__init__.pyi | 1 + .../other/with_external_data/BUILD.bazel | 23 +++ .../site-packages/with_external_data.py | 1 + tests/venv_site_packages_libs/BUILD.bazel | 16 ++ tests/venv_site_packages_libs/bin.py | 49 ++++++- 27 files changed, 296 insertions(+), 105 deletions(-) create mode 100644 tests/modules/another_module/BUILD.bazel create mode 100644 tests/modules/another_module/MODULE.bazel create mode 100644 tests/modules/another_module/another_module_data.txt create mode 100644 tests/modules/other/simple_v1/BUILD.bazel create mode 100644 tests/modules/other/simple_v1/site-packages/simple-1.0.0.dist-info/METADATA create mode 100644 tests/modules/other/simple_v1/site-packages/simple/__init__.py create mode 100644 tests/modules/other/simple_v1/site-packages/simple_v1_extras/data.txt create mode 100644 tests/modules/other/simple_v2/BUILD.bazel create mode 100644 tests/modules/other/simple_v2/site-packages/simple-2.0.0.dist-info/METADATA create mode 100644 tests/modules/other/simple_v2/site-packages/simple-2.0.0.dist-info/licenses/LICENSE create mode 100644 tests/modules/other/simple_v2/site-packages/simple.libs/data.so create mode 100644 tests/modules/other/simple_v2/site-packages/simple/__init__.py create mode 100644 tests/modules/other/simple_v2/site-packages/simple/__init__.pyi create mode 100644 tests/modules/other/with_external_data/BUILD.bazel create mode 100644 tests/modules/other/with_external_data/site-packages/with_external_data.py diff --git a/.bazelrc b/.bazelrc index 7e744fb67a..f7f31aed98 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data test --test_output=errors diff --git a/MODULE.bazel b/MODULE.bazel index 144e130c1b..77fa12d113 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -86,6 +86,7 @@ bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) bazel_dep(name = "bazel_ci_rules", version = "1.0.0", dev_dependency = True) bazel_dep(name = "rules_pkg", version = "1.0.1", dev_dependency = True) bazel_dep(name = "other", version = "0", dev_dependency = True) +bazel_dep(name = "another_module", version = "0", dev_dependency = True) # Extra gazelle plugin deps so that WORKSPACE.bzlmod can continue including it for e2e tests. # We use `WORKSPACE.bzlmod` because it is impossible to have dev-only local overrides. @@ -116,6 +117,11 @@ local_path_override( path = "tests/modules/other", ) +local_path_override( + module_name = "another_module", + path = "tests/modules/another_module", +) + dev_python = use_extension( "//python/extensions:python.bzl", "python", diff --git a/internal_dev_deps.bzl b/internal_dev_deps.bzl index f2b33e279e..e6ade4035c 100644 --- a/internal_dev_deps.bzl +++ b/internal_dev_deps.bzl @@ -48,6 +48,11 @@ def rules_python_internal_deps(): path = "tests/modules/other", ) + local_repository( + name = "another_module", + path = "tests/modules/another_module", + ) + http_archive( name = "bazel_skylib", sha256 = "bc283cdfcd526a52c3201279cda4bc298652efa898b10b4db0837dc51652756f", diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index b319919305..8bcc6eaebe 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -450,11 +450,13 @@ bzl_library( ":attributes_bzl", ":common_bzl", ":flags_bzl", + ":normalize_name_bzl", ":precompile_bzl", ":py_cc_link_params_info_bzl", ":py_internal_bzl", ":rule_builders_bzl", ":toolchain_types_bzl", + ":version_bzl", "@bazel_skylib//lib:dicts", "@bazel_skylib//rules:common_settings", ], diff --git a/python/private/attributes.bzl b/python/private/attributes.bzl index ad8cba2e6c..c3b1cade91 100644 --- a/python/private/attributes.bzl +++ b/python/private/attributes.bzl @@ -260,7 +260,7 @@ The order of this list can matter because it affects the order that information from dependencies is merged in, which can be relevant depending on the ordering mode of depsets that are merged. -* {obj}`PyInfo.venv_symlinks` uses topological ordering. +* {obj}`PyInfo.venv_symlinks` uses default ordering. See {obj}`PyInfo` for more information about the ordering of its depsets and how its fields are merged. diff --git a/python/private/common.bzl b/python/private/common.bzl index e49dbad20c..163fb54d77 100644 --- a/python/private/common.bzl +++ b/python/private/common.bzl @@ -331,7 +331,7 @@ def collect_runfiles(ctx, files = depset()): # If the target is a File, then add that file to the runfiles. # Otherwise, add the target's **data runfiles** to the runfiles. # - # Note that, contray to best practice, the default outputs of the + # Note that, contrary to best practice, the default outputs of the # targets in `data` are *not* added, nor are the default runfiles. # # This ends up being important for several reasons, some of which are @@ -396,9 +396,8 @@ def create_py_info( implicit_pyc_files: {type}`depset[File]` Implicitly generated pyc files that a binary can choose to include. imports: depset of strings; the import path values to propagate. - venv_symlinks: {type}`list[tuple[str, str]]` tuples of - `(runfiles_path, site_packages_path)` for symlinks to create - in the consuming binary's venv site packages. + venv_symlinks: {type}`list[VenvSymlinkEntry]` instances for + symlinks to create in the consuming binary's venv. Returns: A tuple of the PyInfo instance and a depset of the diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 7c3e0cb757..7e50247e61 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -650,10 +650,6 @@ def _create_venv_symlinks(ctx, venv_dir_map): # maps venv-relative path to the runfiles path it should point to entries = depset( - # NOTE: Topological ordering is used so that dependencies closer to the - # binary have precedence in creating their symlinks. This allows the - # binary a modicum of control over the result. - order = "topological", transitive = [ dep[PyInfo].venv_symlinks for dep in ctx.attr.deps @@ -680,43 +676,52 @@ def _create_venv_symlinks(ctx, venv_dir_map): return venv_files def _build_link_map(entries): - # dict[str kind, dict[str rel_path, str link_to_path]] - link_map = {} + # dict[str package, dict[str kind, dict[str rel_path, str link_to_path]]] + pkg_link_map = {} + + # dict[str package, str version] + version_by_pkg = {} + for entry in entries: - kind = entry.kind - kind_map = link_map.setdefault(kind, {}) - if entry.venv_path in kind_map: - # We ignore duplicates by design. The dependency closer to the - # binary gets precedence due to the topological ordering. + link_map = pkg_link_map.setdefault(entry.package, {}) + kind_map = link_map.setdefault(entry.kind, {}) + + if version_by_pkg.setdefault(entry.package, entry.version) != entry.version: + # We ignore duplicates by design. + continue + elif entry.venv_path in kind_map: + # We ignore duplicates by design. continue else: kind_map[entry.venv_path] = entry.link_to_path - # An empty link_to value means to not create the site package symlink. - # Because of the topological ordering, this allows binaries to remove - # entries by having an earlier dependency produce empty link_to values. - for kind, kind_map in link_map.items(): - for dir_path, link_to in kind_map.items(): - if not link_to: - kind_map.pop(dir_path) + # An empty link_to value means to not create the site package symlink. Because of the + # ordering, this allows binaries to remove entries by having an earlier dependency produce + # empty link_to values. + for link_map in pkg_link_map.values(): + for kind, kind_map in link_map.items(): + for dir_path, link_to in kind_map.items(): + if not link_to: + kind_map.pop(dir_path) # dict[str kind, dict[str rel_path, str link_to_path]] keep_link_map = {} # Remove entries that would be a child path of a created symlink. # Earlier entries have precedence to match how exact matches are handled. - for kind, kind_map in link_map.items(): - keep_kind_map = keep_link_map.setdefault(kind, {}) - for _ in range(len(kind_map)): - if not kind_map: - break - dirname, value = kind_map.popitem() - keep_kind_map[dirname] = value - prefix = dirname + "/" # Add slash to prevent /X matching /XY - for maybe_suffix in kind_map.keys(): - maybe_suffix += "/" # Add slash to prevent /X matching /XY - if maybe_suffix.startswith(prefix) or prefix.startswith(maybe_suffix): - kind_map.pop(maybe_suffix) + for link_map in pkg_link_map.values(): + for kind, kind_map in link_map.items(): + keep_kind_map = keep_link_map.setdefault(kind, {}) + for _ in range(len(kind_map)): + if not kind_map: + break + dirname, value = kind_map.popitem() + keep_kind_map[dirname] = value + prefix = dirname + "/" # Add slash to prevent /X matching /XY + for maybe_suffix in kind_map.keys(): + maybe_suffix += "/" # Add slash to prevent /X matching /XY + if maybe_suffix.startswith(prefix) or prefix.startswith(maybe_suffix): + kind_map.pop(maybe_suffix) return keep_link_map def _map_each_identity(v): diff --git a/python/private/py_info.bzl b/python/private/py_info.bzl index 2a2f4554e3..17c5e4e79e 100644 --- a/python/private/py_info.bzl +++ b/python/private/py_info.bzl @@ -67,11 +67,24 @@ the venv to create the path under. A runfiles-root relative path that `venv_path` will symlink to. If `None`, it means to not create a symlink. +""", + "package": """ +:type: str | None + +Represents the PyPI package name that the code originates from. It is normalized according to the +PEP440 with all `-` replaced with `_`, i.e. the same as the package name in the hub repository that +it would come from. """, "venv_path": """ :type: str A path relative to the `kind` directory within the venv. +""", + "version": """ +:type: str | None + +Represents the PyPI package version that the code originates from. It is normalized according to the +PEP440 standard. """, }, ) @@ -296,29 +309,9 @@ This field is currently unused in Bazel and may go away in the future. "venv_symlinks": """ :type: depset[VenvSymlinkEntry] -A depset with `topological` ordering. - - -Tuples of `(runfiles_path, site_packages_path)`. Where -* `runfiles_path` is a runfiles-root relative path. It is the path that - has the code to make importable. If `None` or empty string, then it means - to not create a site packages directory with the `site_packages_path` - name. -* `site_packages_path` is a path relative to the site-packages directory of - the venv for whatever creates the venv (typically py_binary). It makes - the code in `runfiles_path` available for import. Note that this - is created as a "raw" symlink (via `declare_symlink`). - :::{include} /_includes/experimental_api.md ::: -:::{tip} -The topological ordering means dependencies earlier and closer to the consumer -have precedence. This allows e.g. a binary to add dependencies that override -values from further way dependencies, such as forcing symlinks to point to -specific paths or preventing symlinks from being created. -::: - :::{versionadded} VERSION_NEXT_FEATURE ::: """, @@ -375,9 +368,6 @@ def _PyInfoBuilder_typedef(): :::{field} venv_symlinks :type: DepsetBuilder[tuple[str | None, str]] - - NOTE: This depset has `topological` order - ::: """ def _PyInfoBuilder_new(): @@ -417,7 +407,7 @@ def _PyInfoBuilder_new(): transitive_pyc_files = builders.DepsetBuilder(), transitive_pyi_files = builders.DepsetBuilder(), transitive_sources = builders.DepsetBuilder(), - venv_symlinks = builders.DepsetBuilder(order = "topological"), + venv_symlinks = builders.DepsetBuilder(), ) return self diff --git a/python/private/py_library.bzl b/python/private/py_library.bzl index fabc880a8d..e727694b32 100644 --- a/python/private/py_library.bzl +++ b/python/private/py_library.bzl @@ -41,6 +41,7 @@ load( "runfiles_root_path", ) load(":flags.bzl", "AddSrcsToRunfilesFlag", "PrecompileFlag", "VenvsSitePackages") +load(":normalize_name.bzl", "normalize_name") load(":precompile.bzl", "maybe_precompile") load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") load(":py_info.bzl", "PyInfo", "VenvSymlinkEntry", "VenvSymlinkKind") @@ -52,6 +53,7 @@ load( "EXEC_TOOLS_TOOLCHAIN_TYPE", TOOLCHAIN_TYPE = "TARGET_TOOLCHAIN_TYPE", ) +load(":version.bzl", "version") _py_builtins = py_internal @@ -84,20 +86,22 @@ under the binary's venv site-packages directory that should be made available (i namespace packages]( https://packaging.python.org/en/latest/guides/packaging-namespace-packages/#native-namespace-packages). However, the *content* of the files cannot be taken into account, merely their -presence or absense. Stated another way: [pkgutil-style namespace packages]( +presence or absence. Stated another way: [pkgutil-style namespace packages]( https://packaging.python.org/en/latest/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages) won't be understood as namespace packages; they'll be seen as regular packages. This will likely lead to conflicts with other targets that contribute to the namespace. -:::{tip} -This attributes populates {obj}`PyInfo.venv_symlinks`, which is -a topologically ordered depset. This means dependencies closer and earlier -to a consumer have precedence. See {obj}`PyInfo.venv_symlinks` for -more information. +:::{seealso} +This attributes populates {obj}`PyInfo.venv_symlinks`. ::: :::{versionadded} 1.4.0 ::: +:::{versionchanged} VERSION_NEXT_FEATURE +The topological order has been removed and if 2 different versions of the same PyPI +package are observed, the behaviour has no guarantees except that it is deterministic +and that only one package version will be included. +::: """, ), "_add_srcs_to_runfiles_flag": lambda: attrb.Label( @@ -157,7 +161,8 @@ def py_library_impl(ctx, *, semantics): imports = [] venv_symlinks = [] - imports, venv_symlinks = _get_imports_and_venv_symlinks(ctx, semantics) + package, version_str = _get_package_and_version(ctx) + imports, venv_symlinks = _get_imports_and_venv_symlinks(ctx, semantics, package, version_str) cc_info = semantics.get_cc_info_for_library(ctx) py_info, deps_transitive_sources, builtins_py_info = create_py_info( @@ -206,16 +211,46 @@ Source files are no longer added to the runfiles directly. ::: """ -def _get_imports_and_venv_symlinks(ctx, semantics): +def _get_package_and_version(ctx): + """Return package name and version + + If the package comes from PyPI then it will have a `.dist-info` as part of `data`, which + allows us to get the name of the package and its version. + """ + dist_info_metadata = None + for d in ctx.files.data: + # work on case insensitive FSes + if d.basename.lower() != "metadata": + continue + + if d.dirname.endswith(".dist-info"): + dist_info_metadata = d + + if not dist_info_metadata: + return None, None + + # in order to be able to have replacements in the venv, we have to add a + # third value into the venv_symlinks, which would be the normalized + # package name. This allows us to ensure that we can replace the `dist-info` + # directories by checking if the package key is there. + dist_info_dir = paths.basename(dist_info_metadata.dirname) + package, _, _suffix = dist_info_dir.rpartition(".dist-info") + package, _, version_str = package.rpartition("-") + return ( + normalize_name(package), # will have no dashes + version.normalize(version_str), # will have no dashes either + ) + +def _get_imports_and_venv_symlinks(ctx, semantics, package, version_str): imports = depset() - venv_symlinks = depset() + venv_symlinks = [] if VenvsSitePackages.is_enabled(ctx): - venv_symlinks = _get_venv_symlinks(ctx) + venv_symlinks = _get_venv_symlinks(ctx, package, version_str) else: imports = collect_imports(ctx, semantics) return imports, venv_symlinks -def _get_venv_symlinks(ctx): +def _get_venv_symlinks(ctx, package, version_str): imports = ctx.attr.imports if len(imports) == 0: fail("When venvs_site_packages is enabled, exactly one `imports` " + @@ -236,50 +271,61 @@ def _get_venv_symlinks(ctx): # Append slash to prevent incorrectly prefix-string matches site_packages_root += "/" - # We have to build a list of (runfiles path, site-packages path) pairs of - # the files to create in the consuming binary's venv site-packages directory. - # To minimize the number of files to create, we just return the paths - # to the directories containing the code of interest. + # We have to build a list of (runfiles path, site-packages path) pairs of the files to + # create in the consuming binary's venv site-packages directory. To minimize the number of + # files to create, we just return the paths to the directories containing the code of + # interest. + # + # However, namespace packages complicate matters: multiple distributions install in the + # same directory in site-packages. This works out because they don't overlap in their + # files. Typically, they install to different directories within the namespace package + # directory. We also need to ensure that we can handle a case where the main package (e.g. + # airflow) has directories only containing data files and then namespace packages coming + # along and being next to it. # - # However, namespace packages complicate matters: multiple - # distributions install in the same directory in site-packages. This - # works out because they don't overlap in their files. Typically, they - # install to different directories within the namespace package - # directory. Namespace package directories are simply directories - # within site-packages that *don't* have an `__init__.py` file, which - # can be arbitrarily deep. Thus, we simply have to look for the - # directories that _do_ have an `__init__.py` file and treat those as - # the path to symlink to. - - repo_runfiles_dirname = None - dirs_with_init = {} # dirname -> runfile path + # Lastly we have to assume python modules just being `.py` files (e.g. typing-extensions) + # is just a single Python file. + + dir_symlinks = {} # dirname -> runfile path venv_symlinks = [] - for src in ctx.files.srcs: - if src.extension not in PYTHON_FILE_EXTENSIONS: - continue + for src in ctx.files.srcs + ctx.files.data + ctx.files.pyi_srcs: path = _repo_relative_short_path(src.short_path) if not path.startswith(site_packages_root): continue path = path.removeprefix(site_packages_root) dir_name, _, filename = path.rpartition("/") - if dir_name and filename.startswith("__init__."): - dirs_with_init[dir_name] = None - repo_runfiles_dirname = runfiles_root_path(ctx, src.short_path).partition("/")[0] - elif not dir_name: - repo_runfiles_dirname = runfiles_root_path(ctx, src.short_path).partition("/")[0] + if dir_name in dir_symlinks: + # we already have this dir, this allows us to short-circuit since most of the + # ctx.files.data might share the same directories as ctx.files.srcs + continue + runfiles_dir_name, _, _ = runfiles_root_path(ctx, src.short_path).partition("/") + if dir_name: + # This can be either: + # * a directory with libs (e.g. numpy.libs, created by auditwheel) + # * a directory with `__init__.py` file that potentially also needs to be + # symlinked. + # * `.dist-info` directory + # + # This could be also regular files, that just need to be symlinked, so we will + # add the directory here. + dir_symlinks[dir_name] = runfiles_dir_name + elif src.extension in PYTHON_FILE_EXTENSIONS: # This would be files that do not have directories and we just need to add - # direct symlinks to them as is: - venv_symlinks.append(VenvSymlinkEntry( + # direct symlinks to them as is, we only allow Python files in here + entry = VenvSymlinkEntry( kind = VenvSymlinkKind.LIB, - link_to_path = paths.join(repo_runfiles_dirname, site_packages_root, filename), + link_to_path = paths.join(runfiles_dir_name, site_packages_root, filename), + package = package, + version = version_str, venv_path = filename, - )) + ) + venv_symlinks.append(entry) # Sort so that we encounter `foo` before `foo/bar`. This ensures we # see the top-most explicit package first. - dirnames = sorted(dirs_with_init.keys()) + dirnames = sorted(dir_symlinks.keys()) first_level_explicit_packages = [] for d in dirnames: is_sub_package = False @@ -292,11 +338,16 @@ def _get_venv_symlinks(ctx): first_level_explicit_packages.append(d) for dirname in first_level_explicit_packages: - venv_symlinks.append(VenvSymlinkEntry( + prefix = dir_symlinks[dirname] + entry = VenvSymlinkEntry( kind = VenvSymlinkKind.LIB, - link_to_path = paths.join(repo_runfiles_dirname, site_packages_root, dirname), + link_to_path = paths.join(prefix, site_packages_root, dirname), + package = package, + version = version_str, venv_path = dirname, - )) + ) + venv_symlinks.append(entry) + return venv_symlinks def _repo_relative_short_path(short_path): diff --git a/tests/modules/another_module/BUILD.bazel b/tests/modules/another_module/BUILD.bazel new file mode 100644 index 0000000000..3b56b6ee83 --- /dev/null +++ b/tests/modules/another_module/BUILD.bazel @@ -0,0 +1,5 @@ +filegroup( + name = "data", + srcs = ["another_module_data.txt"], + visibility = ["//visibility:public"], +) diff --git a/tests/modules/another_module/MODULE.bazel b/tests/modules/another_module/MODULE.bazel new file mode 100644 index 0000000000..8ed5a5543b --- /dev/null +++ b/tests/modules/another_module/MODULE.bazel @@ -0,0 +1 @@ +module(name = "another_module") diff --git a/tests/modules/another_module/another_module_data.txt b/tests/modules/another_module/another_module_data.txt new file mode 100644 index 0000000000..f742ebab60 --- /dev/null +++ b/tests/modules/another_module/another_module_data.txt @@ -0,0 +1 @@ +print("token") diff --git a/tests/modules/other/MODULE.bazel b/tests/modules/other/MODULE.bazel index 7cd3118b81..11a633d56b 100644 --- a/tests/modules/other/MODULE.bazel +++ b/tests/modules/other/MODULE.bazel @@ -1,3 +1,5 @@ module(name = "other") bazel_dep(name = "rules_python", version = "0") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "another_module", version = "0") diff --git a/tests/modules/other/simple_v1/BUILD.bazel b/tests/modules/other/simple_v1/BUILD.bazel new file mode 100644 index 0000000000..da5db8164a --- /dev/null +++ b/tests/modules/other/simple_v1/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_python//python:py_library.bzl", "py_library") + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "simple_v1", + srcs = glob(["site-packages/**/*.py"]), + data = glob( + ["**/*"], + exclude = ["site-packages/**/*.py"], + ), + experimental_venvs_site_packages = "@rules_python//python/config_settings:venvs_site_packages", + imports = [package_name() + "/site-packages"], +) diff --git a/tests/modules/other/simple_v1/site-packages/simple-1.0.0.dist-info/METADATA b/tests/modules/other/simple_v1/site-packages/simple-1.0.0.dist-info/METADATA new file mode 100644 index 0000000000..ee76ec48a4 --- /dev/null +++ b/tests/modules/other/simple_v1/site-packages/simple-1.0.0.dist-info/METADATA @@ -0,0 +1 @@ +inside is v1 diff --git a/tests/modules/other/simple_v1/site-packages/simple/__init__.py b/tests/modules/other/simple_v1/site-packages/simple/__init__.py new file mode 100644 index 0000000000..5becc17c04 --- /dev/null +++ b/tests/modules/other/simple_v1/site-packages/simple/__init__.py @@ -0,0 +1 @@ +__version__ = "1.0.0" diff --git a/tests/modules/other/simple_v1/site-packages/simple_v1_extras/data.txt b/tests/modules/other/simple_v1/site-packages/simple_v1_extras/data.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/modules/other/simple_v2/BUILD.bazel b/tests/modules/other/simple_v2/BUILD.bazel new file mode 100644 index 0000000000..45f83a5a88 --- /dev/null +++ b/tests/modules/other/simple_v2/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_python//python:py_library.bzl", "py_library") + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "simple_v2", + srcs = glob(["site-packages/**/*.py"]), + data = glob( + ["**/*"], + exclude = ["site-packages/**/*.py"], + ), + experimental_venvs_site_packages = "@rules_python//python/config_settings:venvs_site_packages", + imports = [package_name() + "/site-packages"], + pyi_srcs = glob(["**/*.pyi"]), +) diff --git a/tests/modules/other/simple_v2/site-packages/simple-2.0.0.dist-info/METADATA b/tests/modules/other/simple_v2/site-packages/simple-2.0.0.dist-info/METADATA new file mode 100644 index 0000000000..ee76ec48a4 --- /dev/null +++ b/tests/modules/other/simple_v2/site-packages/simple-2.0.0.dist-info/METADATA @@ -0,0 +1 @@ +inside is v1 diff --git a/tests/modules/other/simple_v2/site-packages/simple-2.0.0.dist-info/licenses/LICENSE b/tests/modules/other/simple_v2/site-packages/simple-2.0.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000..0cb5e79499 --- /dev/null +++ b/tests/modules/other/simple_v2/site-packages/simple-2.0.0.dist-info/licenses/LICENSE @@ -0,0 +1 @@ +Some License diff --git a/tests/modules/other/simple_v2/site-packages/simple.libs/data.so b/tests/modules/other/simple_v2/site-packages/simple.libs/data.so new file mode 100644 index 0000000000..f023e3b9ae --- /dev/null +++ b/tests/modules/other/simple_v2/site-packages/simple.libs/data.so @@ -0,0 +1,2 @@ +# This is usually created by auditwheel when processing linux wheels and including +# dependencies. diff --git a/tests/modules/other/simple_v2/site-packages/simple/__init__.py b/tests/modules/other/simple_v2/site-packages/simple/__init__.py new file mode 100644 index 0000000000..8c0d5d5bb2 --- /dev/null +++ b/tests/modules/other/simple_v2/site-packages/simple/__init__.py @@ -0,0 +1 @@ +__version__ = "2.0.0" diff --git a/tests/modules/other/simple_v2/site-packages/simple/__init__.pyi b/tests/modules/other/simple_v2/site-packages/simple/__init__.pyi new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/tests/modules/other/simple_v2/site-packages/simple/__init__.pyi @@ -0,0 +1 @@ +# Intentionally empty diff --git a/tests/modules/other/with_external_data/BUILD.bazel b/tests/modules/other/with_external_data/BUILD.bazel new file mode 100644 index 0000000000..fc047aadab --- /dev/null +++ b/tests/modules/other/with_external_data/BUILD.bazel @@ -0,0 +1,23 @@ +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") +load("@rules_python//python:py_library.bzl", "py_library") + +package(default_visibility = ["//visibility:public"]) + +# The users may include data through other repos via annotations and copy_file +# just add this edge case. +# +# NOTE: if the data is not copied to `site-packages/` then it will not +# appear. +copy_file( + name = "external_data", + src = "@another_module//:data", + out = "site-packages/external_data/another_module_data.txt", +) + +py_library( + name = "with_external_data", + srcs = ["site-packages/with_external_data.py"], + data = [":external_data"], + experimental_venvs_site_packages = "@rules_python//python/config_settings:venvs_site_packages", + imports = [package_name() + "/site-packages"], +) diff --git a/tests/modules/other/with_external_data/site-packages/with_external_data.py b/tests/modules/other/with_external_data/site-packages/with_external_data.py new file mode 100644 index 0000000000..ccd9dcef9e --- /dev/null +++ b/tests/modules/other/with_external_data/site-packages/with_external_data.py @@ -0,0 +1 @@ +# Intentionally blank diff --git a/tests/venv_site_packages_libs/BUILD.bazel b/tests/venv_site_packages_libs/BUILD.bazel index d5a4fe6750..e64299e1ad 100644 --- a/tests/venv_site_packages_libs/BUILD.bazel +++ b/tests/venv_site_packages_libs/BUILD.bazel @@ -1,6 +1,20 @@ +load("//python:py_library.bzl", "py_library") load("//tests/support:py_reconfig.bzl", "py_reconfig_test") load("//tests/support:support.bzl", "SUPPORTS_BOOTSTRAP_SCRIPT") +py_library( + name = "user_lib", + deps = ["@other//simple_v1"], +) + +py_library( + name = "closer_lib", + deps = [ + ":user_lib", + "@other//simple_v2", + ], +) + py_reconfig_test( name = "venvs_site_packages_libs_test", srcs = ["bin.py"], @@ -9,10 +23,12 @@ py_reconfig_test( target_compatible_with = SUPPORTS_BOOTSTRAP_SCRIPT, venvs_site_packages = "yes", deps = [ + ":closer_lib", "//tests/venv_site_packages_libs/nspkg_alpha", "//tests/venv_site_packages_libs/nspkg_beta", "@other//nspkg_delta", "@other//nspkg_gamma", "@other//nspkg_single", + "@other//with_external_data", ], ) diff --git a/tests/venv_site_packages_libs/bin.py b/tests/venv_site_packages_libs/bin.py index 58572a2a1e..7e5838d2c2 100644 --- a/tests/venv_site_packages_libs/bin.py +++ b/tests/venv_site_packages_libs/bin.py @@ -1,7 +1,7 @@ import importlib -import os import sys import unittest +from pathlib import Path class VenvSitePackagesLibraryTest(unittest.TestCase): @@ -27,6 +27,53 @@ def test_imported_from_venv(self): self.assert_imported_from_venv("nspkg.subnspkg.gamma") self.assert_imported_from_venv("nspkg.subnspkg.delta") self.assert_imported_from_venv("single_file") + self.assert_imported_from_venv("simple") + + def test_data_is_included(self): + self.assert_imported_from_venv("simple") + module = importlib.import_module("simple") + module_path = Path(module.__file__) + + site_packages = module_path.parent.parent + + # Ensure that packages from simple v1 are not present + files = [p.name for p in site_packages.glob("*")] + self.assertIn("simple_v1_extras", files) + + def test_override_pkg(self): + self.assert_imported_from_venv("simple") + module = importlib.import_module("simple") + self.assertEqual( + "1.0.0", + module.__version__, + ) + + def test_dirs_from_replaced_package_are_not_present(self): + self.assert_imported_from_venv("simple") + module = importlib.import_module("simple") + module_path = Path(module.__file__) + + site_packages = module_path.parent.parent + dist_info_dirs = [p.name for p in site_packages.glob("*.dist-info")] + self.assertEqual( + ["simple-1.0.0.dist-info"], + dist_info_dirs, + ) + + # Ensure that packages from simple v1 are not present + files = [p.name for p in site_packages.glob("*")] + self.assertNotIn("simple.libs", files) + + def test_data_from_another_pkg_is_included_via_copy_file(self): + self.assert_imported_from_venv("simple") + module = importlib.import_module("simple") + module_path = Path(module.__file__) + + site_packages = module_path.parent.parent + # Ensure that packages from simple v1 are not present + d = site_packages / "external_data" + files = [p.name for p in d.glob("*")] + self.assertIn("another_module_data.txt", files) if __name__ == "__main__": From cb1c382144f59f7190781fb19e090aee23536e65 Mon Sep 17 00:00:00 2001 From: Ted Kaplan Date: Tue, 10 Jun 2025 19:07:41 -0700 Subject: [PATCH 21/43] fix(pypi): Only show index_url_overrides warnings when they are needed (#2967) Fixes #2966 --- python/private/pypi/simpleapi_download.bzl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/private/pypi/simpleapi_download.bzl b/python/private/pypi/simpleapi_download.bzl index 164d4e8dbd..a3ba9691cd 100644 --- a/python/private/pypi/simpleapi_download.bzl +++ b/python/private/pypi/simpleapi_download.bzl @@ -148,10 +148,11 @@ def simpleapi_download( if found_on_index[pkg] != attr.index_url } - # buildifier: disable=print - print("You can use the following `index_url_overrides` to avoid the 404 warnings:\n{}".format( - render.dict(index_url_overrides), - )) + if index_url_overrides: + # buildifier: disable=print + print("You can use the following `index_url_overrides` to avoid the 404 warnings:\n{}".format( + render.dict(index_url_overrides), + )) return contents From 95fb54a5e7146fd9c743f2984814f444798c9233 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 10 Jun 2025 22:11:52 -0700 Subject: [PATCH 22/43] revert: change default bootstrap back to system_python (#2968) Switch the default bootstrap back to system_python, per maintainer discussion. The main reason is downstream consumers are unlikely to be fully ready for the usage of raw symlinks (declare_symlink artifacts). APIs to detect them aren't available until Bazel 8, which makes it difficult for packaging rules, such as rules_pkg, bazel-lib, or tar rules. This reverts the core part of commit 9f3512fe0cc6d7229170e45724e22e64be0b8300 --- CHANGELOG.md | 8 -------- docs/api/rules_python/python/config_settings/index.md | 11 +---------- python/config_settings/BUILD.bazel | 2 +- 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eeafc70bae..e8fa1751c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,14 +55,6 @@ END_UNRELEASED_TEMPLATE {#v0-0-0-changed} ### Changed -* If using the (deprecated) autodetecting/runtime_env toolchain, then the Python - version specified at build-time *must* match the Python version used at - runtime (the {obj}`--@rules_python//python/config_settings:python_version` - flag and the {attr}`python_version` attribute control the build-time version - for a target). If they don't match, dependencies won't be importable. (Such a - misconfiguration was unlikely to work to begin with; this is called out as an - FYI). -* (rules) {obj}`--bootstrap_impl=script` is the default for non-Windows. * (rules) On Windows, {obj}`--bootstrap_impl=system_python` is forced. This allows setting `--bootstrap_impl=script` in bazelrc for mixed-platform environments. diff --git a/docs/api/rules_python/python/config_settings/index.md b/docs/api/rules_python/python/config_settings/index.md index ae84d40b13..7fe25888dd 100644 --- a/docs/api/rules_python/python/config_settings/index.md +++ b/docs/api/rules_python/python/config_settings/index.md @@ -245,12 +245,8 @@ Values: ::::{bzl:flag} bootstrap_impl Determine how programs implement their startup process. -The default for this depends on the platform: -* Windows: `system_python` (**always** used) -* Other: `script` - Values: -* `system_python`: Use a bootstrap that requires a system Python available +* `system_python`: (default) Use a bootstrap that requires a system Python available in order to start programs. This requires {obj}`PyRuntimeInfo.bootstrap_template` to be a Python program. * `script`: Use a bootstrap that uses an arbitrary executable script (usually a @@ -273,11 +269,6 @@ instead. :::{versionadded} 0.33.0 ::: -:::{versionchanged} VERSION_NEXT_FEATURE -* The default for non-Windows changed from `system_python` to `script`. -* On Windows, the value is forced to `system_python`. -::: - :::: ::::{bzl:flag} current_config diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index ee15828fa5..b11580c4cb 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -90,7 +90,7 @@ string_flag( rp_string_flag( name = "bootstrap_impl", - build_setting_default = BootstrapImplFlag.SCRIPT, + build_setting_default = BootstrapImplFlag.SYSTEM_PYTHON, override = select({ # Windows doesn't yet support bootstrap=script, so force disable it ":_is_windows": BootstrapImplFlag.SYSTEM_PYTHON, From fb2298a7f2e6789186d63ef645ceed96261d94a9 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 11 Jun 2025 13:55:19 -0700 Subject: [PATCH 23/43] fix: grammar in an error message (#2971) --- python/private/pypi/extension.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index b79be6e038..867abe0898 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -263,7 +263,7 @@ def _create_whl_repos( repo_name = "{}_{}".format(pip_name, repo.repo_name) if repo_name in whl_libraries: - fail("Attempting to creating a duplicate library {} for {}".format( + fail("attempting to create a duplicate library {} for {}".format( repo_name, whl.name, )) From e03b63c725cbef77a5c9af254331086de4649e15 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Wed, 11 Jun 2025 15:09:44 -0700 Subject: [PATCH 24/43] refactor: Add missing uses of DefaultInfo (#2972) Required for compatibility with https://github.com/bazelbuild/bazel/issues/20183 --- python/private/common.bzl | 4 ++-- python/private/py_wheel.bzl | 4 ++-- python/uv/private/uv_toolchain.bzl | 2 +- sphinxdocs/private/sphinx.bzl | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/private/common.bzl b/python/private/common.bzl index 163fb54d77..96f8ebeab4 100644 --- a/python/private/common.bzl +++ b/python/private/common.bzl @@ -425,7 +425,7 @@ def create_py_info( else: # TODO(b/228692666): Remove this once non-PyInfo targets are no # longer supported in `deps`. - files = target.files.to_list() + files = target[DefaultInfo].files.to_list() for f in files: if f.extension == "py": py_info.transitive_sources.add(f) @@ -449,7 +449,7 @@ def create_py_info( info = _get_py_info(target) py_info.merge_uses_shared_libraries(info.uses_shared_libraries) else: - files = target.files.to_list() + files = target[DefaultInfo].files.to_list() for f in files: py_info.merge_uses_shared_libraries(cc_helper.is_valid_shared_library_artifact(f)) if py_info.get_uses_shared_libraries(): diff --git a/python/private/py_wheel.bzl b/python/private/py_wheel.bzl index ffc24f6846..cfd4efdcda 100644 --- a/python/private/py_wheel.bzl +++ b/python/private/py_wheel.bzl @@ -480,7 +480,7 @@ def _py_wheel_impl(ctx): args.add("--no_compress") for target, filename in ctx.attr.extra_distinfo_files.items(): - target_files = target.files.to_list() + target_files = target[DefaultInfo].files.to_list() if len(target_files) != 1: fail( "Multi-file target listed in extra_distinfo_files %s", @@ -493,7 +493,7 @@ def _py_wheel_impl(ctx): ) for target, filename in ctx.attr.data_files.items(): - target_files = target.files.to_list() + target_files = target[DefaultInfo].files.to_list() if len(target_files) != 1: fail( "Multi-file target listed in data_files %s", diff --git a/python/uv/private/uv_toolchain.bzl b/python/uv/private/uv_toolchain.bzl index 8c7f1b4b8c..bd82e7452f 100644 --- a/python/uv/private/uv_toolchain.bzl +++ b/python/uv/private/uv_toolchain.bzl @@ -24,7 +24,7 @@ def _uv_toolchain_impl(ctx): uv = ctx.attr.uv default_info = DefaultInfo( - files = uv.files, + files = uv[DefaultInfo].files, runfiles = uv[DefaultInfo].default_runfiles, ) uv_toolchain_info = UvToolchainInfo( diff --git a/sphinxdocs/private/sphinx.bzl b/sphinxdocs/private/sphinx.bzl index ee6b994e2e..c1efda3508 100644 --- a/sphinxdocs/private/sphinx.bzl +++ b/sphinxdocs/private/sphinx.bzl @@ -386,7 +386,7 @@ def _sphinx_source_tree_impl(ctx): _relocate(orig_file) for src_target, dest in ctx.attr.renamed_srcs.items(): - src_files = src_target.files.to_list() + src_files = src_target[DefaultInfo].files.to_list() if len(src_files) != 1: fail("A single file must be specified to be renamed. Target {} " + "generate {} files: {}".format( From 108a66cefe3206ba1a15eac4b9dcc586b649aa0b Mon Sep 17 00:00:00 2001 From: honglooker Date: Wed, 11 Jun 2025 18:11:14 -0400 Subject: [PATCH 25/43] docs: fix typo in toolchains.md example code (#2970) Added missing commas in `local toolchains` instructions --- docs/toolchains.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/toolchains.md b/docs/toolchains.md index 57d43d27f1..be85a2471e 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -436,8 +436,8 @@ local_runtime_repo = use_repo_rule( ) local_runtime_toolchains_repo = use_repo_rule( - "@rules_python//python/local_toolchains:repos.bzl" - "local_runtime_toolchains_repo" + "@rules_python//python/local_toolchains:repos.bzl", + "local_runtime_toolchains_repo", dev_dependency = True, ) From ef14ae2143a3707da1b1c865a7b451b154df5353 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 11 Jun 2025 16:43:26 -0700 Subject: [PATCH 26/43] chore: prepare for 1.5 release (#2973) Update version markers with upcoming version. --- CHANGELOG.md | 14 +++++++------- .../rules_python/python/config_settings/index.md | 2 +- docs/environment-variables.md | 2 +- docs/toolchains.md | 2 +- python/features.bzl | 2 +- python/private/local_runtime_toolchains_repo.bzl | 6 +++--- python/private/py_info.bzl | 2 +- python/private/py_library.bzl | 2 +- python/private/py_runtime_info.bzl | 2 +- python/private/py_runtime_rule.bzl | 2 +- python/private/pypi/env_marker_info.bzl | 2 +- python/private/python.bzl | 10 +++++----- 12 files changed, 24 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8fa1751c2..57001ca44f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,12 +47,12 @@ BEGIN_UNRELEASED_TEMPLATE END_UNRELEASED_TEMPLATE --> -{#v0-0-0} -## Unreleased +{#1-5-0} +## [1.5.0] - 2025-06-11 -[0.0.0]: https://github.com/bazel-contrib/rules_python/releases/tag/0.0.0 +[1.5.0]: https://github.com/bazel-contrib/rules_python/releases/tag/1.5.0 -{#v0-0-0-changed} +{#1-5-0-changed} ### Changed * (rules) On Windows, {obj}`--bootstrap_impl=system_python` is forced. This @@ -66,7 +66,7 @@ END_UNRELEASED_TEMPLATE `PyInfo.site_packages_symlinks` * (deps) Updating setuptools to patch CVE-2025-47273. -{#v0-0-0-fixed} +{#1-5-0-fixed} ### Fixed * (rules) PyInfo provider is now advertised by py_test, py_binary, and py_library; @@ -93,7 +93,7 @@ END_UNRELEASED_TEMPLATE by platform even though the same version is used. Fixes [#2648](https://github.com/bazel-contrib/rules_python/issues/2648). * (pypi) `compile_pip_requirements` test rule works behind the proxy -{#v0-0-0-added} +{#1-5-0-added} ### Added * Repo utilities `execute_unchecked`, `execute_checked`, and `execute_checked_stdout` now support `log_stdout` and `log_stderr` keyword arg booleans. When these are `True` @@ -115,7 +115,7 @@ END_UNRELEASED_TEMPLATE Useful when an intermediate dependency needs to be upgraded to pull in security patches. -{#v0-0-0-removed} +{#1-5-0-removed} ### Removed * Nothing removed. diff --git a/docs/api/rules_python/python/config_settings/index.md b/docs/api/rules_python/python/config_settings/index.md index 7fe25888dd..989ebf1128 100644 --- a/docs/api/rules_python/python/config_settings/index.md +++ b/docs/api/rules_python/python/config_settings/index.md @@ -167,7 +167,7 @@ Default: `//python/config_settings:_pip_env_marker_default_config` This flag points to a target providing {obj}`EnvMarkerInfo`, which determines the values used when environment markers are resolved at build time. -:::{versionadded} VERSION_NEXT_FEATURE +:::{versionadded} 1.5.0 ::: :::: diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 26c171095d..8a51bcbfd2 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -65,7 +65,7 @@ The default became `1` if unspecified When `1`, the rules_python Starlark implementation of the pypi/pip integration is used instead of the legacy Python scripts. -:::{versionadded} VERSION_NEXT_FEATURE +:::{versionadded} 1.5.0 ::: :::: diff --git a/docs/toolchains.md b/docs/toolchains.md index be85a2471e..668a458156 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -305,7 +305,7 @@ that can be used with `target_settings`. Some particular ones of note are: {flag}`--py_linux_libc` and {flag}`--py_freethreaded`, among others. ::: -:::{versionadded} VERSION_NEXT_FEATURE +:::{versionadded} 1.5.0 Added support for custom platform names, `target_compatible_with`, and `target_settings` with `single_version_platform_override`. ::: diff --git a/python/features.bzl b/python/features.bzl index b678a45241..e3d1ffdf61 100644 --- a/python/features.bzl +++ b/python/features.bzl @@ -35,7 +35,7 @@ def _features_typedef(): True if the `PyInfo.venv_symlinks` field is available. - :::{versionadded} VERSION_NEXT_FEATURE + :::{versionadded} 1.5.0 ::: :::: diff --git a/python/private/local_runtime_toolchains_repo.bzl b/python/private/local_runtime_toolchains_repo.bzl index 004ca664ad..8ef5ee9728 100644 --- a/python/private/local_runtime_toolchains_repo.bzl +++ b/python/private/local_runtime_toolchains_repo.bzl @@ -96,7 +96,7 @@ conditions are met, typically values from `@platforms`. See the [Local toolchains] docs for examples and further information. -:::{versionadded} VERSION_NEXT_FEATURE +:::{versionadded} 1.5.0 ::: """, ), @@ -145,7 +145,7 @@ The `target_settings` attribute, which handles `config_setting` values, instead of constraints. ::: -:::{versionadded} VERSION_NEXT_FEATURE +:::{versionadded} 1.5.0 ::: """, ), @@ -183,7 +183,7 @@ The `target_compatible_with` attribute, which handles *constraint* values, instead of `config_settings`. ::: -:::{versionadded} VERSION_NEXT_FEATURE +:::{versionadded} 1.5.0 ::: """, ), diff --git a/python/private/py_info.bzl b/python/private/py_info.bzl index 17c5e4e79e..31df5cfbde 100644 --- a/python/private/py_info.bzl +++ b/python/private/py_info.bzl @@ -312,7 +312,7 @@ This field is currently unused in Bazel and may go away in the future. :::{include} /_includes/experimental_api.md ::: -:::{versionadded} VERSION_NEXT_FEATURE +:::{versionadded} 1.5.0 ::: """, }, diff --git a/python/private/py_library.bzl b/python/private/py_library.bzl index e727694b32..24adb5f3ca 100644 --- a/python/private/py_library.bzl +++ b/python/private/py_library.bzl @@ -97,7 +97,7 @@ This attributes populates {obj}`PyInfo.venv_symlinks`. :::{versionadded} 1.4.0 ::: -:::{versionchanged} VERSION_NEXT_FEATURE +:::{versionchanged} 1.5.0 The topological order has been removed and if 2 different versions of the same PyPI package are observed, the behaviour has no guarantees except that it is deterministic and that only one package version will be included. diff --git a/python/private/py_runtime_info.bzl b/python/private/py_runtime_info.bzl index d2ae17e360..efe14b2c06 100644 --- a/python/private/py_runtime_info.bzl +++ b/python/private/py_runtime_info.bzl @@ -334,7 +334,7 @@ to meet two criteria: interpreter. This typically requires the Python version to be known at build-time and match at runtime. -:::{versionadded} VERSION_NEXT_FEATURE +:::{versionadded} 1.5.0 ::: """, "zip_main_template": """ diff --git a/python/private/py_runtime_rule.bzl b/python/private/py_runtime_rule.bzl index 6dadcfeac3..861014e117 100644 --- a/python/private/py_runtime_rule.bzl +++ b/python/private/py_runtime_rule.bzl @@ -360,7 +360,7 @@ Whether this runtime supports virtualenvs created at build time. See {obj}`PyRuntimeInfo.supports_build_time_venv` for docs. -:::{versionadded} VERSION_NEXT_FEATURE +:::{versionadded} 1.5.0 ::: """, default = True, diff --git a/python/private/pypi/env_marker_info.bzl b/python/private/pypi/env_marker_info.bzl index b483436d98..c3c5ec69ed 100644 --- a/python/private/pypi/env_marker_info.bzl +++ b/python/private/pypi/env_marker_info.bzl @@ -8,7 +8,7 @@ The values to use during environment marker evaluation. The {obj}`--//python/config_settings:pip_env_marker_config` flag. ::: -:::{versionadded} VERSION_NEXT_FEATURE +:::{versionadded} 1.5.0 """, fields = { "env": """ diff --git a/python/private/python.bzl b/python/private/python.bzl index 8e23668879..6eb8a3742e 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -1240,7 +1240,7 @@ The values should be one of the values in `@platforms//cpu` Docs for [Registering custom runtimes] ::: -:::{{versionadded}} VERSION_NEXT_FEATURE +:::{{versionadded}} 1.5.0 ::: """, ), @@ -1265,7 +1265,7 @@ The values should be one of the values in `@platforms//os` Docs for [Registering custom runtimes] ::: -:::{{versionadded}} VERSION_NEXT_FEATURE +:::{{versionadded}} 1.5.0 ::: """, ), @@ -1288,7 +1288,7 @@ Other values are allowed, in which case, `target_compatible_with`, `target_settings`, `os_name`, and `arch` should be specified so the toolchain is only used when appropriate. -:::{{versionchanged}} VERSION_NEXT_FEATURE +:::{{versionchanged}} 1.5.0 Arbitrary platform strings allowed. ::: """.format( @@ -1320,7 +1320,7 @@ If set, `target_settings`, `os_name`, and `arch` should also be set. Docs for [Registering custom runtimes] ::: -:::{{versionadded}} VERSION_NEXT_FEATURE +:::{{versionadded}} 1.5.0 ::: """, ), @@ -1334,7 +1334,7 @@ If set, `target_compatible_with`, `os_name`, and `arch` should also be set. Docs for [Registering custom runtimes] ::: -:::{{versionadded}} VERSION_NEXT_FEATURE +:::{{versionadded}} 1.5.0 ::: """, ), From 9b8f6501e8b814b4120ff23d787f2cb7ba8422c6 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 12 Jun 2025 11:51:50 +0900 Subject: [PATCH 27/43] fix: support pre-release versions and add new toolchain versions (#2969) Add latest toolchain builds and attempt adding a beta build. This shows/tests that we can handle pre-release versions just fine and we are able to test the toolchain matching. Whilst at it it implements the static advertising of the remaining interpreter information. Fixes #2837 --------- Co-authored-by: Richard Levasseur --- CHANGELOG.md | 9 + .../private/hermetic_runtime_repo_setup.bzl | 10 ++ python/versions.bzl | 160 ++++++++++++++++-- tests/python/python_tests.bzl | 25 +-- tests/toolchains/defs.bzl | 10 +- tests/toolchains/python_toolchain_test.py | 13 +- .../transitions/transitions_tests.bzl | 17 +- 7 files changed, 209 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57001ca44f..488f1054a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,12 @@ END_UNRELEASED_TEMPLATE {#1-5-0-changed} ### Changed +* (toolchain) Bundled toolchain version updates: + * 3.9 now references 3.9.23 + * 3.10 now references 3.10.18 + * 3.11 now references 3.11.13 + * 3.12 now references 3.12.11 + * 3.13 now references 3.13.4 * (rules) On Windows, {obj}`--bootstrap_impl=system_python` is forced. This allows setting `--bootstrap_impl=script` in bazelrc for mixed-platform environments. @@ -92,6 +98,8 @@ END_UNRELEASED_TEMPLATE * (pypi) Correctly aggregate the sources when the hashes specified in the lockfile differ by platform even though the same version is used. Fixes [#2648](https://github.com/bazel-contrib/rules_python/issues/2648). * (pypi) `compile_pip_requirements` test rule works behind the proxy +* (toolchains) The hermetic toolchains now correctly statically advertise the + `releaselevel` and `serial` for pre-release hermetic toolchains ({gh-issue}`2837`). {#1-5-0-added} ### Added @@ -114,6 +122,7 @@ END_UNRELEASED_TEMPLATE * (rules) Added support for a using constraints files with `compile_pip_requirements`. Useful when an intermediate dependency needs to be upgraded to pull in security patches. +* (toolchains): 3.14.0b2 has been added as a preview. {#1-5-0-removed} ### Removed diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index f944b0b914..98adba51d0 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -195,6 +195,14 @@ def define_hermetic_runtime_toolchain_impl( values = {"collect_code_coverage": "true"}, visibility = ["//visibility:private"], ) + if not version_info.pre: + releaselevel = "final" + else: + releaselevel = { + "a": "alpha", + "b": "beta", + "rc": "candidate", + }.get(version_info.pre[0]) py_runtime( name = "py3_runtime", @@ -204,6 +212,8 @@ def define_hermetic_runtime_toolchain_impl( "major": str(version_info.release[0]), "micro": str(version_info.release[2]), "minor": str(version_info.release[1]), + "releaselevel": releaselevel, + "serial": str(version_info.pre[1]) if version_info.pre else "0", }, coverage_tool = select({ # Convert empty string to None diff --git a/python/versions.bzl b/python/versions.bzl index e712a2e126..44af7baf69 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -186,6 +186,21 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.9.23": { + "url": "20250610/cpython-{python_version}+20250610-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "f1a60528b6088ee8b8a34ca0e960998f4f664bed300ec0bbfe9d66ccbda74e50", + "aarch64-unknown-linux-gnu": "2871cf240bce3c021de829d73da04026febd7a775d1a1a1b37603ec6419fb6c1", + "ppc64le-unknown-linux-gnu": "2ba44a8e084a4661dbe50c0f0e3cf0a57227c6f1cff13fc2ae2f4d8ceae699fc", + "riscv64-unknown-linux-gnu": "7a735aebfc8b19a8af1f03e28babaf18a46cf8db0a931343dac1269376a1f693", + "s390x-unknown-linux-gnu": "27cfc030f782e2683c664e41dcef36051467c98676e133cbef04d4b7155ac4aa", + "x86_64-apple-darwin": "debd576badb6fdabb793ec9956512102f5a813c837449b1fe007c0af977db36c", + "x86_64-pc-windows-msvc": "28fbf2026929e00a300466220917c7029a69331700badb34b1691f1a99aa38e3", + "x86_64-unknown-linux-gnu": "21440e51aee78f3d92faf9375a90713542d8332e83d94c284f8f3d52c58eb5ca", + "x86_64-unknown-linux-musl": "7a881405a41cb4edf8c0d7c469c2f4759f601bc6f3c47978424a1ab1d0f1fada", + }, + "strip_prefix": "python", + }, "3.10.2": { "url": "20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz", "sha256": { @@ -321,6 +336,21 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.10.18": { + "url": "20250610/cpython-{python_version}+20250610-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "a6590f71f670c7d121ac4f068dc83e271cf03309b80b1fa5890ee4875b7b691d", + "aarch64-unknown-linux-gnu": "b4d7cfb2cb5163da1ae5955ae8b33ac0b356780483d2993099899cf59efaea70", + "ppc64le-unknown-linux-gnu": "36aeae5cc61ff07c78b061f1b6aac628998a380ad45fadc82b8764185544fd7f", + "riscv64-unknown-linux-gnu": "2f6dd270598b655db5da5d98d1c43e560f6fb46c67a8fd68ff9b11ee9f6d79ff", + "s390x-unknown-linux-gnu": "616e56fe69c97a1d0ff13c00f337b2a91c972323c5d9a1828fdfc4d764b440fa", + "x86_64-apple-darwin": "4d72c1c1dcd2c4fe80055ef1b24fe4146f2de938aea1e3676faf91476f3f17e8", + "x86_64-pc-windows-msvc": "867b6dbcdb71d8ebb709ff54fbca8ad43d05cc21e5c157f39745c4dc44c1f8e2", + "x86_64-unknown-linux-gnu": "58f88ed6117078fdbc98976c9bc83b918f1f9c0c2ec21b80a582104f4839861c", + "x86_64-unknown-linux-musl": "d782c0569d6d7e21a5ed195ad7b41d0af8456b031e0814714d18cdeaa876f262", + }, + "strip_prefix": "python", + }, "3.11.1": { "url": "20230116/cpython-{python_version}+20230116-{platform}-{build}.tar.gz", "sha256": { @@ -436,18 +466,18 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, - "3.11.11": { - "url": "20250317/cpython-{python_version}+20250317-{platform}-{build}.tar.gz", + "3.11.13": { + "url": "20250610/cpython-{python_version}+20250610-{platform}-{build}.tar.gz", "sha256": { - "aarch64-apple-darwin": "19b147c7e4b742656da4cb6ba35bc3ea2f15aa5f4d1bbbc38d09e2e85551e927", - "aarch64-unknown-linux-gnu": "7d52b5206afe617de2899af477f5a1d275ecbce80fb8300301b254ebf1da5a90", - "ppc64le-unknown-linux-gnu": "17c049f70ce719adc89dd0ae26f4e6a28f6aaedc63c2efef6bbb9c112ea4d692", - "riscv64-unknown-linux-gnu": "83ed50713409576756f5708e8f0549a15c17071bea22b71f15e11a7084f09481", - "s390x-unknown-linux-gnu": "298507f1f8d962b1bb98cb506c99e7e0d291a63eb9117e1521141e6b3825fd56", - "x86_64-apple-darwin": "a870cd965e7dded5100d13b1d34cab1c32a92811e000d10fbfe9bbdb36cdaa0e", - "x86_64-pc-windows-msvc": "1cf5760eea0a9df3308ca2c4111b5cc18fd638b2a912dbe07606193e3f9aa123", - "x86_64-unknown-linux-gnu": "51e47bc0d1b9f4bf68dd395f7a39f60c58a87cde854cab47264a859eb666bb69", - "x86_64-unknown-linux-musl": "ee4d84f992c6a1df42096e26b970fe5938fd6c1eadd245894bc94c5737ff9977", + "aarch64-apple-darwin": "365037494ba4f53563c22292e49a8e4d0d495bcb6534fca9666bdd1b474abf36", + "aarch64-unknown-linux-gnu": "a5954f147e87d9bff3d9733ebb3e74fe997eec5b38eaf5cb4429038228962a16", + "ppc64le-unknown-linux-gnu": "9214126866418f290fda88832fa3e244630f918ebc8a4a9ee15ba922e9c98afd", + "riscv64-unknown-linux-gnu": "fd99008c3123f50ec2ad407c5c1e17c1a86590daaf88dae8e6f1fd28f099b7c2", + "s390x-unknown-linux-gnu": "e27ab1fff8bf9e507677252a03ed524c685a8629b56475e26ab6dd0f88465179", + "x86_64-apple-darwin": "b49044115a545e67d73f5265a613a25da7c9523431281aa7b94691f1013355af", + "x86_64-pc-windows-msvc": "c0f89e3776211147817d54084fa046e2603571e18ff2ae4a4a8ff84ca4f7defc", + "x86_64-unknown-linux-gnu": "d93a7699505ee0ac7dec0f09324ffb19a31cce3066a287bb1fe95285ce3ea0c7", + "x86_64-unknown-linux-musl": "499121bb917e5baeeb954f76bdbce36bb63af579ff1530966ae2280e8d812c5b", }, "strip_prefix": "python", }, @@ -559,6 +589,21 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.12.11": { + "url": "20250610/cpython-{python_version}+20250610-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "9c5826a93ddc15e8aa08de1e6e65b3ae0d45ea8eb0c2e9547b80ff4121b870ce", + "aarch64-unknown-linux-gnu": "eb33bc5a87443daf2fd218109df811bc4e4ea5ef9aec4fad75aa55da0258b96f", + "ppc64le-unknown-linux-gnu": "7b90bc528c5ddf30579dec52926d68fa6d5c90b65e24fc185d5fe283fdf0cbd9", + "riscv64-unknown-linux-gnu": "0f3103675102e351762a8fe574eae20335552a246a45a006d2a9ca14ce0952f8", + "s390x-unknown-linux-gnu": "a7ff0432208450ccebd5d328f69b84cc7c25b4af54fbab44803ddb11a2da5028", + "x86_64-apple-darwin": "199631baa35f3747ddfa2f1e28fc062b97ccd15b94a60c9294d4d129a73c9e53", + "x86_64-pc-windows-msvc": "e05fa165841c416d60365ca2216cad570f05ae5d3d027b9ad3beaad0529dd8cc", + "x86_64-unknown-linux-gnu": "77ab3efe5c6637fe8da0fdfbff5de1730c3b824874fe1368917886908b4c517b", + "x86_64-unknown-linux-musl": "9dd768494c4a34abcec316bc4802e957db98ed283024b527c0c40dfefd08b6fe", + }, + "strip_prefix": "python", + }, "3.13.0": { "url": "20241016/cpython-{python_version}+20241016-{platform}-{build}.{ext}", "sha256": { @@ -674,16 +719,99 @@ TOOL_VERSIONS = { "x86_64-unknown-linux-gnu-freethreaded": "python/install", }, }, + "3.13.4": { + "url": "20250610/cpython-{python_version}+20250610-{platform}-{build}.{ext}", + "sha256": { + "aarch64-apple-darwin": "c2ce6601b2668c7bd1f799986af5ddfbff36e88795741864aba6e578cb02ed7f", + "aarch64-unknown-linux-gnu": "3c2596ece08ffe17e11bc1f27aeb4ce1195d2490a83d695d36ef4933d5c5ca53", + "ppc64le-unknown-linux-gnu": "b3cc13ee177b8db1d3e9b2eac413484e3c6a356f97d91dc59de8d3fd8cf79d6b", + "riscv64-unknown-linux-gnu": "d1b989e57a9ce29f6c945eeffe0e9750c222fdd09e99d2f8d6b0d8532a523053", + "s390x-unknown-linux-gnu": "d1d19fb01961ac6476712fdd6c5031f74c83666f6f11aa066207e9a158f7e3d8", + "x86_64-apple-darwin": "79feb6ca68f3921d07af52d9db06cf134e6f36916941ea850ab0bc20f5ff638b", + "x86_64-pc-windows-msvc": "29ac3585cc2dcfd79e3fe380c272d00e9d34351fc456e149403c86d3fea34057", + "x86_64-unknown-linux-gnu": "44e5477333ebca298a7a0a316985c6c3533b8645f92a83f7f73c44033832bf32", + "x86_64-unknown-linux-musl": "a3afbfa94b9ff4d9fc426b47eb3c8446cada535075b8d51b7bdc9d9ab9911fc2", + "aarch64-apple-darwin-freethreaded": "278dccade56b4bbeecb9a613b77012cf5c1433a5e9b8ef99230d5e61f31d9e02", + "aarch64-unknown-linux-gnu-freethreaded": "b1c1bd6ab9ef95b464d92a6a911cef1a8d9f0b0f6a192f694ef18ed15d882edf", + "ppc64le-unknown-linux-gnu-freethreaded": "ed66ae213a62b286b9b7338b816ccd2815f5248b7a28a185dc8159fe004149ae", + "riscv64-unknown-linux-gnu-freethreaded": "913264545215236660e4178bc3e5b57a20a444a8deb5c11680c95afc960b4016", + "s390x-unknown-linux-gnu-freethreaded": "7556a38ab5e507c1ec22bc38f9859982bc956cab7f4de05a2faac114feb306db", + "x86_64-apple-darwin-freethreaded": "64ab7ac8c88002d9ba20a92f72945bfa350268e944a7922500af75d20330574d", + "x86_64-pc-windows-msvc-freethreaded": "9457504547edb2e0156bf76b53c7e4941c7f61c0eff9fd5f4d816d3df51c58e3", + "x86_64-unknown-linux-gnu-freethreaded": "864df6e6819e8f8e855ce30f34410fdc5867d0616e904daeb9a40e5806e970d7", + }, + "strip_prefix": { + "aarch64-apple-darwin": "python", + "aarch64-unknown-linux-gnu": "python", + "ppc64le-unknown-linux-gnu": "python", + "s390x-unknown-linux-gnu": "python", + "riscv64-unknown-linux-gnu": "python", + "x86_64-apple-darwin": "python", + "x86_64-pc-windows-msvc": "python", + "x86_64-unknown-linux-gnu": "python", + "x86_64-unknown-linux-musl": "python", + "aarch64-apple-darwin-freethreaded": "python/install", + "aarch64-unknown-linux-gnu-freethreaded": "python/install", + "ppc64le-unknown-linux-gnu-freethreaded": "python/install", + "riscv64-unknown-linux-gnu-freethreaded": "python/install", + "s390x-unknown-linux-gnu-freethreaded": "python/install", + "x86_64-apple-darwin-freethreaded": "python/install", + "x86_64-pc-windows-msvc-freethreaded": "python/install", + "x86_64-unknown-linux-gnu-freethreaded": "python/install", + }, + }, + "3.14.0b2": { + "url": "20250610/cpython-{python_version}+20250610-{platform}-{build}.{ext}", + "sha256": { + "aarch64-apple-darwin": "6607351d140e83feb6e11dbde46ab5f99fa9fe039bdbaa12611d26bda0ed9343", + "aarch64-unknown-linux-gnu": "cc388d567f7c23921e0bef8dcae959dfab9ee24d10aeeb23688b21eac402817f", + "ppc64le-unknown-linux-gnu": "f9379ecc5dc71f9c58adf03d5524176ec36e1b40c788d29c260df54d09ad351c", + "riscv64-unknown-linux-gnu": "e6fbe4f7928ec606edee1506752659bf59216fdb208c744d268082ec79b16f42", + "s390x-unknown-linux-gnu": "1cf32c1173adc1cb70952bb47c92177a196f9e83b7a874f09599682e92ba0010", + "x86_64-apple-darwin": "a6d8196b174409e0ce67829c4e4ee5005c4be20a2efb41116e0521ad1fa1a717", + "x86_64-pc-windows-msvc": "0d88ec80c6c3e3ac462368850c19d3930bf2b1a1a5fe89da60c8534d0fac1a01", + "x86_64-unknown-linux-gnu": "93b29eea5214d19f0420ef8e459b007e15ea58349d60811122c78241fe51cb92", + "x86_64-unknown-linux-musl": "90e90a58ebff3416eb5a3f93ecb59b6eda945e2b706f5c13b0ba85f6b2bee130", + "aarch64-apple-darwin-freethreaded": "af0f34aa0dcd02bd3d960a1572a1ed8a17d55b373a22866f05041aaf16f8607d", + "aarch64-unknown-linux-gnu-freethreaded": "e76c7ab98e1c0f86a6996d1ec775ba8497bf46aa8ffa8c7b0f2e761f37305329", + "ppc64le-unknown-linux-gnu-freethreaded": "df2ae00827406e247f1aaaec76ffc7963b909c81075fc9940eee1ea9f753dd16", + "riscv64-unknown-linux-gnu-freethreaded": "09e347cb5f29e0eafd1eba73105ea9d853184b55fbaf4746cebec217430d6db5", + "s390x-unknown-linux-gnu-freethreaded": "f911605eee0eb7845a69acaf8bfb2e1811c76e9a5e3980d97fae93135df4b773", + "x86_64-apple-darwin-freethreaded": "dd27d519cf2a04917cb566366d6539477791d1b2f1fb42037d9179f469ff55a9", + "x86_64-pc-windows-msvc-freethreaded": "da966a17e434094d8f10b719d93c782d82eaf5207f2843cbaa58c3d91a8f0e32", + "x86_64-unknown-linux-gnu-freethreaded": "abd60d3a302e9d9c32ec78581fb3a9903079c56ec7a949ce658a7950423f350a", + }, + "strip_prefix": { + "aarch64-apple-darwin": "python", + "aarch64-unknown-linux-gnu": "python", + "ppc64le-unknown-linux-gnu": "python", + "s390x-unknown-linux-gnu": "python", + "riscv64-unknown-linux-gnu": "python", + "x86_64-apple-darwin": "python", + "x86_64-pc-windows-msvc": "python", + "x86_64-unknown-linux-gnu": "python", + "x86_64-unknown-linux-musl": "python", + "aarch64-apple-darwin-freethreaded": "python/install", + "aarch64-unknown-linux-gnu-freethreaded": "python/install", + "ppc64le-unknown-linux-gnu-freethreaded": "python/install", + "riscv64-unknown-linux-gnu-freethreaded": "python/install", + "s390x-unknown-linux-gnu-freethreaded": "python/install", + "x86_64-apple-darwin-freethreaded": "python/install", + "x86_64-pc-windows-msvc-freethreaded": "python/install", + "x86_64-unknown-linux-gnu-freethreaded": "python/install", + }, + }, } # buildifier: disable=unsorted-dict-items MINOR_MAPPING = { "3.8": "3.8.20", - "3.9": "3.9.21", - "3.10": "3.10.16", - "3.11": "3.11.11", - "3.12": "3.12.9", - "3.13": "3.13.2", + "3.9": "3.9.23", + "3.10": "3.10.18", + "3.11": "3.11.13", + "3.12": "3.12.11", + "3.13": "3.13.4", + "3.14": "3.14.0b2", } def _generate_platforms(): diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index 116afa76ad..f0dc4825ac 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -304,11 +304,11 @@ def _test_toolchain_ordering(env): toolchain = [ _toolchain("3.10"), _toolchain("3.10.15"), - _toolchain("3.10.16"), - _toolchain("3.10.11"), + _toolchain("3.10.18"), + _toolchain("3.10.13"), _toolchain("3.11.1"), _toolchain("3.11.10"), - _toolchain("3.11.11", is_default = True), + _toolchain("3.11.13", is_default = True), ], ), _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), @@ -320,14 +320,15 @@ def _test_toolchain_ordering(env): for t in py.toolchains ] - env.expect.that_str(py.default_python_version).equals("3.11.11") + env.expect.that_str(py.default_python_version).equals("3.11.13") env.expect.that_dict(py.config.minor_mapping).contains_exactly({ - "3.10": "3.10.16", - "3.11": "3.11.11", - "3.12": "3.12.9", - "3.13": "3.13.2", + "3.10": "3.10.18", + "3.11": "3.11.13", + "3.12": "3.12.11", + "3.13": "3.13.4", + "3.14": "3.14.0b2", "3.8": "3.8.20", - "3.9": "3.9.21", + "3.9": "3.9.23", }) env.expect.that_collection(got_versions).contains_exactly([ # First the full-version toolchains that are in minor_mapping @@ -336,13 +337,13 @@ def _test_toolchain_ordering(env): # The default version is always set in the `python_version` flag, so know, that # the default match will be somewhere in the first bunch. "3.10", - "3.10.16", + "3.10.18", "3.11", - "3.11.11", + "3.11.13", # Next, the rest, where we will match things based on the `python_version` being # the same "3.10.15", - "3.10.11", + "3.10.13", "3.11.1", "3.11.10", ]).in_order() diff --git a/tests/toolchains/defs.bzl b/tests/toolchains/defs.bzl index a883b0af33..25863d18c4 100644 --- a/tests/toolchains/defs.bzl +++ b/tests/toolchains/defs.bzl @@ -15,6 +15,7 @@ "" load("//python:versions.bzl", "PLATFORMS", "TOOL_VERSIONS") +load("//python/private:version.bzl", "version") # buildifier: disable=bzl-visibility load("//tests/support:py_reconfig.bzl", "py_reconfig_test") def define_toolchain_tests(name): @@ -38,13 +39,20 @@ def define_toolchain_tests(name): is_platform = "_is_{}".format(platform_key) target_compatible_with[is_platform] = [] + parsed = version.parse(python_version, strict = True) + expect_python_version = "{0}.{1}.{2}".format(*parsed.release) + if parsed.pre: + expect_python_version = "{0}{1}{2}".format( + expect_python_version, + *parsed.pre + ) py_reconfig_test( name = "python_{}_test".format(python_version), srcs = ["python_toolchain_test.py"], main = "python_toolchain_test.py", python_version = python_version, env = { - "EXPECT_PYTHON_VERSION": python_version, + "EXPECT_PYTHON_VERSION": expect_python_version, }, deps = ["//python/runfiles"], data = ["//tests/support:current_build_settings"], diff --git a/tests/toolchains/python_toolchain_test.py b/tests/toolchains/python_toolchain_test.py index 591d7dbe8a..63ed42488f 100644 --- a/tests/toolchains/python_toolchain_test.py +++ b/tests/toolchains/python_toolchain_test.py @@ -27,7 +27,18 @@ def test_expected_toolchain_matches(self): ) self.assertIn(expected, settings["toolchain_label"], msg) - actual = "{v.major}.{v.minor}.{v.micro}".format(v=sys.version_info) + if sys.version_info.releaselevel == "final": + actual = "{v.major}.{v.minor}.{v.micro}".format(v=sys.version_info) + elif sys.version_info.releaselevel in ["beta"]: + actual = ( + "{v.major}.{v.minor}.{v.micro}{v.releaselevel[0]}{v.serial}".format( + v=sys.version_info + ) + ) + else: + raise NotImplementedError( + "Unsupported release level, please update the test" + ) self.assertEqual(actual, expect_version) diff --git a/tests/toolchains/transitions/transitions_tests.bzl b/tests/toolchains/transitions/transitions_tests.bzl index bddd1745f0..ef071188bb 100644 --- a/tests/toolchains/transitions/transitions_tests.bzl +++ b/tests/toolchains/transitions/transitions_tests.bzl @@ -56,14 +56,21 @@ def _impl(ctx): exec_tools = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN_TYPE].exec_tools got_version = exec_tools.exec_interpreter[platform_common.ToolchainInfo].py3_runtime.interpreter_version_info + got = "{}.{}.{}".format( + got_version.major, + got_version.minor, + got_version.micro, + ) + if got_version.releaselevel != "final": + got = "{}{}{}".format( + got, + got_version.releaselevel[0], + got_version.serial, + ) return [ TestInfo( - got = "{}.{}.{}".format( - got_version.major, - got_version.minor, - got_version.micro, - ), + got = got, want = ctx.attr.want_version, ), ] From e225a1eddd6055b08cb832f7d4e73922d5f7d956 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Wed, 11 Jun 2025 22:27:04 -0700 Subject: [PATCH 28/43] chore: Fixup some typos in BuildKite job names (#2977) --- .bazelci/presubmit.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 7e9d4dea53..01af217924 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -90,7 +90,7 @@ tasks: gazelle_extension_min: <<: *common_workspace_flags_min_bazel <<: *minimum_supported_version - name: "Gazelle: workspace, minumum supported Bazel version" + name: "Gazelle: workspace, minimum supported Bazel version" platform: ubuntu2004 build_targets: ["//..."] test_targets: ["//..."] @@ -338,7 +338,7 @@ tasks: integration_test_bzlmod_build_file_generation_windows: <<: *reusable_build_test_all # coverage is not supported on Windows - name: "examples/bzlmod_build_file_generateion: Windows" + name: "examples/bzlmod_build_file_generation: Windows" working_directory: examples/bzlmod_build_file_generation platform: windows From f2fa07a56f575028cd84d4d4d169b734507c34d7 Mon Sep 17 00:00:00 2001 From: John Cater Date: Fri, 13 Jun 2025 12:31:51 -0400 Subject: [PATCH 29/43] refactor: Remove unused CC_TOOLCHAIN definition (#2981) Fixes #2979. The definition appears unused and helps advance the goal of entirely removing current_cc_toolchain: see https://github.com/bazelbuild/bazel/issues/26282. --- python/private/attributes.bzl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/python/private/attributes.bzl b/python/private/attributes.bzl index c3b1cade91..641fa13a23 100644 --- a/python/private/attributes.bzl +++ b/python/private/attributes.bzl @@ -156,12 +156,6 @@ def copy_common_test_kwargs(kwargs): if key in kwargs } -CC_TOOLCHAIN = { - # NOTE: The `cc_helper.find_cpp_toolchain()` function expects the attribute - # name to be this name. - "_cc_toolchain": attr.label(default = "@bazel_tools//tools/cpp:current_cc_toolchain"), -} - # The common "data" attribute definition. DATA_ATTRS = { # NOTE: The "flags" attribute is deprecated, but there isn't an alternative From 94e08f7dfe61962fa50508f01ea05c624307d487 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 13 Jun 2025 19:20:26 -0700 Subject: [PATCH 30/43] Fix argument name typo (#2984) ``` ERROR: Traceback (most recent call last): File ".../rules_python++pip+rules_mypy_pip_312_click/BUILD.bazel", line 5, column 20, in whl_library_targets( File ".../rules_python+/python/private/pypi/whl_library_targets.bzl", line 337, column 53, in whl_library_targets "//conditions:default": create_inits( File ".../rules_python+/python/private/pypi/namespace_pkgs.bzl", line 72, column 25, in create_inits for out in get_files(**kwargs): File ".../rules_python+/python/private/pypi/namespace_pkgs.bzl", line 20, column 5, in get_files def get_files(*, srcs, ignored_dirnames = [], root = None): Error: get_files() got unexpected keyword argument: ignore_dirnames (did you mean 'ignored_dirnames'?) ``` --- python/private/pypi/whl_library_targets.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 3529566c49..518d17163f 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -336,7 +336,7 @@ def whl_library_targets( Label("//python/config_settings:is_venvs_site_packages"): [], "//conditions:default": create_inits( srcs = srcs + data + pyi_srcs, - ignore_dirnames = [], # If you need to ignore certain folders, you can patch rules_python here to do so. + ignored_dirnames = [], # If you need to ignore certain folders, you can patch rules_python here to do so. root = "site-packages", ), }) From ca235368d04fb0ebf39fc9174acfd883ca3e3675 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Jun 2025 16:23:13 +0900 Subject: [PATCH 31/43] build(deps): bump certifi from 2025.1.31 to 2025.6.15 in /tools/publish (#2999) Bumps [certifi](https://github.com/certifi/python-certifi) from 2025.1.31 to 2025.6.15.
Commits
  • e767d59 2025.06.15 (#357)
  • 3e70765 Bump actions/setup-python from 5.5.0 to 5.6.0
  • 9afd2ff Bump actions/download-artifact from 4.2.1 to 4.3.0
  • d7c816c remove code that's no longer required that 3.7 is our minimum (#351)
  • 1899613 Declare setuptools as the build backend in pyproject.toml (#350)
  • c874142 update CI for ubuntu 20.04 deprecation (#348)
  • 275c9eb 2025.04.26 (#347)
  • 3788331 Bump actions/setup-python from 5.4.0 to 5.5.0 (#346)
  • 9d1f1b7 Bump actions/download-artifact from 4.1.9 to 4.2.1 (#344)
  • 96b97a5 Bump actions/upload-artifact from 4.6.1 to 4.6.2 (#343)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=certifi&package-manager=pip&previous-version=2025.1.31&new-version=2025.6.15)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/publish/requirements_darwin.txt | 6 +++--- tools/publish/requirements_linux.txt | 6 +++--- tools/publish/requirements_universal.txt | 6 +++--- tools/publish/requirements_windows.txt | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/publish/requirements_darwin.txt b/tools/publish/requirements_darwin.txt index 483f88444e..af5bad246d 100644 --- a/tools/publish/requirements_darwin.txt +++ b/tools/publish/requirements_darwin.txt @@ -6,9 +6,9 @@ backports-tarfile==1.2.0 \ --hash=sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34 \ --hash=sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991 # via jaraco-context -certifi==2025.1.31 \ - --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ - --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe +certifi==2025.6.15 \ + --hash=sha256:2e0c7ce7cb5d8f8634ca55d2ba7e6ec2689a2fd6537d8dec1296a477a4910057 \ + --hash=sha256:d747aa5a8b9bbbb1bb8c22bb13e22bd1f18e9796defa16bab421f7f7a317323b # via requests charset-normalizer==3.4.1 \ --hash=sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537 \ diff --git a/tools/publish/requirements_linux.txt b/tools/publish/requirements_linux.txt index 62dbf1eb77..b2e9ccf5ab 100644 --- a/tools/publish/requirements_linux.txt +++ b/tools/publish/requirements_linux.txt @@ -6,9 +6,9 @@ backports-tarfile==1.2.0 \ --hash=sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34 \ --hash=sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991 # via jaraco-context -certifi==2025.1.31 \ - --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ - --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe +certifi==2025.6.15 \ + --hash=sha256:2e0c7ce7cb5d8f8634ca55d2ba7e6ec2689a2fd6537d8dec1296a477a4910057 \ + --hash=sha256:d747aa5a8b9bbbb1bb8c22bb13e22bd1f18e9796defa16bab421f7f7a317323b # via requests cffi==1.17.1 \ --hash=sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8 \ diff --git a/tools/publish/requirements_universal.txt b/tools/publish/requirements_universal.txt index e4e876b176..8a7426e517 100644 --- a/tools/publish/requirements_universal.txt +++ b/tools/publish/requirements_universal.txt @@ -6,9 +6,9 @@ backports-tarfile==1.2.0 ; python_full_version < '3.12' \ --hash=sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34 \ --hash=sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991 # via jaraco-context -certifi==2025.1.31 \ - --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ - --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe +certifi==2025.6.15 \ + --hash=sha256:2e0c7ce7cb5d8f8634ca55d2ba7e6ec2689a2fd6537d8dec1296a477a4910057 \ + --hash=sha256:d747aa5a8b9bbbb1bb8c22bb13e22bd1f18e9796defa16bab421f7f7a317323b # via requests cffi==1.17.1 ; platform_python_implementation != 'PyPy' and sys_platform == 'linux' \ --hash=sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8 \ diff --git a/tools/publish/requirements_windows.txt b/tools/publish/requirements_windows.txt index 043de9ecb1..11017aa4f9 100644 --- a/tools/publish/requirements_windows.txt +++ b/tools/publish/requirements_windows.txt @@ -6,9 +6,9 @@ backports-tarfile==1.2.0 \ --hash=sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34 \ --hash=sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991 # via jaraco-context -certifi==2025.1.31 \ - --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ - --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe +certifi==2025.6.15 \ + --hash=sha256:2e0c7ce7cb5d8f8634ca55d2ba7e6ec2689a2fd6537d8dec1296a477a4910057 \ + --hash=sha256:d747aa5a8b9bbbb1bb8c22bb13e22bd1f18e9796defa16bab421f7f7a317323b # via requests charset-normalizer==3.4.1 \ --hash=sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537 \ From 60b48e2156574ed40e24df32f7dbef59f6f6c4f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Jun 2025 17:00:40 +0900 Subject: [PATCH 32/43] build(deps): bump certifi from 2025.1.31 to 2025.6.15 in /docs (#3000) Bumps [certifi](https://github.com/certifi/python-certifi) from 2025.1.31 to 2025.6.15.
Commits
  • e767d59 2025.06.15 (#357)
  • 3e70765 Bump actions/setup-python from 5.5.0 to 5.6.0
  • 9afd2ff Bump actions/download-artifact from 4.2.1 to 4.3.0
  • d7c816c remove code that's no longer required that 3.7 is our minimum (#351)
  • 1899613 Declare setuptools as the build backend in pyproject.toml (#350)
  • c874142 update CI for ubuntu 20.04 deprecation (#348)
  • 275c9eb 2025.04.26 (#347)
  • 3788331 Bump actions/setup-python from 5.4.0 to 5.5.0 (#346)
  • 9d1f1b7 Bump actions/download-artifact from 4.1.9 to 4.2.1 (#344)
  • 96b97a5 Bump actions/upload-artifact from 4.6.1 to 4.6.2 (#343)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=certifi&package-manager=pip&previous-version=2025.1.31&new-version=2025.6.15)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 87c13aa8ba..b0a84d476b 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -17,9 +17,9 @@ babel==2.17.0 \ --hash=sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d \ --hash=sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2 # via sphinx -certifi==2025.1.31 \ - --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ - --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe +certifi==2025.6.15 \ + --hash=sha256:2e0c7ce7cb5d8f8634ca55d2ba7e6ec2689a2fd6537d8dec1296a477a4910057 \ + --hash=sha256:d747aa5a8b9bbbb1bb8c22bb13e22bd1f18e9796defa16bab421f7f7a317323b # via requests charset-normalizer==3.4.1 \ --hash=sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537 \ From be86f4acae5571c39c2d6a952e28c435cd722a91 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Jun 2025 17:38:57 +0900 Subject: [PATCH 33/43] build(deps): bump requests from 2.32.3 to 2.32.4 in /docs (#2965) Bumps [requests](https://github.com/psf/requests) from 2.32.3 to 2.32.4.
Release notes

Sourced from requests's releases.

v2.32.4

2.32.4 (2025-06-10)

Security

  • CVE-2024-47081 Fixed an issue where a maliciously crafted URL and trusted environment will retrieve credentials for the wrong hostname/machine from a netrc file. (#6965)

Improvements

  • Numerous documentation improvements

Deprecations

  • Added support for pypy 3.11 for Linux and macOS. (#6926)
  • Dropped support for pypy 3.9 following its end of support. (#6926)
Changelog

Sourced from requests's changelog.

2.32.4 (2025-06-10)

Security

  • CVE-2024-47081 Fixed an issue where a maliciously crafted URL and trusted environment will retrieve credentials for the wrong hostname/machine from a netrc file.

Improvements

  • Numerous documentation improvements

Deprecations

  • Added support for pypy 3.11 for Linux and macOS.
  • Dropped support for pypy 3.9 following its end of support.
Commits
  • 021dc72 Polish up release tooling for last manual release
  • 821770e Bump version and add release notes for v2.32.4
  • 59f8aa2 Add netrc file search information to authentication documentation (#6876)
  • 5b4b64c Add more tests to prevent regression of CVE 2024 47081
  • 7bc4587 Add new test to check netrc auth leak (#6962)
  • 96ba401 Only use hostname to do netrc lookup instead of netloc
  • 7341690 Merge pull request #6951 from tswast/patch-1
  • 6716d7c remove links
  • a7e1c74 Update docs/conf.py
  • c799b81 docs: fix dead links to kenreitz.org
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=requests&package-manager=pip&previous-version=2.32.3&new-version=2.32.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/bazel-contrib/rules_python/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index b0a84d476b..cfeb0cbf31 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -291,9 +291,9 @@ readthedocs-sphinx-ext==2.2.5 \ --hash=sha256:ee5fd5b99db9f0c180b2396cbce528aa36671951b9526bb0272dbfce5517bd27 \ --hash=sha256:f8c56184ea011c972dd45a90122568587cc85b0127bc9cf064d17c68bc809daa # via rules-python-docs (docs/pyproject.toml) -requests==2.32.3 \ - --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ - --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 +requests==2.32.4 \ + --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \ + --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422 # via # readthedocs-sphinx-ext # sphinx From 107a8781cdd207c9079ecd733c0028d2706a49f2 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 18 Jun 2025 02:44:55 +0900 Subject: [PATCH 34/43] fix: use platform_info.target_settings in toolchain aliases (#3001) During the refactor we forgot one more place where the `flag_values` on the platform information was used. They were no longer populated and broke. The solution is to use `selects.config_setting_group` to maintain behaviour and in order to smoke test I have added a target to verify that the aliases work. Related to #2875 Fixes #2993 Co-authored-by: Richard Levasseur --- python/config_settings/BUILD.bazel | 12 ++++++---- python/private/config_settings.bzl | 2 +- .../private/hermetic_runtime_repo_setup.bzl | 19 ++++++++-------- python/private/pypi/config_settings.bzl | 22 +++++++++---------- python/private/toolchain_aliases.bzl | 12 +++++++--- tests/toolchains/BUILD.bazel | 8 +++++++ 6 files changed, 47 insertions(+), 28 deletions(-) diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index b11580c4cb..82a73cee6c 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -125,15 +125,19 @@ string_flag( visibility = ["//visibility:public"], ) -config_setting( +alias( name = "is_py_freethreaded", - flag_values = {":py_freethreaded": FreeThreadedFlag.YES}, + actual = ":_is_py_freethreaded_yes", + deprecation = "not actually public, please create your own config_setting using the flag that rules_python exposes", + tags = ["manual"], visibility = ["//visibility:public"], ) -config_setting( +alias( name = "is_py_non_freethreaded", - flag_values = {":py_freethreaded": FreeThreadedFlag.NO}, + actual = ":_is_py_freethreaded_no", + deprecation = "not actually public, please create your own config_setting using the flag that rules_python exposes", + tags = ["manual"], visibility = ["//visibility:public"], ) diff --git a/python/private/config_settings.bzl b/python/private/config_settings.bzl index aff5d016fb..3089b9c6cf 100644 --- a/python/private/config_settings.bzl +++ b/python/private/config_settings.bzl @@ -143,7 +143,7 @@ def construct_config_settings(*, name, default_version, versions, minor_mapping, ) native.config_setting( name = "_is_py_linux_libc_musl", - flag_values = {libc: "glibc"}, + flag_values = {libc: "musl"}, visibility = _NOT_ACTUALLY_PUBLIC, ) freethreaded = Label("//python/config_settings:py_freethreaded") diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index 98adba51d0..6910ea14a1 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -22,7 +22,8 @@ load(":glob_excludes.bzl", "glob_excludes") load(":py_exec_tools_toolchain.bzl", "py_exec_tools_toolchain") load(":version.bzl", "version") -_IS_FREETHREADED = Label("//python/config_settings:is_py_freethreaded") +_IS_FREETHREADED_YES = Label("//python/config_settings:_is_py_freethreaded_yes") +_IS_FREETHREADED_NO = Label("//python/config_settings:_is_py_freethreaded_no") def define_hermetic_runtime_toolchain_impl( *, @@ -87,16 +88,16 @@ def define_hermetic_runtime_toolchain_impl( cc_import( name = "interface", interface_library = select({ - _IS_FREETHREADED: "libs/python{major}{minor}t.lib".format(**version_dict), - "//conditions:default": "libs/python{major}{minor}.lib".format(**version_dict), + _IS_FREETHREADED_YES: "libs/python{major}{minor}t.lib".format(**version_dict), + _IS_FREETHREADED_NO: "libs/python{major}{minor}.lib".format(**version_dict), }), system_provided = True, ) cc_import( name = "abi3_interface", interface_library = select({ - _IS_FREETHREADED: "libs/python3t.lib", - "//conditions:default": "libs/python3.lib", + _IS_FREETHREADED_YES: "libs/python3t.lib", + _IS_FREETHREADED_NO: "libs/python3.lib", }), system_provided = True, ) @@ -115,10 +116,10 @@ def define_hermetic_runtime_toolchain_impl( includes = [ "include", ] + select({ - _IS_FREETHREADED: [ + _IS_FREETHREADED_YES: [ "include/python{major}.{minor}t".format(**version_dict), ], - "//conditions:default": [ + _IS_FREETHREADED_NO: [ "include/python{major}.{minor}".format(**version_dict), "include/python{major}.{minor}m".format(**version_dict), ], @@ -224,8 +225,8 @@ def define_hermetic_runtime_toolchain_impl( implementation_name = "cpython", # See https://peps.python.org/pep-3147/ for pyc tag infix format pyc_tag = select({ - _IS_FREETHREADED: "cpython-{major}{minor}t".format(**version_dict), - "//conditions:default": "cpython-{major}{minor}".format(**version_dict), + _IS_FREETHREADED_YES: "cpython-{major}{minor}t".format(**version_dict), + _IS_FREETHREADED_NO: "cpython-{major}{minor}".format(**version_dict), }), ) diff --git a/python/private/pypi/config_settings.bzl b/python/private/pypi/config_settings.bzl index d1b85d16c1..3e828e59f5 100644 --- a/python/private/pypi/config_settings.bzl +++ b/python/private/pypi/config_settings.bzl @@ -80,8 +80,8 @@ FLAGS = struct( "is_pip_whl_auto", "is_pip_whl_no", "is_pip_whl_only", - "is_py_freethreaded", - "is_py_non_freethreaded", + "_is_py_freethreaded_yes", + "_is_py_freethreaded_no", "pip_whl_glibc_version", "pip_whl_muslc_version", "pip_whl_osx_arch", @@ -205,12 +205,12 @@ def _dist_config_settings(*, suffix, plat_flag_values, python_version, **kwargs) for name, f, compatible_with in [ ("py_none", _flags.whl, None), ("py3_none", _flags.whl_py3, None), - ("py3_abi3", _flags.whl_py3_abi3, (FLAGS.is_py_non_freethreaded,)), + ("py3_abi3", _flags.whl_py3_abi3, (FLAGS._is_py_freethreaded_no,)), ("none", _flags.whl_pycp3x, None), - ("abi3", _flags.whl_pycp3x_abi3, (FLAGS.is_py_non_freethreaded,)), + ("abi3", _flags.whl_pycp3x_abi3, (FLAGS._is_py_freethreaded_no,)), # The below are not specializations of one another, they are variants - (cpv, _flags.whl_pycp3x_abicp, (FLAGS.is_py_non_freethreaded,)), - (cpv + "t", _flags.whl_pycp3x_abicp, (FLAGS.is_py_freethreaded,)), + (cpv, _flags.whl_pycp3x_abicp, (FLAGS._is_py_freethreaded_no,)), + (cpv + "t", _flags.whl_pycp3x_abicp, (FLAGS._is_py_freethreaded_yes,)), ]: if (f, compatible_with) in used_flags: # This should never happen as all of the different whls should have @@ -237,12 +237,12 @@ def _dist_config_settings(*, suffix, plat_flag_values, python_version, **kwargs) for name, f, compatible_with in [ ("py_none", _flags.whl_plat, None), ("py3_none", _flags.whl_plat_py3, None), - ("py3_abi3", _flags.whl_plat_py3_abi3, (FLAGS.is_py_non_freethreaded,)), + ("py3_abi3", _flags.whl_plat_py3_abi3, (FLAGS._is_py_freethreaded_no,)), ("none", _flags.whl_plat_pycp3x, None), - ("abi3", _flags.whl_plat_pycp3x_abi3, (FLAGS.is_py_non_freethreaded,)), + ("abi3", _flags.whl_plat_pycp3x_abi3, (FLAGS._is_py_freethreaded_no,)), # The below are not specializations of one another, they are variants - (cpv, _flags.whl_plat_pycp3x_abicp, (FLAGS.is_py_non_freethreaded,)), - (cpv + "t", _flags.whl_plat_pycp3x_abicp, (FLAGS.is_py_freethreaded,)), + (cpv, _flags.whl_plat_pycp3x_abicp, (FLAGS._is_py_freethreaded_no,)), + (cpv + "t", _flags.whl_plat_pycp3x_abicp, (FLAGS._is_py_freethreaded_yes,)), ]: if (f, compatible_with) in used_flags: # This should never happen as all of the different whls should have @@ -329,7 +329,7 @@ def _dist_config_setting(*, name, compatible_with = None, native = native, **kwa compatible_with: {type}`tuple[Label]` A collection of config settings that are compatible with the given dist config setting. For example, if only non-freethreaded python builds are allowed, add - FLAGS.is_py_non_freethreaded here. + FLAGS._is_py_freethreaded_no here. native (struct): The struct containing alias and config_setting rules to use for creating the objects. Can be overridden for unit tests reasons. diff --git a/python/private/toolchain_aliases.bzl b/python/private/toolchain_aliases.bzl index 31ac4a8fdf..092863260c 100644 --- a/python/private/toolchain_aliases.bzl +++ b/python/private/toolchain_aliases.bzl @@ -14,7 +14,8 @@ """Create toolchain alias targets.""" -load("@rules_python//python:versions.bzl", "PLATFORMS") +load("@bazel_skylib//lib:selects.bzl", "selects") +load("//python:versions.bzl", "PLATFORMS") def toolchain_aliases(*, name, platforms, visibility = None, native = native): """Create toolchain aliases for the python toolchains. @@ -30,12 +31,17 @@ def toolchain_aliases(*, name, platforms, visibility = None, native = native): if platform not in platforms: continue + _platform = "_" + platform native.config_setting( - name = platform, - flag_values = PLATFORMS[platform].flag_values, + name = _platform, constraint_values = PLATFORMS[platform].compatible_with, visibility = ["//visibility:private"], ) + selects.config_setting_group( + name = platform, + match_all = PLATFORMS[platform].target_settings + [_platform], + visibility = ["//visibility:private"], + ) prefix = name for name in [ diff --git a/tests/toolchains/BUILD.bazel b/tests/toolchains/BUILD.bazel index f346651d46..b9952865cb 100644 --- a/tests/toolchains/BUILD.bazel +++ b/tests/toolchains/BUILD.bazel @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") load(":defs.bzl", "define_toolchain_tests") @@ -30,3 +31,10 @@ py_reconfig_test( "@platforms//cpu:x86_64", ] if BZLMOD_ENABLED else ["@platforms//:incompatible"], ) + +build_test( + name = "build_test", + targets = [ + "@python_3_11//:python_headers", + ], +) From 175a33610e853388c83730d9e2b5b2ac3626649d Mon Sep 17 00:00:00 2001 From: yushan26 <107004874+yushan26@users.noreply.github.com> Date: Wed, 18 Jun 2025 09:18:35 -0700 Subject: [PATCH 35/43] refactor(gazelle) Types for exposed members of `python.ParserOutput` are now all public (#2959) Export the members of `python.ParserOutput` struct to make it publicly accessible. This allows other `py` extensions to leverage the Python resolver logic for resolving Python imports, instead of have to duplicate the resolving logic. --------- Co-authored-by: yushan --- CHANGELOG.md | 21 +++++++++++++++++++++ gazelle/python/file_parser.go | 16 ++++++++-------- gazelle/python/file_parser_test.go | 22 +++++++++++----------- gazelle/python/generate.go | 2 +- gazelle/python/parser.go | 14 +++++++------- gazelle/python/resolve.go | 4 ++-- gazelle/python/std_modules.go | 2 +- gazelle/python/std_modules_test.go | 6 +++--- gazelle/python/target.go | 4 ++-- 9 files changed, 56 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 488f1054a1..bf3d25c792 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,27 @@ BEGIN_UNRELEASED_TEMPLATE END_UNRELEASED_TEMPLATE --> +{#v0-0-0} +## Unreleased + +[0.0.0]: https://github.com/bazel-contrib/rules_python/releases/tag/0.0.0 + +{#v0-0-0-changed} +### Changed +* (gazelle) Types for exposed members of `python.ParserOutput` are now all public. + +{#v0-0-0-fixed} +### Fixed +* Nothing fixed. + +{#v0-0-0-added} +### Added +* Nothing added. + +{#v0-0-0-removed} +### Removed +* Nothing removed. + {#1-5-0} ## [1.5.0] - 2025-06-11 diff --git a/gazelle/python/file_parser.go b/gazelle/python/file_parser.go index c147984fc3..3f8363fbdf 100644 --- a/gazelle/python/file_parser.go +++ b/gazelle/python/file_parser.go @@ -41,8 +41,8 @@ const ( type ParserOutput struct { FileName string - Modules []module - Comments []comment + Modules []Module + Comments []Comment HasMain bool } @@ -127,24 +127,24 @@ func (p *FileParser) parseMain(ctx context.Context, node *sitter.Node) bool { return false } -// parseImportStatement parses a node for an import statement, returning a `module` and a boolean +// parseImportStatement parses a node for an import statement, returning a `Module` and a boolean // representing if the parse was OK or not. -func parseImportStatement(node *sitter.Node, code []byte) (module, bool) { +func parseImportStatement(node *sitter.Node, code []byte) (Module, bool) { switch node.Type() { case sitterNodeTypeDottedName: - return module{ + return Module{ Name: node.Content(code), LineNumber: node.StartPoint().Row + 1, }, true case sitterNodeTypeAliasedImport: return parseImportStatement(node.Child(0), code) case sitterNodeTypeWildcardImport: - return module{ + return Module{ Name: "*", LineNumber: node.StartPoint().Row + 1, }, true } - return module{}, false + return Module{}, false } // parseImportStatements parses a node for import statements, returning true if the node is @@ -188,7 +188,7 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool { // It updates FileParser.output.Comments with the parsed comment. func (p *FileParser) parseComments(node *sitter.Node) bool { if node.Type() == sitterNodeTypeComment { - p.output.Comments = append(p.output.Comments, comment(node.Content(p.code))) + p.output.Comments = append(p.output.Comments, Comment(node.Content(p.code))) return true } return false diff --git a/gazelle/python/file_parser_test.go b/gazelle/python/file_parser_test.go index 3682cff753..20085f0e76 100644 --- a/gazelle/python/file_parser_test.go +++ b/gazelle/python/file_parser_test.go @@ -27,7 +27,7 @@ func TestParseImportStatements(t *testing.T) { name string code string filepath string - result []module + result []Module }{ { name: "not has import", @@ -39,7 +39,7 @@ func TestParseImportStatements(t *testing.T) { name: "has import", code: "import unittest\nimport os.path\nfrom foo.bar import abc.xyz", filepath: "abc.py", - result: []module{ + result: []Module{ { Name: "unittest", LineNumber: 1, @@ -66,7 +66,7 @@ func TestParseImportStatements(t *testing.T) { import unittest `, filepath: "abc.py", - result: []module{ + result: []Module{ { Name: "unittest", LineNumber: 2, @@ -79,7 +79,7 @@ func TestParseImportStatements(t *testing.T) { name: "invalid syntax", code: "import os\nimport", filepath: "abc.py", - result: []module{ + result: []Module{ { Name: "os", LineNumber: 1, @@ -92,7 +92,7 @@ func TestParseImportStatements(t *testing.T) { name: "import as", code: "import os as b\nfrom foo import bar as c# 123", filepath: "abc.py", - result: []module{ + result: []Module{ { Name: "os", LineNumber: 1, @@ -111,7 +111,7 @@ func TestParseImportStatements(t *testing.T) { { name: "complex import", code: "from unittest import *\nfrom foo import (bar as c, baz, qux as d)\nfrom . import abc", - result: []module{ + result: []Module{ { Name: "unittest.*", LineNumber: 1, @@ -152,7 +152,7 @@ func TestParseComments(t *testing.T) { units := []struct { name string code string - result []comment + result []Comment }{ { name: "not has comment", @@ -162,17 +162,17 @@ func TestParseComments(t *testing.T) { { name: "has comment", code: "# a = 1\n# b = 2", - result: []comment{"# a = 1", "# b = 2"}, + result: []Comment{"# a = 1", "# b = 2"}, }, { name: "has comment in if", code: "if True:\n # a = 1\n # b = 2", - result: []comment{"# a = 1", "# b = 2"}, + result: []Comment{"# a = 1", "# b = 2"}, }, { name: "has comment inline", code: "import os# 123\nfrom pathlib import Path as b#456", - result: []comment{"# 123", "#456"}, + result: []Comment{"# 123", "#456"}, }, } for _, u := range units { @@ -248,7 +248,7 @@ func TestParseFull(t *testing.T) { output, err := p.Parse(context.Background()) assert.NoError(t, err) assert.Equal(t, ParserOutput{ - Modules: []module{{Name: "bar.abc", LineNumber: 1, Filepath: "foo/a.py", From: "bar"}}, + Modules: []Module{{Name: "bar.abc", LineNumber: 1, Filepath: "foo/a.py", From: "bar"}}, Comments: nil, HasMain: false, FileName: "a.py", diff --git a/gazelle/python/generate.go b/gazelle/python/generate.go index 27930c1025..5eedbd9601 100644 --- a/gazelle/python/generate.go +++ b/gazelle/python/generate.go @@ -471,7 +471,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes for _, pyTestTarget := range pyTestTargets { if conftest != nil { - pyTestTarget.addModuleDependency(module{Name: strings.TrimSuffix(conftestFilename, ".py")}) + pyTestTarget.addModuleDependency(Module{Name: strings.TrimSuffix(conftestFilename, ".py")}) } pyTest := pyTestTarget.build() diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go index 1b2a90dddf..cf80578220 100644 --- a/gazelle/python/parser.go +++ b/gazelle/python/parser.go @@ -145,9 +145,9 @@ func removeDupesFromStringTreeSetSlice(array []string) []string { return dedupe } -// module represents a fully-qualified, dot-separated, Python module as seen on +// Module represents a fully-qualified, dot-separated, Python module as seen on // the import statement, alongside the line number where it happened. -type module struct { +type Module struct { // The fully-qualified, dot-separated, Python module name as seen on import // statements. Name string `json:"name"` @@ -162,7 +162,7 @@ type module struct { // moduleComparator compares modules by name. func moduleComparator(a, b interface{}) int { - return godsutils.StringComparator(a.(module).Name, b.(module).Name) + return godsutils.StringComparator(a.(Module).Name, b.(Module).Name) } // annotationKind represents Gazelle annotation kinds. @@ -176,12 +176,12 @@ const ( annotationKindIncludeDep annotationKind = "include_dep" ) -// comment represents a Python comment. -type comment string +// Comment represents a Python comment. +type Comment string // asAnnotation returns an annotation object if the comment has the // annotationPrefix. -func (c *comment) asAnnotation() (*annotation, error) { +func (c *Comment) asAnnotation() (*annotation, error) { uncomment := strings.TrimLeft(string(*c), "# ") if !strings.HasPrefix(uncomment, annotationPrefix) { return nil, nil @@ -215,7 +215,7 @@ type annotations struct { // annotationsFromComments returns all the annotations parsed out of the // comments of a Python module. -func annotationsFromComments(comments []comment) (*annotations, error) { +func annotationsFromComments(comments []Comment) (*annotations, error) { ignore := make(map[string]struct{}) includeDeps := []string{} for _, comment := range comments { diff --git a/gazelle/python/resolve.go b/gazelle/python/resolve.go index 7a2ec3d68a..996cbbadc0 100644 --- a/gazelle/python/resolve.go +++ b/gazelle/python/resolve.go @@ -151,7 +151,7 @@ func (py *Resolver) Resolve( hasFatalError := false MODULES_LOOP: for it.Next() { - mod := it.Value().(module) + mod := it.Value().(Module) moduleParts := strings.Split(mod.Name, ".") possibleModules := []string{mod.Name} for len(moduleParts) > 1 { @@ -214,7 +214,7 @@ func (py *Resolver) Resolve( matches := ix.FindRulesByImportWithConfig(c, imp, languageName) if len(matches) == 0 { // Check if the imported module is part of the standard library. - if isStdModule(module{Name: moduleName}) { + if isStdModule(Module{Name: moduleName}) { continue MODULES_LOOP } else if cfg.ValidateImportStatements() { err := fmt.Errorf( diff --git a/gazelle/python/std_modules.go b/gazelle/python/std_modules.go index e10f87b6ea..ecb4f4c454 100644 --- a/gazelle/python/std_modules.go +++ b/gazelle/python/std_modules.go @@ -34,7 +34,7 @@ func init() { } } -func isStdModule(m module) bool { +func isStdModule(m Module) bool { _, ok := stdModules[m.Name] return ok } diff --git a/gazelle/python/std_modules_test.go b/gazelle/python/std_modules_test.go index bc22638e69..dbcd18c9d6 100644 --- a/gazelle/python/std_modules_test.go +++ b/gazelle/python/std_modules_test.go @@ -21,7 +21,7 @@ import ( ) func TestIsStdModule(t *testing.T) { - assert.True(t, isStdModule(module{Name: "unittest"})) - assert.True(t, isStdModule(module{Name: "os.path"})) - assert.False(t, isStdModule(module{Name: "foo"})) + assert.True(t, isStdModule(Module{Name: "unittest"})) + assert.True(t, isStdModule(Module{Name: "os.path"})) + assert.False(t, isStdModule(Module{Name: "foo"})) } diff --git a/gazelle/python/target.go b/gazelle/python/target.go index c40d6fb3b7..1fb9218656 100644 --- a/gazelle/python/target.go +++ b/gazelle/python/target.go @@ -69,7 +69,7 @@ func (t *targetBuilder) addSrcs(srcs *treeset.Set) *targetBuilder { } // addModuleDependency adds a single module dep to the target. -func (t *targetBuilder) addModuleDependency(dep module) *targetBuilder { +func (t *targetBuilder) addModuleDependency(dep Module) *targetBuilder { fileName := dep.Name + ".py" if dep.From != "" { fileName = dep.From + ".py" @@ -87,7 +87,7 @@ func (t *targetBuilder) addModuleDependency(dep module) *targetBuilder { func (t *targetBuilder) addModuleDependencies(deps *treeset.Set) *targetBuilder { it := deps.Iterator() for it.Next() { - t.addModuleDependency(it.Value().(module)) + t.addModuleDependency(it.Value().(Module)) } return t } From 5b1db075d0810d09db7b1411c273a968ee3e4be0 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 19 Jun 2025 15:49:03 +0900 Subject: [PATCH 36/43] feat(pypi): pip.defaults API for customizing pipstar 1/n (#2987) Parse env markers in pip.parse using starlark Summary: - Allow switching to the Starlark implementation of the marker evaluation function. - Add a way for users to modify the `env` for the marker evaluation when parsing the requirements. This can only be done by `rules_python` or the root module. - Limit the platform selection when parsing the requirements files. Work towards #2747 Work towards #2949 Split out from #2909 --------- Co-authored-by: Richard Levasseur --- CHANGELOG.md | 4 +- python/private/pypi/BUILD.bazel | 5 +- python/private/pypi/env_marker_info.bzl | 2 +- python/private/pypi/evaluate_markers.bzl | 19 +- python/private/pypi/extension.bzl | 258 ++++++++++++++++-- python/private/pypi/pep508_evaluate.bzl | 2 +- python/private/pypi/pip_repository.bzl | 10 + .../pypi/requirements_files_by_platform.bzl | 54 ++-- tests/pypi/extension/extension_tests.bzl | 111 +++++++- .../requirements_files_by_platform_tests.bzl | 41 ++- 10 files changed, 437 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf3d25c792..9897dc9ec8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,7 +62,9 @@ END_UNRELEASED_TEMPLATE {#v0-0-0-added} ### Added -* Nothing added. +* (pypi) To configure the environment for `requirements.txt` evaluation, use the newly added + developer preview of the `pip.default` tag class. Only `rules_python` and root modules can use + this feature. {#v0-0-0-removed} ### Removed diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index d89dc6c228..b569b2217c 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -97,10 +97,10 @@ bzl_library( name = "evaluate_markers_bzl", srcs = ["evaluate_markers.bzl"], deps = [ - ":pep508_env_bzl", + ":deps_bzl", ":pep508_evaluate_bzl", - ":pep508_platform_bzl", ":pep508_requirement_bzl", + ":pypi_repo_utils_bzl", ], ) @@ -113,6 +113,7 @@ bzl_library( ":hub_repository_bzl", ":parse_requirements_bzl", ":parse_whl_name_bzl", + ":pep508_env_bzl", ":pip_repository_attrs_bzl", ":simpleapi_download_bzl", ":whl_config_setting_bzl", diff --git a/python/private/pypi/env_marker_info.bzl b/python/private/pypi/env_marker_info.bzl index c3c5ec69ed..37eefb2a0f 100644 --- a/python/private/pypi/env_marker_info.bzl +++ b/python/private/pypi/env_marker_info.bzl @@ -17,7 +17,7 @@ The {obj}`--//python/config_settings:pip_env_marker_config` flag. The values to use for environment markers when evaluating an expression. The keys and values should be compatible with the [PyPA dependency specifiers -specification](https://packaging.python.org/en/latest/specifications/dependency-specifiers/) +specification](https://packaging.python.org/en/latest/specifications/dependency-specifiers/). Missing values will be set to the specification's defaults or computed using available toolchain information. diff --git a/python/private/pypi/evaluate_markers.bzl b/python/private/pypi/evaluate_markers.bzl index 191933596e..58a29a9181 100644 --- a/python/private/pypi/evaluate_markers.bzl +++ b/python/private/pypi/evaluate_markers.bzl @@ -15,9 +15,7 @@ """A simple function that evaluates markers using a python interpreter.""" load(":deps.bzl", "record_files") -load(":pep508_env.bzl", "env") load(":pep508_evaluate.bzl", "evaluate") -load(":pep508_platform.bzl", "platform_from_str") load(":pep508_requirement.bzl", "requirement") load(":pypi_repo_utils.bzl", "pypi_repo_utils") @@ -30,22 +28,27 @@ SRCS = [ Label("//python/private/pypi/whl_installer:platform.py"), ] -def evaluate_markers(requirements, python_version = None): +def evaluate_markers(*, requirements, platforms): """Return the list of supported platforms per requirements line. Args: requirements: {type}`dict[str, list[str]]` of the requirement file lines to evaluate. - python_version: {type}`str | None` the version that can be used when evaluating the markers. + platforms: {type}`dict[str, dict[str, str]]` The environments that we for each requirement + file to evaluate. The keys between the platforms and requirements should be shared. Returns: dict of string lists with target platforms """ ret = {} - for req_string, platforms in requirements.items(): + for req_string, platform_strings in requirements.items(): req = requirement(req_string) - for platform in platforms: - if evaluate(req.marker, env = env(platform_from_str(platform, python_version))): - ret.setdefault(req_string, []).append(platform) + for platform_str in platform_strings: + env = platforms.get(platform_str) + if not env: + fail("Please define platform: '{}'".format(platform_str)) + + if evaluate(req.marker, env = env): + ret.setdefault(req_string, []).append(platform_str) return ret diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 867abe0898..97b6825e51 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -25,10 +25,11 @@ load("//python/private:repo_utils.bzl", "repo_utils") load("//python/private:version.bzl", "version") load("//python/private:version_label.bzl", "version_label") load(":attrs.bzl", "use_isolated") -load(":evaluate_markers.bzl", "evaluate_markers_py", EVALUATE_MARKERS_SRCS = "SRCS") +load(":evaluate_markers.bzl", "evaluate_markers_py", EVALUATE_MARKERS_SRCS = "SRCS", evaluate_markers_star = "evaluate_markers") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_requirements.bzl", "parse_requirements") load(":parse_whl_name.bzl", "parse_whl_name") +load(":pep508_env.bzl", "env") load(":pip_repository_attrs.bzl", "ATTRS") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") load(":simpleapi_download.bzl", "simpleapi_download") @@ -65,22 +66,36 @@ def _whl_mods_impl(whl_mods_dict): whl_mods = whl_mods, ) +def _platforms(*, python_version, minor_mapping, config): + platforms = {} + python_version = full_version( + version = python_version, + minor_mapping = minor_mapping, + ) + abi = "cp3{}".format(python_version[2:]) + + for platform, values in config.platforms.items(): + key = "{}_{}".format(abi, platform) + platforms[key] = env(key) | values.env + return platforms + def _create_whl_repos( module_ctx, *, pip_attr, whl_overrides, + config, available_interpreters = INTERPRETER_LABELS, minor_mapping = MINOR_MAPPING, - evaluate_markers = evaluate_markers_py, - get_index_urls = None, - enable_pipstar = False): + evaluate_markers = None, + get_index_urls = None): """create all of the whl repositories Args: module_ctx: {type}`module_ctx`. pip_attr: {type}`struct` - the struct that comes from the tag class iteration. whl_overrides: {type}`dict[str, struct]` - per-wheel overrides. + config: The platform configuration. get_index_urls: A function used to get the index URLs available_interpreters: {type}`dict[str, Label]` The dictionary of available interpreters that have been registered using the `python` bzlmod extension. @@ -89,7 +104,6 @@ def _create_whl_repos( minor_mapping: {type}`dict[str, str]` The dictionary needed to resolve the full python version used to parse package METADATA files. evaluate_markers: the function used to evaluate the markers. - enable_pipstar: enable the pipstar feature. Returns a {type}`struct` with the following attributes: whl_map: {type}`dict[str, list[struct]]` the output is keyed by the @@ -160,23 +174,19 @@ def _create_whl_repos( whl_group_mapping = {} requirement_cycles = {} - requirements_by_platform = parse_requirements( - module_ctx, - requirements_by_platform = requirements_files_by_platform( - requirements_by_platform = pip_attr.requirements_by_platform, - requirements_linux = pip_attr.requirements_linux, - requirements_lock = pip_attr.requirements_lock, - requirements_osx = pip_attr.requirements_darwin, - requirements_windows = pip_attr.requirements_windows, - extra_pip_args = pip_attr.extra_pip_args, - python_version = full_version( - version = pip_attr.python_version, + if evaluate_markers: + # This is most likely unit tests + pass + elif config.enable_pipstar: + evaluate_markers = lambda _, requirements: evaluate_markers_star( + requirements = requirements, + platforms = _platforms( + python_version = pip_attr.python_version, minor_mapping = minor_mapping, + config = config, ), - logger = logger, - ), - extra_pip_args = pip_attr.extra_pip_args, - get_index_urls = get_index_urls, + ) + else: # NOTE @aignas 2024-08-02: , we will execute any interpreter that we find either # in the PATH or if specified as a label. We will configure the env # markers when evaluating the requirement lines based on the output @@ -191,14 +201,34 @@ def _create_whl_repos( # instances to perform this manipulation. This function should be executed # only once by the underlying code to minimize the overhead needed to # spin up a Python interpreter. - evaluate_markers = lambda module_ctx, requirements: evaluate_markers( + evaluate_markers = lambda module_ctx, requirements: evaluate_markers_py( module_ctx, requirements = requirements, python_interpreter = pip_attr.python_interpreter, python_interpreter_target = python_interpreter_target, srcs = pip_attr._evaluate_markers_srcs, logger = logger, + ) + + requirements_by_platform = parse_requirements( + module_ctx, + requirements_by_platform = requirements_files_by_platform( + requirements_by_platform = pip_attr.requirements_by_platform, + requirements_linux = pip_attr.requirements_linux, + requirements_lock = pip_attr.requirements_lock, + requirements_osx = pip_attr.requirements_darwin, + requirements_windows = pip_attr.requirements_windows, + extra_pip_args = pip_attr.extra_pip_args, + platforms = sorted(config.platforms), # here we only need keys + python_version = full_version( + version = pip_attr.python_version, + minor_mapping = minor_mapping, + ), + logger = logger, ), + extra_pip_args = pip_attr.extra_pip_args, + get_index_urls = get_index_urls, + evaluate_markers = evaluate_markers, logger = logger, ) @@ -233,7 +263,7 @@ def _create_whl_repos( for p, args in whl_overrides.get(whl.name, {}).items() }, ) - if not enable_pipstar: + if not config.enable_pipstar: maybe_args["experimental_target_platforms"] = pip_attr.experimental_target_platforms whl_library_args.update({k: v for k, v in maybe_args.items() if v}) @@ -258,7 +288,7 @@ def _create_whl_repos( auth_patterns = pip_attr.auth_patterns, python_version = major_minor, is_multiple_versions = whl.is_multiple_versions, - enable_pipstar = enable_pipstar, + enable_pipstar = config.enable_pipstar, ) repo_name = "{}_{}".format(pip_name, repo.repo_name) @@ -342,16 +372,85 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net ), ) +def _configure(config, *, platform, os_name, arch_name, override = False, env = {}): + """Set the value in the config if the value is provided""" + config.setdefault("platforms", {}) + if platform: + if not override and config.get("platforms", {}).get(platform): + return + + for key in env: + if key not in _SUPPORTED_PEP508_KEYS: + fail("Unsupported key in the PEP508 environment: {}".format(key)) + + config["platforms"][platform] = struct( + name = platform.replace("-", "_").lower(), + os_name = os_name, + arch_name = arch_name, + env = env, + ) + else: + config["platforms"].pop(platform) + +def _create_config(defaults): + if defaults["platforms"]: + return struct(**defaults) + + # NOTE: We have this so that it is easier to maintain unit tests assuming certain + # defaults + for cpu in [ + "x86_64", + "aarch64", + # TODO @aignas 2025-05-19: only leave tier 0-1 cpus when stabilizing the + # `pip.default` extension. i.e. drop the below values - users will have to + # define themselves if they need them. + "arm", + "ppc", + "s390x", + ]: + _configure( + defaults, + arch_name = cpu, + os_name = "linux", + platform = "linux_{}".format(cpu), + env = {"platform_version": "0"}, + ) + for cpu in [ + "aarch64", + "x86_64", + ]: + _configure( + defaults, + arch_name = cpu, + # We choose the oldest non-EOL version at the time when we release `rules_python`. + # See https://endoflife.date/macos + env = {"platform_version": "14.0"}, + os_name = "osx", + platform = "osx_{}".format(cpu), + ) + + _configure( + defaults, + arch_name = "x86_64", + env = {"platform_version": "0"}, + os_name = "windows", + platform = "windows_x86_64", + ) + return struct(**defaults) + def parse_modules( module_ctx, _fail = fail, simpleapi_download = simpleapi_download, + enable_pipstar = False, **kwargs): """Implementation of parsing the tag classes for the extension and return a struct for registering repositories. Args: module_ctx: {type}`module_ctx` module context. simpleapi_download: Used for testing overrides + enable_pipstar: {type}`bool` a flag to enable dropping Python dependency for + evaluation of the extension. _fail: {type}`function` the failure function, mainly for testing. **kwargs: Extra arguments passed to the layers below. @@ -389,6 +488,34 @@ You cannot use both the additive_build_content and additive_build_content_file a srcs_exclude_glob = whl_mod.srcs_exclude_glob, ) + defaults = { + "enable_pipstar": enable_pipstar, + "platforms": {}, + } + for mod in module_ctx.modules: + if not (mod.is_root or mod.name == "rules_python"): + continue + + for tag in mod.tags.default: + _configure( + defaults, + arch_name = tag.arch_name, + env = tag.env, + os_name = tag.os_name, + platform = tag.platform, + override = mod.is_root, + # TODO @aignas 2025-05-19: add more attr groups: + # * for AUTH - the default `netrc` usage could be configured through a common + # attribute. + # * for index/downloader config. This includes all of those attributes for + # overrides, etc. Index overrides per platform could be also used here. + # * for whl selection - selecting preferences of which `platform_tag`s we should use + # for what. We could also model the `cp313t` freethreaded as separate platforms. + ) + + config = _create_config(defaults) + + # TODO @aignas 2025-06-03: Merge override API with the builder? _overriden_whl_set = {} whl_overrides = {} for module in module_ctx.modules: @@ -498,11 +625,13 @@ You cannot use both the additive_build_content and additive_build_content_file a elif pip_attr.experimental_index_url_overrides: fail("'experimental_index_url_overrides' is a no-op unless 'experimental_index_url' is set") + # TODO @aignas 2025-05-19: express pip.parse as a series of configure calls out = _create_whl_repos( module_ctx, pip_attr = pip_attr, get_index_urls = get_index_urls, whl_overrides = whl_overrides, + config = config, **kwargs ) hub_whl_map.setdefault(hub_name, {}) @@ -651,6 +780,72 @@ def _pip_impl(module_ctx): else: return None +_default_attrs = { + "arch_name": attr.string( + doc = """\ +The CPU architecture name to be used. + +:::{note} +Either this or {attr}`env` `platform_machine` key should be specified. +::: +""", + ), + "os_name": attr.string( + doc = """\ +The OS name to be used. + +:::{note} +Either this or the appropriate `env` keys should be specified. +::: +""", + ), + "platform": attr.string( + doc = """\ +A platform identifier which will be used as the unique identifier within the extension evaluation. +If you are defining custom platforms in your project and don't want things to clash, use extension +[isolation] feature. + +[isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate +""", + ), +} | { + "env": attr.string_dict( + doc = """\ +The values to use for environment markers when evaluating an expression. + +The keys and values should be compatible with the [PyPA dependency specifiers +specification](https://packaging.python.org/en/latest/specifications/dependency-specifiers/). + +Missing values will be set to the specification's defaults or computed using +available toolchain information. + +Supported keys: +* `implementation_name`, defaults to `cpython`. +* `os_name`, defaults to a value inferred from the {attr}`os_name`. +* `platform_machine`, defaults to a value inferred from the {attr}`arch_name`. +* `platform_release`, defaults to an empty value. +* `platform_system`, defaults to a value inferred from the {attr}`os_name`. +* `platform_version`, defaults to `0`. +* `sys_platform`, defaults to a value inferred from the {attr}`os_name`. + +::::{note} +This is only used if the {envvar}`RULES_PYTHON_ENABLE_PIPSTAR` is enabled. +:::: +""", + ), + # The values for PEP508 env marker evaluation during the lock file parsing +} + +_SUPPORTED_PEP508_KEYS = [ + "implementation_name", + "os_name", + "platform_machine", + "platform_release", + "platform_system", + "platform_version", + "sys_platform", +] + def _pip_parse_ext_attrs(**kwargs): """Get the attributes for the pip extension. @@ -907,6 +1102,23 @@ the BUILD files for wheels. """, implementation = _pip_impl, tag_classes = { + "default": tag_class( + attrs = _default_attrs, + doc = """\ +This tag class allows for more customization of how the configuration for the hub repositories is built. + + +:::{include} /_includes/experimtal_api.md +::: + +:::{seealso} +The [environment markers][environment_markers] specification for the explanation of the +terms used in this extension. + +[environment_markers]: https://packaging.python.org/en/latest/specifications/dependency-specifiers/#environment-markers +::: +""", + ), "override": _override_tag, "parse": tag_class( attrs = _pip_parse_ext_attrs(), diff --git a/python/private/pypi/pep508_evaluate.bzl b/python/private/pypi/pep508_evaluate.bzl index d4492a75bb..fe2cac965a 100644 --- a/python/private/pypi/pep508_evaluate.bzl +++ b/python/private/pypi/pep508_evaluate.bzl @@ -117,7 +117,7 @@ def evaluate(marker, *, env, strict = True, **kwargs): Args: marker: {type}`str` The string marker to evaluate. - env: {type}`dict` The environment to evaluate the marker against. + env: {type}`dict[str, str]` The environment to evaluate the marker against. strict: {type}`bool` A setting to not fail on missing values in the env. **kwargs: Extra kwargs to be passed to the expression evaluator. diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index 724fb6ddba..e63bd6c3d1 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -80,6 +80,16 @@ def _pip_repository_impl(rctx): requirements_osx = rctx.attr.requirements_darwin, requirements_windows = rctx.attr.requirements_windows, extra_pip_args = rctx.attr.extra_pip_args, + platforms = [ + "linux_aarch64", + "linux_arm", + "linux_ppc", + "linux_s390x", + "linux_x86_64", + "osx_aarch64", + "osx_x86_64", + "windows_x86_64", + ], ), extra_pip_args = rctx.attr.extra_pip_args, evaluate_markers = lambda rctx, requirements: evaluate_markers_py( diff --git a/python/private/pypi/requirements_files_by_platform.bzl b/python/private/pypi/requirements_files_by_platform.bzl index 9165c05bed..d8d3651461 100644 --- a/python/private/pypi/requirements_files_by_platform.bzl +++ b/python/private/pypi/requirements_files_by_platform.bzl @@ -16,20 +16,7 @@ load(":whl_target_platforms.bzl", "whl_target_platforms") -# TODO @aignas 2024-05-13: consider using the same platform tags as are used in -# the //python:versions.bzl -DEFAULT_PLATFORMS = [ - "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", - "linux_x86_64", - "osx_aarch64", - "osx_x86_64", - "windows_x86_64", -] - -def _default_platforms(*, filter): +def _default_platforms(*, filter, platforms): if not filter: fail("Must specific a filter string, got: {}".format(filter)) @@ -48,11 +35,13 @@ def _default_platforms(*, filter): fail("The filter can only contain '*' at the end of it") if not prefix: - return DEFAULT_PLATFORMS + return platforms - return [p for p in DEFAULT_PLATFORMS if p.startswith(prefix)] + match = [p for p in platforms if p.startswith(prefix)] else: - return [p for p in DEFAULT_PLATFORMS if filter in p] + match = [p for p in platforms if filter in p] + + return match def _platforms_from_args(extra_pip_args): platform_values = [] @@ -105,6 +94,7 @@ def requirements_files_by_platform( requirements_linux = None, requirements_lock = None, requirements_windows = None, + platforms, extra_pip_args = None, python_version = None, logger = None, @@ -123,6 +113,8 @@ def requirements_files_by_platform( be joined with args fined in files. python_version: str or None. This is needed when the get_index_urls is specified. It should be of the form "3.x.x", + platforms: {type}`list[str]` the list of human-friendly platform labels that should + be used for the evaluation. logger: repo_utils.logger or None, a simple struct to log diagnostic messages. fail_fn (Callable[[str], None]): A failure function used in testing failure cases. @@ -144,11 +136,13 @@ def requirements_files_by_platform( ) return None - platforms = _platforms_from_args(extra_pip_args) + platforms_from_args = _platforms_from_args(extra_pip_args) if logger: - logger.debug(lambda: "Platforms from pip args: {}".format(platforms)) + logger.debug(lambda: "Platforms from pip args: {}".format(platforms_from_args)) + + default_platforms = [_platform(p, python_version) for p in platforms] - if platforms: + if platforms_from_args: lock_files = [ f for f in [ @@ -168,7 +162,7 @@ def requirements_files_by_platform( return None files_by_platform = [ - (lock_files[0], platforms), + (lock_files[0], platforms_from_args), ] if logger: logger.debug(lambda: "Files by platform with the platform set in the args: {}".format(files_by_platform)) @@ -177,7 +171,7 @@ def requirements_files_by_platform( file: [ platform for filter_or_platform in specifier.split(",") - for platform in (_default_platforms(filter = filter_or_platform) if filter_or_platform.endswith("*") else [filter_or_platform]) + for platform in (_default_platforms(filter = filter_or_platform, platforms = platforms) if filter_or_platform.endswith("*") else [filter_or_platform]) ] for file, specifier in requirements_by_platform.items() }.items() @@ -188,9 +182,9 @@ def requirements_files_by_platform( for f in [ # If the users need a greater span of the platforms, they should consider # using the 'requirements_by_platform' attribute. - (requirements_linux, _default_platforms(filter = "linux_*")), - (requirements_osx, _default_platforms(filter = "osx_*")), - (requirements_windows, _default_platforms(filter = "windows_*")), + (requirements_linux, _default_platforms(filter = "linux_*", platforms = platforms)), + (requirements_osx, _default_platforms(filter = "osx_*", platforms = platforms)), + (requirements_windows, _default_platforms(filter = "windows_*", platforms = platforms)), (requirements_lock, None), ]: if f[0]: @@ -215,8 +209,7 @@ def requirements_files_by_platform( return None configured_platforms[p] = file - else: - default_platforms = [_platform(p, python_version) for p in DEFAULT_PLATFORMS] + elif plats == None: plats = [ p for p in default_platforms @@ -231,6 +224,13 @@ def requirements_files_by_platform( for p in plats: configured_platforms[p] = file + elif logger: + logger.warn(lambda: "File {} will be ignored because there are no configured platforms: {}".format( + file, + default_platforms, + )) + continue + if logger: logger.debug(lambda: "Configured platforms for file {} are {}".format(file, plats)) diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 8e325724f4..3d205a23c4 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -49,23 +49,22 @@ simple==0.0.1 \ ], ) -def _mod(*, name, parse = [], override = [], whl_mods = [], is_root = True): +def _mod(*, name, default = [], parse = [], override = [], whl_mods = [], is_root = True): return struct( name = name, tags = struct( parse = parse, override = override, whl_mods = whl_mods, + default = default, ), is_root = is_root, ) -def _parse_modules(env, **kwargs): +def _parse_modules(env, enable_pipstar = 0, **kwargs): return env.expect.that_struct( parse_modules( - # TODO @aignas 2025-05-11: start integration testing the branch which - # includes this. - enable_pipstar = 0, + enable_pipstar = enable_pipstar, **kwargs ), attrs = dict( @@ -77,6 +76,26 @@ def _parse_modules(env, **kwargs): ), ) +def _default( + arch_name = None, + constraint_values = None, + os_name = None, + platform = None, + target_settings = None, + env = None, + whl_limit = None, + whl_platforms = None): + return struct( + arch_name = arch_name, + constraint_values = constraint_values, + os_name = os_name, + platform = platform, + target_settings = target_settings, + env = env or {}, + whl_platforms = whl_platforms, + whl_limit = whl_limit, + ) + def _parse( *, hub_name, @@ -1023,6 +1042,88 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' _tests.append(_test_optimum_sys_platform_extra) +def _test_pipstar_platforms(env): + pypi = _parse_modules( + env, + module_ctx = _mock_mctx( + _mod( + name = "rules_python", + default = [ + _default( + platform = "{}_{}".format(os, cpu), + ) + for os, cpu in [ + ("linux", "x86_64"), + ("osx", "aarch64"), + ] + ], + parse = [ + _parse( + hub_name = "pypi", + python_version = "3.15", + requirements_lock = "universal.txt", + ), + ], + ), + read = lambda x: { + "universal.txt": """\ +optimum[onnxruntime]==1.17.1 ; sys_platform == 'darwin' +optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' +""", + }[x], + ), + enable_pipstar = True, + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + + pypi.exposed_packages().contains_exactly({"pypi": ["optimum"]}) + pypi.hub_group_map().contains_exactly({"pypi": {}}) + pypi.hub_whl_map().contains_exactly({ + "pypi": { + "optimum": { + "pypi_315_optimum_linux_x86_64": [ + whl_config_setting( + version = "3.15", + target_platforms = [ + "cp315_linux_x86_64", + ], + config_setting = None, + filename = None, + ), + ], + "pypi_315_optimum_osx_aarch64": [ + whl_config_setting( + version = "3.15", + target_platforms = [ + "cp315_osx_aarch64", + ], + config_setting = None, + filename = None, + ), + ], + }, + }, + }) + + pypi.whl_libraries().contains_exactly({ + "pypi_315_optimum_linux_x86_64": { + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": "optimum[onnxruntime-gpu]==1.17.1", + }, + "pypi_315_optimum_osx_aarch64": { + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": "optimum[onnxruntime]==1.17.1", + }, + }) + pypi.whl_mods().contains_exactly({}) + +_tests.append(_test_pipstar_platforms) + def extension_test_suite(name): """Create the test suite. diff --git a/tests/pypi/requirements_files_by_platform/requirements_files_by_platform_tests.bzl b/tests/pypi/requirements_files_by_platform/requirements_files_by_platform_tests.bzl index b729b0eaf0..6688d72ffe 100644 --- a/tests/pypi/requirements_files_by_platform/requirements_files_by_platform_tests.bzl +++ b/tests/pypi/requirements_files_by_platform/requirements_files_by_platform_tests.bzl @@ -15,10 +15,27 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:requirements_files_by_platform.bzl", "requirements_files_by_platform") # buildifier: disable=bzl-visibility +load("//python/private/pypi:requirements_files_by_platform.bzl", _sut = "requirements_files_by_platform") # buildifier: disable=bzl-visibility _tests = [] +requirements_files_by_platform = lambda **kwargs: _sut( + platforms = kwargs.pop( + "platforms", + [ + "linux_aarch64", + "linux_arm", + "linux_ppc", + "linux_s390x", + "linux_x86_64", + "osx_aarch64", + "osx_x86_64", + "windows_x86_64", + ], + ), + **kwargs +) + def _test_fail_no_requirements(env): errors = [] requirements_files_by_platform( @@ -86,6 +103,28 @@ def _test_simple(env): _tests.append(_test_simple) +def _test_simple_limited(env): + for got in [ + requirements_files_by_platform( + requirements_lock = "requirements_lock", + platforms = ["linux_x86_64", "osx_x86_64"], + ), + requirements_files_by_platform( + requirements_by_platform = { + "requirements_lock": "*", + }, + platforms = ["linux_x86_64", "osx_x86_64"], + ), + ]: + env.expect.that_dict(got).contains_exactly({ + "requirements_lock": [ + "linux_x86_64", + "osx_x86_64", + ], + }) + +_tests.append(_test_simple_limited) + def _test_simple_with_python_version(env): for got in [ requirements_files_by_platform( From b8d6fa3f135fa7da2eed0c857bc25a43517f21fa Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 20 Jun 2025 09:46:07 +0900 Subject: [PATCH 37/43] feat(pypi): pip.defaults API for customizing repo selection 2/n (#2988) WIP: stacked on #2987 This is adding `constraint_values` attribute to `pip.configure` and is threading it all the way down to the generation of `BUILD.bazel` file of for config settings used in the hub repository. Out of scope: - Passing `flag_values` or target settings. I am torn about it - doing it in this PR would flesh out the design more, but at the same time it might become harder to review. - `whl_target_platforms` and `select_whls` is still unchanged, not sure if it is related to this attribute addition. Work towards #2747 Work towards #2548 Work towards #260 --------- Co-authored-by: Richard Levasseur --- CHANGELOG.md | 2 +- python/private/pypi/config_settings.bzl | 31 +++++++------- python/private/pypi/extension.bzl | 34 +++++++++++++-- python/private/pypi/hub_repository.bzl | 5 +++ python/private/pypi/render_pkg_aliases.bzl | 12 ++++-- .../config_settings/config_settings_tests.bzl | 39 +++++++++++++---- tests/pypi/extension/extension_tests.bzl | 4 ++ tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 42 +++++++++++++++---- .../render_pkg_aliases_test.bzl | 13 +++++- 9 files changed, 140 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9897dc9ec8..da3dcc8efc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,7 +64,7 @@ END_UNRELEASED_TEMPLATE ### Added * (pypi) To configure the environment for `requirements.txt` evaluation, use the newly added developer preview of the `pip.default` tag class. Only `rules_python` and root modules can use - this feature. + this feature. You can also configure `constraint_values` using `pip.default`. {#v0-0-0-removed} ### Removed diff --git a/python/private/pypi/config_settings.bzl b/python/private/pypi/config_settings.bzl index 3e828e59f5..7edc578d7a 100644 --- a/python/private/pypi/config_settings.bzl +++ b/python/private/pypi/config_settings.bzl @@ -111,8 +111,8 @@ def config_settings( glibc_versions = [], muslc_versions = [], osx_versions = [], - target_platforms = [], name = None, + platform_constraint_values = {}, **kwargs): """Generate all of the pip config settings. @@ -126,8 +126,10 @@ def config_settings( configure config settings for. osx_versions (list[str]): The list of OSX OS versions to configure config settings for. - target_platforms (list[str]): The list of "{os}_{cpu}" for deriving - constraint values for each condition. + platform_constraint_values: {type}`dict[str, list[str]]` the constraint + values to use instead of the default ones. Key are platform names + (a human-friendly platform string). Values are lists of + `constraint_value` label strings. **kwargs: Other args passed to the underlying implementations, such as {obj}`native`. """ @@ -135,22 +137,17 @@ def config_settings( glibc_versions = [""] + glibc_versions muslc_versions = [""] + muslc_versions osx_versions = [""] + osx_versions - target_platforms = [("", ""), ("osx", "universal2")] + [ - t.split("_", 1) - for t in target_platforms - ] + target_platforms = { + "": [], + # TODO @aignas 2025-06-15: allowing universal2 and platform specific wheels in one + # closure is making things maybe a little bit too complicated. + "osx_universal2": ["@platforms//os:osx"], + } | platform_constraint_values for python_version in python_versions: - for os, cpu in target_platforms: - constraint_values = [] - suffix = "" - if os: - constraint_values.append("@platforms//os:" + os) - suffix += "_" + os - if cpu: - suffix += "_" + cpu - if cpu != "universal2": - constraint_values.append("@platforms//cpu:" + cpu) + for platform_name, constraint_values in target_platforms.items(): + suffix = "_{}".format(platform_name) if platform_name else "" + os, _, cpu = platform_name.partition("_") _dist_config_settings( suffix = suffix, diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 97b6825e51..78511b4c27 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -372,7 +372,7 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net ), ) -def _configure(config, *, platform, os_name, arch_name, override = False, env = {}): +def _configure(config, *, platform, os_name, arch_name, constraint_values, env = {}, override = False): """Set the value in the config if the value is provided""" config.setdefault("platforms", {}) if platform: @@ -387,6 +387,7 @@ def _configure(config, *, platform, os_name, arch_name, override = False, env = name = platform.replace("-", "_").lower(), os_name = os_name, arch_name = arch_name, + constraint_values = constraint_values, env = env, ) else: @@ -413,6 +414,10 @@ def _create_config(defaults): arch_name = cpu, os_name = "linux", platform = "linux_{}".format(cpu), + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:{}".format(cpu), + ], env = {"platform_version": "0"}, ) for cpu in [ @@ -424,17 +429,25 @@ def _create_config(defaults): arch_name = cpu, # We choose the oldest non-EOL version at the time when we release `rules_python`. # See https://endoflife.date/macos - env = {"platform_version": "14.0"}, os_name = "osx", platform = "osx_{}".format(cpu), + constraint_values = [ + "@platforms//os:osx", + "@platforms//cpu:{}".format(cpu), + ], + env = {"platform_version": "14.0"}, ) _configure( defaults, arch_name = "x86_64", - env = {"platform_version": "0"}, os_name = "windows", platform = "windows_x86_64", + constraint_values = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], + env = {"platform_version": "0"}, ) return struct(**defaults) @@ -500,6 +513,7 @@ You cannot use both the additive_build_content and additive_build_content_file a _configure( defaults, arch_name = tag.arch_name, + constraint_values = tag.constraint_values, env = tag.env, os_name = tag.os_name, platform = tag.platform, @@ -679,6 +693,13 @@ You cannot use both the additive_build_content and additive_build_content_file a } for hub_name, extra_whl_aliases in extra_aliases.items() }, + platform_constraint_values = { + hub_name: { + platform_name: sorted([str(Label(cv)) for cv in p.constraint_values]) + for platform_name, p in config.platforms.items() + } + for hub_name in hub_whl_map + }, whl_libraries = { k: dict(sorted(args.items())) for k, args in sorted(whl_libraries.items()) @@ -769,6 +790,7 @@ def _pip_impl(module_ctx): for key, values in whl_map.items() }, packages = mods.exposed_packages.get(hub_name, []), + platform_constraint_values = mods.platform_constraint_values.get(hub_name, {}), groups = mods.hub_group_map.get(hub_name), ) @@ -788,6 +810,12 @@ The CPU architecture name to be used. :::{note} Either this or {attr}`env` `platform_machine` key should be specified. ::: +""", + ), + "constraint_values": attr.label_list( + mandatory = True, + doc = """\ +The constraint_values to use in select statements. """, ), "os_name": attr.string( diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 0dbc6c29c2..4398d7b597 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -34,6 +34,7 @@ def _impl(rctx): }, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, + platform_constraint_values = rctx.attr.platform_constraint_values, ) for path, contents in aliases.items(): rctx.file(path, contents) @@ -83,6 +84,10 @@ hub_repository = repository_rule( The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys. """, ), + "platform_constraint_values": attr.string_list_dict( + doc = "The constraint values for each platform name. The values are string canonical string Label representations", + mandatory = False, + ), "repo_name": attr.string( mandatory = True, doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 28f32edc78..267d7ce85d 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -155,12 +155,14 @@ def _major_minor_versions(python_versions): # Use a dict as a simple set return sorted({_major_minor(v): None for v in python_versions}) -def render_multiplatform_pkg_aliases(*, aliases, **kwargs): +def render_multiplatform_pkg_aliases(*, aliases, platform_constraint_values = {}, **kwargs): """Render the multi-platform pkg aliases. Args: aliases: dict[str, list(whl_config_setting)] A list of aliases that will be transformed from ones having `filename` to ones having `config_setting`. + platform_constraint_values: {type}`dict[str, list[str]]` contains all of the + target platforms and their appropriate `constraint_values`. **kwargs: extra arguments passed to render_pkg_aliases. Returns: @@ -187,18 +189,22 @@ def render_multiplatform_pkg_aliases(*, aliases, **kwargs): muslc_versions = flag_versions.get("muslc_versions", []), osx_versions = flag_versions.get("osx_versions", []), python_versions = _major_minor_versions(flag_versions.get("python_versions", [])), - target_platforms = flag_versions.get("target_platforms", []), + platform_constraint_values = platform_constraint_values, visibility = ["//:__subpackages__"], ) return contents -def _render_config_settings(**kwargs): +def _render_config_settings(platform_constraint_values, **kwargs): return """\ load("@rules_python//python/private/pypi:config_settings.bzl", "config_settings") {}""".format(render.call( "config_settings", name = repr("config_settings"), + platform_constraint_values = render.dict( + platform_constraint_values, + value_repr = render.list, + ), **_repr_dict(value_repr = render.list, **kwargs) )) diff --git a/tests/pypi/config_settings/config_settings_tests.bzl b/tests/pypi/config_settings/config_settings_tests.bzl index f111d0c55c..9551d42d10 100644 --- a/tests/pypi/config_settings/config_settings_tests.bzl +++ b/tests/pypi/config_settings/config_settings_tests.bzl @@ -657,13 +657,34 @@ def config_settings_test_suite(name): # buildifier: disable=function-docstring glibc_versions = [(2, 14), (2, 17)], muslc_versions = [(1, 1)], osx_versions = [(10, 9), (11, 0)], - target_platforms = [ - "windows_x86_64", - "windows_aarch64", - "linux_x86_64", - "linux_ppc", - "linux_aarch64", - "osx_x86_64", - "osx_aarch64", - ], + platform_constraint_values = { + "linux_aarch64": [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + "linux_ppc": [ + "@platforms//cpu:ppc", + "@platforms//os:linux", + ], + "linux_x86_64": [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + "osx_aarch64": [ + "@platforms//cpu:aarch64", + "@platforms//os:osx", + ], + "osx_x86_64": [ + "@platforms//cpu:x86_64", + "@platforms//os:osx", + ], + "windows_aarch64": [ + "@platforms//cpu:aarch64", + "@platforms//os:windows", + ], + "windows_x86_64": [ + "@platforms//cpu:x86_64", + "@platforms//os:windows", + ], + }, ) diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 3d205a23c4..231e8cab41 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -1051,6 +1051,10 @@ def _test_pipstar_platforms(env): default = [ _default( platform = "{}_{}".format(os, cpu), + constraint_values = [ + "@platforms//os:{}".format(os), + "@platforms//cpu:{}".format(cpu), + ], ) for os, cpu in [ ("linux", "x86_64"), diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 71ca811fee..0fbcd4e7a6 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -419,10 +419,16 @@ def _test_config_settings_exist_legacy(env): alias = _mock_alias(available_config_settings), config_setting = _mock_config_setting(available_config_settings), ), - target_platforms = [ - "linux_aarch64", - "linux_x86_64", - ], + platform_constraint_values = { + "linux_aarch64": [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + "linux_x86_64": [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + }, ) got_aliases = multiplatform_whl_aliases( @@ -448,19 +454,39 @@ def _test_config_settings_exist(env): "any": {}, "macosx_11_0_arm64": { "osx_versions": [(11, 0)], - "target_platforms": ["osx_aarch64"], + "platform_constraint_values": { + "osx_aarch64": [ + "@platforms//cpu:aarch64", + "@platforms//os:osx", + ], + }, }, "manylinux_2_17_x86_64": { "glibc_versions": [(2, 17), (2, 18)], - "target_platforms": ["linux_x86_64"], + "platform_constraint_values": { + "linux_x86_64": [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + }, }, "manylinux_2_18_x86_64": { "glibc_versions": [(2, 17), (2, 18)], - "target_platforms": ["linux_x86_64"], + "platform_constraint_values": { + "linux_x86_64": [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + }, }, "musllinux_1_1_aarch64": { "muslc_versions": [(1, 2), (1, 1), (1, 0)], - "target_platforms": ["linux_aarch64"], + "platform_constraint_values": { + "linux_aarch64": [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + }, }, }.items(): aliases = { diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 416d50bd80..c262ed6823 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -93,6 +93,12 @@ def _test_bzlmod_aliases(env): }, }, extra_hub_aliases = {"bar_baz": ["foo"]}, + platform_constraint_values = { + "linux_x86_64": [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + }, ) want_key = "bar_baz/BUILD.bazel" @@ -130,8 +136,13 @@ load("@rules_python//python/private/pypi:config_settings.bzl", "config_settings" config_settings( name = "config_settings", + platform_constraint_values = { + "linux_x86_64": [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + }, python_versions = ["3.2"], - target_platforms = ["linux_x86_64"], visibility = ["//:__subpackages__"], )""", ) From c4543cd193752d0248226dcd07cc027e63ed7b8b Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 19 Jun 2025 23:28:52 -0700 Subject: [PATCH 38/43] fix(toolchains): use posix-compatible exec -a alternative (#3010) The `exec -a` command doesn't work in dash, the default shell for Ubuntu/debian. To work around, use `sh -c`, which is posix and dash compatible. This allows changing the argv0 while invoking a different command. Also adds a test to verify the the runtime_env toolchain works with bootstrap script. Fixes https://github.com/bazel-contrib/rules_python/issues/3009 --- .../runtime_env_toolchain_interpreter.sh | 13 ++++++----- tests/runtime_env_toolchain/BUILD.bazel | 23 +++++++++++++++++++ .../toolchain_runs_test.py | 9 ++++++++ tests/support/sh_py_run_test.bzl | 6 +++++ 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/python/private/runtime_env_toolchain_interpreter.sh b/python/private/runtime_env_toolchain_interpreter.sh index 7b3ec598b2..dd4d648d12 100755 --- a/python/private/runtime_env_toolchain_interpreter.sh +++ b/python/private/runtime_env_toolchain_interpreter.sh @@ -71,14 +71,15 @@ if [ -e "$self_dir/pyvenv.cfg" ] || [ -e "$self_dir/../pyvenv.cfg" ]; then if [ ! -e "$PYTHON_BIN" ]; then die "ERROR: Python interpreter does not exist: $PYTHON_BIN" fi - # PYTHONEXECUTABLE is also used because `exec -a` doesn't fully trick the - # pyenv wrappers. + # PYTHONEXECUTABLE is also used because switching argv0 doesn't fully trick + # the pyenv wrappers. # NOTE: The PYTHONEXECUTABLE envvar only works for non-Mac starting in Python 3.11 export PYTHONEXECUTABLE="$venv_bin" - # Python looks at argv[0] to determine sys.executable, so use exec -a - # to make it think it's the venv's binary, not the actual one invoked. - # NOTE: exec -a isn't strictly posix-compatible, but very widespread - exec -a "$venv_bin" "$PYTHON_BIN" "$@" + # Python looks at argv[0] to determine sys.executable, so set that to the venv + # binary, not the actual one invoked. + # NOTE: exec -a would be simpler, but isn't posix-compatible, and dash shell + # (Ubuntu/debian default) doesn't support it; see #3009. + exec sh -c "$PYTHON_BIN \$@" "$venv_bin" "$@" else exec "$PYTHON_BIN" "$@" fi diff --git a/tests/runtime_env_toolchain/BUILD.bazel b/tests/runtime_env_toolchain/BUILD.bazel index 2f82d204ff..f1bda251f9 100644 --- a/tests/runtime_env_toolchain/BUILD.bazel +++ b/tests/runtime_env_toolchain/BUILD.bazel @@ -40,3 +40,26 @@ py_reconfig_test( tags = ["no-remote-exec"], deps = ["//python/runfiles"], ) + +py_reconfig_test( + name = "bootstrap_script_test", + srcs = ["toolchain_runs_test.py"], + bootstrap_impl = "script", + data = [ + "//tests/support:current_build_settings", + ], + extra_toolchains = [ + "//python/runtime_env_toolchains:all", + # Necessary for RBE CI + CC_TOOLCHAIN, + ], + main = "toolchain_runs_test.py", + # With bootstrap=script, the build version must match the runtime version + # because the venv has the version in the lib/site-packages dir name. + python_version = PYTHON_VERSION, + # Our RBE has Python 3.6, which is too old for the language features + # we use now. Using the runtime-env toolchain on RBE is pretty + # questionable anyways. + tags = ["no-remote-exec"], + deps = ["//python/runfiles"], +) diff --git a/tests/runtime_env_toolchain/toolchain_runs_test.py b/tests/runtime_env_toolchain/toolchain_runs_test.py index 7be2472e8b..c66b0bbd8a 100644 --- a/tests/runtime_env_toolchain/toolchain_runs_test.py +++ b/tests/runtime_env_toolchain/toolchain_runs_test.py @@ -1,6 +1,7 @@ import json import pathlib import platform +import sys import unittest from python.runfiles import runfiles @@ -23,6 +24,14 @@ def test_ran(self): settings["interpreter"]["short_path"], ) + if settings["bootstrap_impl"] == "script": + # Verify we're running in a venv + self.assertNotEqual(sys.prefix, sys.base_prefix) + # .venv/ occurs for a build-time venv. + # For a runtime created venv, it goes into a temp dir, so + # look for the /bin/ dir as an indicator. + self.assertRegex(sys.executable, r"[.]venv/|/bin/") + if __name__ == "__main__": unittest.main() diff --git a/tests/support/sh_py_run_test.bzl b/tests/support/sh_py_run_test.bzl index 69141fe8a4..49445ed304 100644 --- a/tests/support/sh_py_run_test.bzl +++ b/tests/support/sh_py_run_test.bzl @@ -135,6 +135,7 @@ def _current_build_settings_impl(ctx): ctx.actions.write( output = info, content = json.encode({ + "bootstrap_impl": ctx.attr._bootstrap_impl_flag[config_common.FeatureFlagInfo].value, "interpreter": { "short_path": runtime.interpreter.short_path if runtime.interpreter else None, }, @@ -153,6 +154,11 @@ Writes information about the current build config to JSON for testing. This is so tests can verify information about the build config used for them. """, implementation = _current_build_settings_impl, + attrs = { + "_bootstrap_impl_flag": attr.label( + default = "//python/config_settings:bootstrap_impl", + ), + }, toolchains = [ TARGET_TOOLCHAIN_TYPE, ], From b924c43e0fadc78fe8de7d91c318c5299c8ab68b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Jun 2025 23:56:13 -0700 Subject: [PATCH 39/43] build(deps): bump urllib3 from 2.4.0 to 2.5.0 in /tools/publish (#3008) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.4.0 to 2.5.0.
Release notes

Sourced from urllib3's releases.

2.5.0

🚀 urllib3 is fundraising for HTTP/2 support

urllib3 is raising ~$40,000 USD to release HTTP/2 support and ensure long-term sustainable maintenance of the project after a sharp decline in financial support. If your company or organization uses Python and would benefit from HTTP/2 support in Requests, pip, cloud SDKs, and thousands of other projects please consider contributing financially to ensure HTTP/2 support is developed sustainably and maintained for the long-haul.

Thank you for your support.

Security issues

urllib3 2.5.0 fixes two moderate security issues:

  • Pool managers now properly control redirects when retries is passed — CVE-2025-50181 reported by @​sandumjacob (5.3 Medium, GHSA-pq67-6m6q-mj2v)
  • Redirects are now controlled by urllib3 in the Node.js runtime — CVE-2025-50182 (5.3 Medium, GHSA-48p4-8xcf-vxj5)

Features

  • Added support for the compression.zstd module that is new in Python 3.14. See PEP 784 for more information. (#3610)
  • Added support for version 0.5 of hatch-vcs (#3612)

Bugfixes

  • Raised exception for HTTPResponse.shutdown on a connection already released to the pool. (#3581)
  • Fixed incorrect CONNECT statement when using an IPv6 proxy with connection_from_host. Previously would not be wrapped in []. (#3615)
Changelog

Sourced from urllib3's changelog.

2.5.0 (2025-06-18)

Features

  • Added support for the compression.zstd module that is new in Python 3.14. See PEP 784 <https://peps.python.org/pep-0784/>_ for more information. ([#3610](https://github.com/urllib3/urllib3/issues/3610) <https://github.com/urllib3/urllib3/issues/3610>__)
  • Added support for version 0.5 of hatch-vcs ([#3612](https://github.com/urllib3/urllib3/issues/3612) <https://github.com/urllib3/urllib3/issues/3612>__)

Bugfixes

  • Fixed a security issue where restricting the maximum number of followed redirects at the urllib3.PoolManager level via the retries parameter did not work.
  • Made the Node.js runtime respect redirect parameters such as retries and redirects.
  • Raised exception for HTTPResponse.shutdown on a connection already released to the pool. ([#3581](https://github.com/urllib3/urllib3/issues/3581) <https://github.com/urllib3/urllib3/issues/3581>__)
  • Fixed incorrect CONNECT statement when using an IPv6 proxy with connection_from_host. Previously would not be wrapped in []. ([#3615](https://github.com/urllib3/urllib3/issues/3615) <https://github.com/urllib3/urllib3/issues/3615>__)
Commits
  • aaab4ec Release 2.5.0
  • 7eb4a2a Merge commit from fork
  • f05b132 Merge commit from fork
  • d03fe32 Fix HTTP tunneling with IPv6 in older Python versions
  • 11661e9 Bump github/codeql-action from 3.28.0 to 3.29.0 (#3624)
  • 6a0ecc6 Update v2 migration guide to 2.4.0 (#3621)
  • 8e32e60 Raise exception for shutdown on a connection already released to the pool (#3...
  • 9996e0f Fix emscripten CI for Chrome 137+ (#3599)
  • 4fd1a99 Bump RECENT_DATE (#3617)
  • c4b5917 Add support for the new compression.zstd module in Python 3.14 (#3611)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=urllib3&package-manager=pip&previous-version=2.4.0&new-version=2.5.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/bazel-contrib/rules_python/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/publish/requirements_darwin.txt | 6 +++--- tools/publish/requirements_linux.txt | 6 +++--- tools/publish/requirements_universal.txt | 6 +++--- tools/publish/requirements_windows.txt | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/publish/requirements_darwin.txt b/tools/publish/requirements_darwin.txt index af5bad246d..58973acb6f 100644 --- a/tools/publish/requirements_darwin.txt +++ b/tools/publish/requirements_darwin.txt @@ -202,9 +202,9 @@ twine==5.1.1 \ --hash=sha256:215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997 \ --hash=sha256:9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db # via -r tools/publish/requirements.in -urllib3==2.4.0 \ - --hash=sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466 \ - --hash=sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813 +urllib3==2.5.0 \ + --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \ + --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc # via # requests # twine diff --git a/tools/publish/requirements_linux.txt b/tools/publish/requirements_linux.txt index b2e9ccf5ab..73edfce02f 100644 --- a/tools/publish/requirements_linux.txt +++ b/tools/publish/requirements_linux.txt @@ -318,9 +318,9 @@ twine==5.1.1 \ --hash=sha256:215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997 \ --hash=sha256:9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db # via -r tools/publish/requirements.in -urllib3==2.4.0 \ - --hash=sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466 \ - --hash=sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813 +urllib3==2.5.0 \ + --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \ + --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc # via # requests # twine diff --git a/tools/publish/requirements_universal.txt b/tools/publish/requirements_universal.txt index 8a7426e517..c080f1d7de 100644 --- a/tools/publish/requirements_universal.txt +++ b/tools/publish/requirements_universal.txt @@ -322,9 +322,9 @@ twine==5.1.1 \ --hash=sha256:215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997 \ --hash=sha256:9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db # via -r tools/publish/requirements.in -urllib3==2.4.0 \ - --hash=sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466 \ - --hash=sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813 +urllib3==2.5.0 \ + --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \ + --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc # via # requests # twine diff --git a/tools/publish/requirements_windows.txt b/tools/publish/requirements_windows.txt index 11017aa4f9..a4d5e3e25d 100644 --- a/tools/publish/requirements_windows.txt +++ b/tools/publish/requirements_windows.txt @@ -206,9 +206,9 @@ twine==5.1.1 \ --hash=sha256:215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997 \ --hash=sha256:9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db # via -r tools/publish/requirements.in -urllib3==2.4.0 \ - --hash=sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466 \ - --hash=sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813 +urllib3==2.5.0 \ + --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \ + --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc # via # requests # twine From 6fd4c0bdc9eca48449c1f2b77a44f59a62a88dde Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 20 Jun 2025 16:10:13 +0900 Subject: [PATCH 40/43] feat: support arbitrary target_settings in our platforms 3/n (#2990) With this PR we can support arbitrary target settings instead of just plain `constraint_values`. We still have custom logic to ensure that all of the tests pass. However, the plan is to remove those tests once we have simplified the wheel selection mechanisms and the `pkg_aliases` macro. I.e. if we have at most 1 wheel per platform that the `pypi` bzlmod extension passes to the `pkg_aliases` macro, then we can just have a simple `selects.with_or` where we list out all of the target platform values. This PR may result in us creating more targets but that is the price that we have to pay if we want to do this incrementally. Work towards #2747 Work towards #2548 Work towards #260 Co-authored-by: Richard Levasseur --- CHANGELOG.md | 2 +- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/config_settings.bzl | 41 ++++++++++++++++--- python/private/pypi/extension.bzl | 26 +++++++----- python/private/pypi/hub_repository.bzl | 4 +- python/private/pypi/render_pkg_aliases.bzl | 14 +++---- .../config_settings/config_settings_tests.bzl | 2 +- tests/pypi/extension/extension_tests.bzl | 8 ++-- tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 23 +++++++---- .../render_pkg_aliases_test.bzl | 4 +- 10 files changed, 83 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da3dcc8efc..f2fa98f73f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,7 +64,7 @@ END_UNRELEASED_TEMPLATE ### Added * (pypi) To configure the environment for `requirements.txt` evaluation, use the newly added developer preview of the `pip.default` tag class. Only `rules_python` and root modules can use - this feature. You can also configure `constraint_values` using `pip.default`. + this feature. You can also configure custom `config_settings` using `pip.default`. {#v0-0-0-removed} ### Removed diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index b569b2217c..2666197786 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -64,6 +64,7 @@ bzl_library( deps = [ ":flags_bzl", "//python/private:flags_bzl", + "@bazel_skylib//lib:selects", ], ) diff --git a/python/private/pypi/config_settings.bzl b/python/private/pypi/config_settings.bzl index 7edc578d7a..f4826007f8 100644 --- a/python/private/pypi/config_settings.bzl +++ b/python/private/pypi/config_settings.bzl @@ -70,6 +70,7 @@ suffix. ::: """ +load("@bazel_skylib//lib:selects.bzl", "selects") load("//python/private:flags.bzl", "LibcFlag") load(":flags.bzl", "INTERNAL_FLAGS", "UniversalWhlFlag") @@ -112,7 +113,7 @@ def config_settings( muslc_versions = [], osx_versions = [], name = None, - platform_constraint_values = {}, + platform_config_settings = {}, **kwargs): """Generate all of the pip config settings. @@ -126,7 +127,7 @@ def config_settings( configure config settings for. osx_versions (list[str]): The list of OSX OS versions to configure config settings for. - platform_constraint_values: {type}`dict[str, list[str]]` the constraint + platform_config_settings: {type}`dict[str, list[str]]` the constraint values to use instead of the default ones. Key are platform names (a human-friendly platform string). Values are lists of `constraint_value` label strings. @@ -142,13 +143,24 @@ def config_settings( # TODO @aignas 2025-06-15: allowing universal2 and platform specific wheels in one # closure is making things maybe a little bit too complicated. "osx_universal2": ["@platforms//os:osx"], - } | platform_constraint_values + } | platform_config_settings for python_version in python_versions: - for platform_name, constraint_values in target_platforms.items(): + for platform_name, config_settings in target_platforms.items(): suffix = "_{}".format(platform_name) if platform_name else "" os, _, cpu = platform_name.partition("_") + # We parse the target settings and if there is a "platforms//os" or + # "platforms//cpu" value in here, we also add it into the constraint_values + # + # this is to ensure that we can still pass all of the unit tests for config + # setting specialization. + constraint_values = [] + for setting in config_settings: + setting_label = Label(setting) + if setting_label.repo_name == "platforms" and setting_label.package in ["os", "cpu"]: + constraint_values.append(setting) + _dist_config_settings( suffix = suffix, plat_flag_values = _plat_flag_values( @@ -158,6 +170,7 @@ def config_settings( glibc_versions = glibc_versions, muslc_versions = muslc_versions, ), + config_settings = config_settings, constraint_values = constraint_values, python_version = python_version, **kwargs @@ -318,7 +331,7 @@ def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions): return ret -def _dist_config_setting(*, name, compatible_with = None, native = native, **kwargs): +def _dist_config_setting(*, name, compatible_with = None, selects = selects, native = native, config_settings = None, **kwargs): """A macro to create a target for matching Python binary and source distributions. Args: @@ -327,6 +340,12 @@ def _dist_config_setting(*, name, compatible_with = None, native = native, **kwa compatible with the given dist config setting. For example, if only non-freethreaded python builds are allowed, add FLAGS._is_py_freethreaded_no here. + config_settings: {type}`list[str | Label]` the list of target settings that must + be matched before we try to evaluate the config_setting that we may create in + this function. + selects (struct): The struct containing config_setting_group function + to use for creating config setting groups. Can be overridden for unit tests + reasons. native (struct): The struct containing alias and config_setting rules to use for creating the objects. Can be overridden for unit tests reasons. @@ -346,4 +365,14 @@ def _dist_config_setting(*, name, compatible_with = None, native = native, **kwa ) name = dist_config_setting_name - native.config_setting(name = name, **kwargs) + # first define the config setting that has all of the constraint values + _name = "_" + name + native.config_setting( + name = _name, + **kwargs + ) + selects.config_setting_group( + name = name, + match_all = config_settings + [_name], + visibility = kwargs.get("visibility"), + ) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 78511b4c27..a0095f8f15 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -372,7 +372,7 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net ), ) -def _configure(config, *, platform, os_name, arch_name, constraint_values, env = {}, override = False): +def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, override = False): """Set the value in the config if the value is provided""" config.setdefault("platforms", {}) if platform: @@ -387,7 +387,7 @@ def _configure(config, *, platform, os_name, arch_name, constraint_values, env = name = platform.replace("-", "_").lower(), os_name = os_name, arch_name = arch_name, - constraint_values = constraint_values, + config_settings = config_settings, env = env, ) else: @@ -414,7 +414,7 @@ def _create_config(defaults): arch_name = cpu, os_name = "linux", platform = "linux_{}".format(cpu), - constraint_values = [ + config_settings = [ "@platforms//os:linux", "@platforms//cpu:{}".format(cpu), ], @@ -431,7 +431,7 @@ def _create_config(defaults): # See https://endoflife.date/macos os_name = "osx", platform = "osx_{}".format(cpu), - constraint_values = [ + config_settings = [ "@platforms//os:osx", "@platforms//cpu:{}".format(cpu), ], @@ -443,7 +443,7 @@ def _create_config(defaults): arch_name = "x86_64", os_name = "windows", platform = "windows_x86_64", - constraint_values = [ + config_settings = [ "@platforms//os:windows", "@platforms//cpu:x86_64", ], @@ -513,7 +513,7 @@ You cannot use both the additive_build_content and additive_build_content_file a _configure( defaults, arch_name = tag.arch_name, - constraint_values = tag.constraint_values, + config_settings = tag.config_settings, env = tag.env, os_name = tag.os_name, platform = tag.platform, @@ -693,9 +693,9 @@ You cannot use both the additive_build_content and additive_build_content_file a } for hub_name, extra_whl_aliases in extra_aliases.items() }, - platform_constraint_values = { + platform_config_settings = { hub_name: { - platform_name: sorted([str(Label(cv)) for cv in p.constraint_values]) + platform_name: sorted([str(Label(cv)) for cv in p.config_settings]) for platform_name, p in config.platforms.items() } for hub_name in hub_whl_map @@ -790,7 +790,7 @@ def _pip_impl(module_ctx): for key, values in whl_map.items() }, packages = mods.exposed_packages.get(hub_name, []), - platform_constraint_values = mods.platform_constraint_values.get(hub_name, {}), + platform_config_settings = mods.platform_config_settings.get(hub_name, {}), groups = mods.hub_group_map.get(hub_name), ) @@ -812,10 +812,11 @@ Either this or {attr}`env` `platform_machine` key should be specified. ::: """, ), - "constraint_values": attr.label_list( + "config_settings": attr.label_list( mandatory = True, doc = """\ -The constraint_values to use in select statements. +The list of labels to `config_setting` targets that need to be matched for the platform to be +selected. """, ), "os_name": attr.string( @@ -1145,6 +1146,9 @@ terms used in this extension. [environment_markers]: https://packaging.python.org/en/latest/specifications/dependency-specifiers/#environment-markers ::: + +:::{versionadded} VERSION_NEXT_FEATURE +::: """, ), "override": _override_tag, diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 4398d7b597..75f3ec98d7 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -34,7 +34,7 @@ def _impl(rctx): }, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, - platform_constraint_values = rctx.attr.platform_constraint_values, + platform_config_settings = rctx.attr.platform_config_settings, ) for path, contents in aliases.items(): rctx.file(path, contents) @@ -84,7 +84,7 @@ hub_repository = repository_rule( The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys. """, ), - "platform_constraint_values": attr.string_list_dict( + "platform_config_settings": attr.string_list_dict( doc = "The constraint values for each platform name. The values are string canonical string Label representations", mandatory = False, ), diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 267d7ce85d..e743fc20f7 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -155,14 +155,14 @@ def _major_minor_versions(python_versions): # Use a dict as a simple set return sorted({_major_minor(v): None for v in python_versions}) -def render_multiplatform_pkg_aliases(*, aliases, platform_constraint_values = {}, **kwargs): +def render_multiplatform_pkg_aliases(*, aliases, platform_config_settings = {}, **kwargs): """Render the multi-platform pkg aliases. Args: aliases: dict[str, list(whl_config_setting)] A list of aliases that will be transformed from ones having `filename` to ones having `config_setting`. - platform_constraint_values: {type}`dict[str, list[str]]` contains all of the - target platforms and their appropriate `constraint_values`. + platform_config_settings: {type}`dict[str, list[str]]` contains all of the + target platforms and their appropriate `target_settings`. **kwargs: extra arguments passed to render_pkg_aliases. Returns: @@ -189,20 +189,20 @@ def render_multiplatform_pkg_aliases(*, aliases, platform_constraint_values = {} muslc_versions = flag_versions.get("muslc_versions", []), osx_versions = flag_versions.get("osx_versions", []), python_versions = _major_minor_versions(flag_versions.get("python_versions", [])), - platform_constraint_values = platform_constraint_values, + platform_config_settings = platform_config_settings, visibility = ["//:__subpackages__"], ) return contents -def _render_config_settings(platform_constraint_values, **kwargs): +def _render_config_settings(platform_config_settings, **kwargs): return """\ load("@rules_python//python/private/pypi:config_settings.bzl", "config_settings") {}""".format(render.call( "config_settings", name = repr("config_settings"), - platform_constraint_values = render.dict( - platform_constraint_values, + platform_config_settings = render.dict( + platform_config_settings, value_repr = render.list, ), **_repr_dict(value_repr = render.list, **kwargs) diff --git a/tests/pypi/config_settings/config_settings_tests.bzl b/tests/pypi/config_settings/config_settings_tests.bzl index 9551d42d10..a15f6b4d32 100644 --- a/tests/pypi/config_settings/config_settings_tests.bzl +++ b/tests/pypi/config_settings/config_settings_tests.bzl @@ -657,7 +657,7 @@ def config_settings_test_suite(name): # buildifier: disable=function-docstring glibc_versions = [(2, 14), (2, 17)], muslc_versions = [(1, 1)], osx_versions = [(10, 9), (11, 0)], - platform_constraint_values = { + platform_config_settings = { "linux_aarch64": [ "@platforms//cpu:aarch64", "@platforms//os:linux", diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 231e8cab41..146293ee8d 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -78,19 +78,17 @@ def _parse_modules(env, enable_pipstar = 0, **kwargs): def _default( arch_name = None, - constraint_values = None, + config_settings = None, os_name = None, platform = None, - target_settings = None, env = None, whl_limit = None, whl_platforms = None): return struct( arch_name = arch_name, - constraint_values = constraint_values, os_name = os_name, platform = platform, - target_settings = target_settings, + config_settings = config_settings, env = env or {}, whl_platforms = whl_platforms, whl_limit = whl_limit, @@ -1051,7 +1049,7 @@ def _test_pipstar_platforms(env): default = [ _default( platform = "{}_{}".format(os, cpu), - constraint_values = [ + config_settings = [ "@platforms//os:{}".format(os), "@platforms//cpu:{}".format(cpu), ], diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 0fbcd4e7a6..123ee725f8 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -392,6 +392,9 @@ _tests.append(_test_multiplatform_whl_aliases_filename_versioned) def _mock_alias(container): return lambda name, **kwargs: container.append(name) +def _mock_config_setting_group(container): + return lambda name, **kwargs: container.append(name) + def _mock_config_setting(container): def _inner(name, flag_values = None, constraint_values = None, **_): if flag_values or constraint_values: @@ -417,9 +420,12 @@ def _test_config_settings_exist_legacy(env): python_versions = ["3.11"], native = struct( alias = _mock_alias(available_config_settings), - config_setting = _mock_config_setting(available_config_settings), + config_setting = _mock_config_setting([]), ), - platform_constraint_values = { + selects = struct( + config_setting_group = _mock_config_setting_group(available_config_settings), + ), + platform_config_settings = { "linux_aarch64": [ "@platforms//cpu:aarch64", "@platforms//os:linux", @@ -454,7 +460,7 @@ def _test_config_settings_exist(env): "any": {}, "macosx_11_0_arm64": { "osx_versions": [(11, 0)], - "platform_constraint_values": { + "platform_config_settings": { "osx_aarch64": [ "@platforms//cpu:aarch64", "@platforms//os:osx", @@ -463,7 +469,7 @@ def _test_config_settings_exist(env): }, "manylinux_2_17_x86_64": { "glibc_versions": [(2, 17), (2, 18)], - "platform_constraint_values": { + "platform_config_settings": { "linux_x86_64": [ "@platforms//cpu:x86_64", "@platforms//os:linux", @@ -472,7 +478,7 @@ def _test_config_settings_exist(env): }, "manylinux_2_18_x86_64": { "glibc_versions": [(2, 17), (2, 18)], - "platform_constraint_values": { + "platform_config_settings": { "linux_x86_64": [ "@platforms//cpu:x86_64", "@platforms//os:linux", @@ -481,7 +487,7 @@ def _test_config_settings_exist(env): }, "musllinux_1_1_aarch64": { "muslc_versions": [(1, 2), (1, 1), (1, 0)], - "platform_constraint_values": { + "platform_config_settings": { "linux_aarch64": [ "@platforms//cpu:aarch64", "@platforms//os:linux", @@ -500,7 +506,10 @@ def _test_config_settings_exist(env): python_versions = ["3.11"], native = struct( alias = _mock_alias(available_config_settings), - config_setting = _mock_config_setting(available_config_settings), + config_setting = _mock_config_setting([]), + ), + selects = struct( + config_setting_group = _mock_config_setting_group(available_config_settings), ), **kwargs ) diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index c262ed6823..ad7f36aed6 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -93,7 +93,7 @@ def _test_bzlmod_aliases(env): }, }, extra_hub_aliases = {"bar_baz": ["foo"]}, - platform_constraint_values = { + platform_config_settings = { "linux_x86_64": [ "@platforms//os:linux", "@platforms//cpu:x86_64", @@ -136,7 +136,7 @@ load("@rules_python//python/private/pypi:config_settings.bzl", "config_settings" config_settings( name = "config_settings", - platform_constraint_values = { + platform_config_settings = { "linux_x86_64": [ "@platforms//os:linux", "@platforms//cpu:x86_64", From 8f8c5b9ba7c7f68f37b7687ebb22931cff075241 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 20 Jun 2025 00:42:42 -0700 Subject: [PATCH 41/43] docs: fix various typos and improve grammar (#3015) Used Jules to do some copy editing. It found a variety of typos. * Consistently use backticks for rules_python, WORKSPACE, and some other terms * Various simple typo and grammar fixes --------- Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com> --- docs/README.md | 16 +-- docs/_includes/py_console_script_binary.md | 25 ++-- docs/coverage.md | 4 +- docs/devguide.md | 28 ++-- docs/environment-variables.md | 24 ++-- docs/extending.md | 12 +- docs/gazelle.md | 4 +- docs/getting-started.md | 10 +- docs/glossary.md | 10 +- docs/index.md | 22 ++-- docs/precompiling.md | 40 +++--- docs/pypi/circular-dependencies.md | 16 +-- docs/pypi/download-workspace.md | 12 +- docs/pypi/download.md | 68 +++++----- docs/pypi/index.md | 6 +- docs/pypi/lock.md | 11 +- docs/pypi/patch.md | 4 +- docs/pypi/use.md | 42 +++--- docs/repl.md | 2 +- docs/support.md | 22 ++-- docs/toolchains.md | 142 ++++++++++----------- 21 files changed, 263 insertions(+), 257 deletions(-) diff --git a/docs/README.md b/docs/README.md index d98be41232..456f1cfd64 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,14 +1,14 @@ # rules_python Sphinx docs generation The docs for rules_python are generated using a combination of Sphinx, Bazel, -and Readthedocs.org. The Markdown files in source control are unlikely to render +and Read the Docs. The Markdown files in source control are unlikely to render properly without the Sphinx processing step because they rely on Sphinx and MyST-specific Markdown functionality. The actual sources that Sphinx consumes are in this directory, with Stardoc -generating additional sources or Sphinx. +generating additional sources for Sphinx. -Manually building the docs isn't necessary -- readthedocs.org will +Manually building the docs isn't necessary -- Read the Docs will automatically build and deploy them when commits are pushed to the repo. ## Generating docs for development @@ -31,8 +31,8 @@ equivalent bazel command if desired. ### Installing ibazel The `ibazel` tool can be used to automatically rebuild the docs as you -development them. See the [ibazel docs](https://github.com/bazelbuild/bazel-watcher) for -how to install it. The quick start for linux is: +develop them. See the [ibazel docs](https://github.com/bazelbuild/bazel-watcher) for +how to install it. The quick start for Linux is: ``` sudo apt install npm @@ -57,9 +57,9 @@ docs/. The Sphinx configuration is `docs/conf.py`. See https://www.sphinx-doc.org/ for details about the configuration file. -## Readthedocs configuration +## Read the Docs configuration -There's two basic parts to the readthedocs configuration: +There's two basic parts to the Read the Docs configuration: * `.readthedocs.yaml`: This configuration file controls most settings, such as the OS version used to build, Python version, dependencies, what Bazel @@ -69,4 +69,4 @@ There's two basic parts to the readthedocs configuration: controls additional settings such as permissions, what versions are published, when to publish changes, etc. -For more readthedocs configuration details, see docs.readthedocs.io. +For more Read the Docs configuration details, see docs.readthedocs.io. diff --git a/docs/_includes/py_console_script_binary.md b/docs/_includes/py_console_script_binary.md index d327091630..cae9f9f2f5 100644 --- a/docs/_includes/py_console_script_binary.md +++ b/docs/_includes/py_console_script_binary.md @@ -1,8 +1,8 @@ This rule is to make it easier to generate `console_script` entry points as per Python [specification]. -Generate a `py_binary` target for a particular console_script `entry_point` -from a PyPI package, e.g. for creating an executable `pylint` target use: +Generate a `py_binary` target for a particular `console_script` entry_point +from a PyPI package, e.g. for creating an executable `pylint` target, use: ```starlark load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") @@ -12,11 +12,12 @@ py_console_script_binary( ) ``` -#### Specifying extra dependencies +#### Specifying extra dependencies You can also specify extra dependencies and the -exact script name you want to call. It is useful for tools like `flake8`, `pylint`, -`pytest`, which have plugin discovery methods and discover dependencies from the -PyPI packages available in the `PYTHONPATH`. +exact script name you want to call. This is useful for tools like `flake8`, +`pylint`, and `pytest`, which have plugin discovery methods and discover +dependencies from the PyPI packages available in the `PYTHONPATH`. + ```starlark load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") @@ -44,13 +45,13 @@ load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_cons py_console_script_binary( name = "yamllint", pkg = "@pip//yamllint", - python_version = "3.9" + python_version = "3.9", ) ``` #### Adding a Shebang Line -You can specify a shebang line for the generated binary, useful for Unix-like +You can specify a shebang line for the generated binary. This is useful for Unix-like systems where the shebang line determines which interpreter is used to execute the script, per [PEP441]: @@ -70,12 +71,12 @@ Python interpreter is available in the environment. #### Using a specific Python Version directly from a Toolchain :::{deprecated} 1.1.0 -The toolchain specific `py_binary` and `py_test` symbols are aliases to the regular rules. -i.e. Deprecated `load("@python_versions//3.11:defs.bzl", "py_binary")` and `load("@python_versions//3.11:defs.bzl", "py_test")` +The toolchain-specific `py_binary` and `py_test` symbols are aliases to the regular rules. +For example, `load("@python_versions//3.11:defs.bzl", "py_binary")` and `load("@python_versions//3.11:defs.bzl", "py_test")` are deprecated. -You should instead specify the desired python version with `python_version`; see above example. +You should instead specify the desired Python version with `python_version`; see the example above. ::: -Alternatively, the [`py_console_script_binary.binary_rule`] arg can be passed +Alternatively, the {obj}`py_console_script_binary.binary_rule` arg can be passed the version-bound `py_binary` symbol, or any other `py_binary`-compatible rule of your choosing: ```starlark diff --git a/docs/coverage.md b/docs/coverage.md index 3e0e67368c..3c7d9e0cfc 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -9,7 +9,7 @@ when configuring toolchains. ## Enabling `rules_python` coverage support Enabling the coverage support bundled with `rules_python` just requires setting an -argument when registerting toolchains. +argument when registering toolchains. For Bzlmod: @@ -32,7 +32,7 @@ python_register_toolchains( This will implicitly add the version of `coverage` bundled with `rules_python` to the dependencies of `py_test` rules when `bazel coverage` is run. If a target already transitively depends on a different version of -`coverage`, then behavior is undefined -- it is undefined which version comes +`coverage`, then the behavior is undefined -- it is undefined which version comes first in the import path. If you find yourself in this situation, then you'll need to manually configure coverage (see below). ::: diff --git a/docs/devguide.md b/docs/devguide.md index f233611cad..345907b374 100644 --- a/docs/devguide.md +++ b/docs/devguide.md @@ -1,7 +1,7 @@ # Dev Guide -This document covers tips and guidance for working on the rules_python code -base. A primary audience for it is first time contributors. +This document covers tips and guidance for working on the `rules_python` code +base. Its primary audience is first-time contributors. ## Running tests @@ -12,8 +12,8 @@ bazel test //... ``` And it will run all the tests it can find. The first time you do this, it will -probably take long time because various dependencies will need to be downloaded -and setup. Subsequent runs will be faster, but there are many tests, and some of +probably take a long time because various dependencies will need to be downloaded +and set up. Subsequent runs will be faster, but there are many tests, and some of them are slow. If you're working on a particular area of code, you can run just the tests in those directories instead, which can speed up your edit-run cycle. @@ -22,14 +22,14 @@ the tests in those directories instead, which can speed up your edit-run cycle. Most code should have tests of some sort. This helps us have confidence that refactors didn't break anything and that releases won't have regressions. -We don't require 100% test coverage, testing certain Bazel functionality is +We don't require 100% test coverage; testing certain Bazel functionality is difficult, and some edge cases are simply too hard to test or not worth the extra complexity. We try to judiciously decide when not having tests is a good idea. Tests go under `tests/`. They are loosely organized into directories for the particular subsystem or functionality they are testing. If an existing directory -doesn't seem like a good match for the functionality being testing, then it's +doesn't seem like a good match for the functionality being tested, then it's fine to create a new directory. Re-usable test helpers and support code go in `tests/support`. Tests don't need @@ -72,9 +72,9 @@ the rule. To have it support setting a new flag: An integration test is one that runs a separate Bazel instance inside the test. These tests are discouraged unless absolutely necessary because they are slow, -require much memory and CPU, and are generally harder to debug. Integration -tests are reserved for things that simple can't be tested otherwise, or for -simple high level verification tests. +require a lot of memory and CPU, and are generally harder to debug. Integration +tests are reserved for things that simply can't be tested otherwise, or for +simple high-level verification tests. Integration tests live in `tests/integration`. When possible, add to an existing integration test. @@ -98,9 +98,9 @@ integration test. ## Updating tool dependencies -It's suggested to routinely update the tool versions within our repo - some of the -tools are using requirement files compiled by `uv` and others use other means. In order -to have everything self-documented, we have a special target - -`//private:requirements.update`, which uses `rules_multirun` to run in sequence all -of the requirement updating scripts in one go. This can be done once per release as +It's suggested to routinely update the tool versions within our repo. Some of the +tools are using requirement files compiled by `uv`, and others use other means. In order +to have everything self-documented, we have a special target, +`//private:requirements.update`, which uses `rules_multirun` to run all +of the requirement-updating scripts in sequence in one go. This can be done once per release as we prepare for releases. diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 8a51bcbfd2..9a8c1dfe99 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -5,16 +5,16 @@ This variable allows for additional arguments to be provided to the Python interpreter at bootstrap time when the `bash` bootstrap is used. If `RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS` were provided as `-Xaaa`, then the command -would be; +would be: ``` python -Xaaa /path/to/file.py ``` This feature is likely to be useful for the integration of debuggers. For example, -it would be possible to configure the `RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS` to -be set to `/path/to/debugger.py --port 12344 --file` resulting -in the command executed being; +it would be possible to configure `RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS` to +be set to `/path/to/debugger.py --port 12344 --file`, resulting +in the command executed being: ``` python /path/to/debugger.py --port 12345 --file /path/to/file.py @@ -42,14 +42,14 @@ doing. This is mostly useful for development to debug errors. :::{envvar} RULES_PYTHON_DEPRECATION_WARNINGS -When `1`, the rules_python will warn users about deprecated functionality that will +When `1`, `rules_python` will warn users about deprecated functionality that will be removed in a subsequent major `rules_python` version. Defaults to `0` if unset. ::: ::::{envvar} RULES_PYTHON_ENABLE_PYSTAR -When `1`, the rules_python Starlark implementation of the core rules is used -instead of the Bazel-builtin rules. Note this requires Bazel 7+. Defaults +When `1`, the `rules_python` Starlark implementation of the core rules is used +instead of the Bazel-builtin rules. Note that this requires Bazel 7+. Defaults to `1`. :::{versionadded} 0.26.0 @@ -62,7 +62,7 @@ The default became `1` if unspecified ::::{envvar} RULES_PYTHON_ENABLE_PIPSTAR -When `1`, the rules_python Starlark implementation of the pypi/pip integration is used +When `1`, the `rules_python` Starlark implementation of the PyPI/pip integration is used instead of the legacy Python scripts. :::{versionadded} 1.5.0 @@ -95,8 +95,8 @@ exit. :::{envvar} RULES_PYTHON_GAZELLE_VERBOSE -When `1`, debug information from gazelle is printed to stderr. -::: +When `1`, debug information from Gazelle is printed to stderr. +:::: :::{envvar} RULES_PYTHON_PIP_ISOLATED @@ -125,9 +125,9 @@ Determines the verbosity of logging output for repo rules. Valid values: :::{envvar} RULES_PYTHON_REPO_TOOLCHAIN_VERSION_OS_ARCH -Determines the python interpreter platform to be used for a particular +Determines the Python interpreter platform to be used for a particular interpreter `(version, os, arch)` triple to be used in repository rules. -Replace the `VERSION_OS_ARCH` part with actual values when using, e.g. +Replace the `VERSION_OS_ARCH` part with actual values when using, e.g., `3_13_0_linux_x86_64`. The version values must have `_` instead of `.` and the os, arch values are the same as the ones mentioned in the `//python:versions.bzl` file. diff --git a/docs/extending.md b/docs/extending.md index 387310e6cf..00018fbd74 100644 --- a/docs/extending.md +++ b/docs/extending.md @@ -41,10 +41,10 @@ wrappers around the keyword arguments eventually passed to the `rule()` function. These builder APIs give access to the _entire_ rule definition and allow arbitrary modifications. -This is level of control is powerful, but also volatile. A rule definition +This level of control is powerful but also volatile. A rule definition contains many details that _must_ change as the implementation changes. What is more or less likely to change isn't known in advance, but some general -rules are: +rules of thumb are: * Additive behavior to public attributes will be less prone to breaking. * Internal attributes that directly support a public attribute are likely @@ -55,7 +55,7 @@ rules are: ## Example: validating a source file -In this example, we derive from `py_library` a custom rule that verifies source +In this example, we derive a custom rule from `py_library` that verifies source code contains the word "snakes". It does this by: * Adding an implicit dependency on a checker program @@ -111,7 +111,7 @@ has_snakes_library = create_has_snakes_rule() ## Example: adding transitions -In this example, we derive from `py_binary` to force building for a particular +In this example, we derive a custom rule from `py_binary` to force building for a particular platform. We do this by: * Adding an additional output to the rule's cfg @@ -136,8 +136,8 @@ def create_rule(): r.cfg.add_output("//command_line_option:platforms") return r.build() -py_linux_binary = create_linux_binary_rule() +py_linux_binary = create_rule() ``` -Users can then use `py_linux_binary` the same as a regular py_binary. It will +Users can then use `py_linux_binary` the same as a regular `py_binary`. It will act as if `--platforms=//my/platforms:linux` was specified when building it. diff --git a/docs/gazelle.md b/docs/gazelle.md index 89f26d67bb..60b46faf2c 100644 --- a/docs/gazelle.md +++ b/docs/gazelle.md @@ -3,7 +3,7 @@ [Gazelle](https://github.com/bazelbuild/bazel-gazelle) is a build file generator for Bazel projects. It can create new `BUILD.bazel` files for a project that follows language conventions and update existing build files to include new sources, dependencies, and options. -Bazel may run Gazelle using the Gazelle rule, or it may be installed and run as a command line tool. +Bazel may run Gazelle using the Gazelle rule, or Gazelle may be installed and run as a command line tool. -See the documentation for Gazelle with rules_python in the {gh-path}`gazelle` +See the documentation for Gazelle with `rules_python` in the {gh-path}`gazelle` directory. diff --git a/docs/getting-started.md b/docs/getting-started.md index 7e7b88aa8a..d81d72f590 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,14 +1,14 @@ # Getting started -This doc is a simplified guide to help get started quickly. It provides +This document is a simplified guide to help you get started quickly. It provides a simplified introduction to having a working Python program for both `bzlmod` and the older way of using `WORKSPACE`. It assumes you have a `requirements.txt` file with your PyPI dependencies. -For more details information about configuring `rules_python`, see: +For more detailed information about configuring `rules_python`, see: * [Configuring the runtime](configuring-toolchains) -* [Configuring third party dependencies (pip/pypi)](./pypi/index) +* [Configuring third-party dependencies (pip/PyPI)](./pypi/index) * [API docs](api/index) ## Including dependencies @@ -32,7 +32,7 @@ use_repo(pip, "pypi") ### Using a WORKSPACE file -Using WORKSPACE is deprecated, but still supported, and a bit more involved than +Using `WORKSPACE` is deprecated but still supported, and it's a bit more involved than using Bzlmod. Here is a simplified setup to download the prebuilt runtimes. ```starlark @@ -72,7 +72,7 @@ pip_parse( ## "Hello World" -Once you've imported the rule set using either Bzlmod or WORKSPACE, you can then +Once you've imported the rule set using either Bzlmod or `WORKSPACE`, you can then load the core rules in your `BUILD` files with the following: ```starlark diff --git a/docs/glossary.md b/docs/glossary.md index 9afbcffb92..c9bd03fd0e 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -5,7 +5,7 @@ common attributes : Every rule has a set of common attributes. See Bazel's [Common attributes](https://bazel.build/reference/be/common-definitions#common-attributes) - for a complete listing + for a complete listing. in-build runtime : An in-build runtime is one where the Python runtime, and all its files, are @@ -21,9 +21,9 @@ which can be a significant number of files. platform runtime : A platform runtime is a Python runtime that is assumed to be installed on the -system where a Python binary runs, whereever that may be. For example, using `/usr/bin/python3` +system where a Python binary runs, wherever that may be. For example, using `/usr/bin/python3` as the interpreter is a platform runtime -- it assumes that, wherever the binary -runs (your local machine, a remote worker, within a container, etc), that path +runs (your local machine, a remote worker, within a container, etc.), that path is available. Such runtimes are _not_ part of a binary's runfiles. The main advantage of platform runtimes is they are lightweight insofar as @@ -42,8 +42,8 @@ rule callable accepted; refer to the respective API accepting this type. simple label -: A `str` or `Label` object but not a _direct_ `select` object. These usually - mean a string manipulation is occuring, which can't be done on `select` + A `str` or `Label` object but not a _direct_ `select` object. This usually + means a string manipulation is occurring, which can't be done on `select` objects. Such attributes are usually still configurable if an alias is used, and a reference to the alias is passed instead. diff --git a/docs/index.md b/docs/index.md index 82023f3ad8..25b423c6c3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,6 @@ # Python Rules for Bazel -`rules_python` is the home for 4 major components with varying maturity levels. +`rules_python` is the home for four major components with varying maturity levels. :::{topic} Core rules @@ -9,8 +9,8 @@ The core Python rules -- `py_library`, `py_binary`, `py_test`, support in Bazel. When using Bazel 6 (or earlier), the core rules are bundled into the Bazel binary, and the symbols -in this repository are simple aliases. On Bazel 7 and above `rules_python` uses -a separate Starlark implementation, +in this repository are simple aliases. On Bazel 7 and above, `rules_python` uses +a separate Starlark implementation; see {ref}`Migrating from the Bundled Rules` below. This repository follows @@ -21,12 +21,12 @@ outlined in the [support](support) page. :::{topic} PyPI integration -Package installation rules for integrating with PyPI and other SimpleAPI +Package installation rules for integrating with PyPI and other Simple API- compatible indexes. These rules work and can be used in production, but the cross-platform building that supports pulling PyPI dependencies for a target platform that is different -from the host platform is still in beta and the APIs that are subject to potential +from the host platform is still in beta, and the APIs that are subject to potential change are marked as `experimental`. ::: @@ -36,9 +36,9 @@ change are marked as `experimental`. `sphinxdocs` rules allow users to generate documentation using Sphinx powered by Bazel, with additional functionality for documenting Starlark and Bazel code. -The functionality is exposed because other projects find it useful, but -it is available as is and **the semantic versioning and -compatibility policy used by `rules_python` does not apply**. +The functionality is exposed because other projects find it useful, but +it is available "as is", and **the semantic versioning and +compatibility policy used by `rules_python` does not apply**. ::: @@ -47,7 +47,7 @@ compatibility policy used by `rules_python` does not apply**. `gazelle` plugin for generating `BUILD.bazel` files based on Python source code. -This is available as is and the semantic versioning used by `rules_python` does +This is available "as is", and the semantic versioning used by `rules_python` does not apply. ::: @@ -78,7 +78,7 @@ appropriate `load()` statements and rewrite uses of `native.py_*`. buildifier --lint=fix --warnings=native-py ``` -Currently, the `WORKSPACE` file needs to be updated manually as per +Currently, the `WORKSPACE` file needs to be updated manually as per [Getting started](getting-started). Note that Starlark-defined bundled symbols underneath @@ -87,7 +87,7 @@ by buildifier. ## Migrating to bzlmod -See {gh-path}`Bzlmod support ` for any behaviour differences between +See {gh-path}`Bzlmod support ` for any behavioral differences between `bzlmod` and `WORKSPACE`. diff --git a/docs/precompiling.md b/docs/precompiling.md index a46608f77e..ea978cddce 100644 --- a/docs/precompiling.md +++ b/docs/precompiling.md @@ -1,6 +1,6 @@ # Precompiling -Precompiling is compiling Python source files (`.py` files) into byte code +Precompiling is compiling Python source files (`.py` files) into bytecode (`.pyc` files) at build time instead of runtime. Doing it at build time can improve performance by skipping that work at runtime. @@ -15,12 +15,12 @@ While precompiling helps runtime performance, it has two main costs: a `.pyc` file. Compiled files are generally around the same size as the source files, so it approximately doubles the disk usage. 2. Precompiling requires running an extra action at build time. While - compiling itself isn't that expensive, the overhead can become noticable + compiling itself isn't that expensive, the overhead can become noticeable as more files need to be compiled. ## Binary-level opt-in -Binary-level opt-in allows enabling precompiling on a per-target basic. This is +Binary-level opt-in allows enabling precompiling on a per-target basis. This is useful for situations such as: * Globally enabling precompiling in your `.bazelrc` isn't feasible. This may @@ -41,7 +41,7 @@ can use an opt-in or opt-out approach by setting its value: ## Pyc-only builds -A pyc-only build (aka "source less" builds) is when only `.pyc` files are +A pyc-only build (aka "sourceless" builds) is when only `.pyc` files are included; the source `.py` files are not included. To enable this, set @@ -55,8 +55,8 @@ The advantage of pyc-only builds are: The disadvantages are: * Error messages will be less precise because the precise line and offset - information isn't in an pyc file. -* pyc files are Python major-version specific. + information isn't in a pyc file. +* pyc files are Python major-version-specific. :::{note} pyc files are not a form of hiding source code. They are trivial to uncompile, @@ -75,11 +75,11 @@ mechanisms are available: the {bzl:attr}`precompiler` attribute. Arbitrary binaries are supported. * The execution requirements can be customized using `--@rules_python//tools/precompiler:execution_requirements`. This is a list - flag that can be repeated. Each entry is a key=value that is added to the + flag that can be repeated. Each entry is a `key=value` pair that is added to the execution requirements of the `PyCompile` action. Note that this flag - is specific to the rules_python precompiler. If a custom binary is used, + is specific to the `rules_python` precompiler. If a custom binary is used, this flag will have to be propagated from the custom binary using the - `testing.ExecutionInfo` provider; refer to the `py_interpreter_program` an + `testing.ExecutionInfo` provider; refer to the `py_interpreter_program` example. The default precompiler implementation is an asynchronous/concurrent implementation. If you find it has bugs or hangs, please report them. In the @@ -90,18 +90,18 @@ as well, but is less likely to have issues. The `execution_requirements` keys of most relevance are: * `supports-workers`: 1 or 0, to indicate if a regular persistent worker is desired. -* `supports-multiplex-workers`: 1 o 0, to indicate if a multiplexed persistent +* `supports-multiplex-workers`: `1` or `0`, to indicate if a multiplexed persistent worker is desired. -* `requires-worker-protocol`: json or proto; the rules_python precompiler - currently only supports json. -* `supports-multiplex-sandboxing`: 1 or 0, to indicate if sanboxing is of the +* `requires-worker-protocol`: `json` or `proto`; the `rules_python` precompiler + currently only supports `json`. +* `supports-multiplex-sandboxing`: `1` or `0`, to indicate if sandboxing of the worker is supported. -* `supports-worker-cancellation`: 1 or 1, to indicate if requests to the worker +* `supports-worker-cancellation`: `1` or `0`, to indicate if requests to the worker can be cancelled. Note that any execution requirements values can be specified in the flag. -## Known issues, caveats, and idiosyncracies +## Known issues, caveats, and idiosyncrasies * Precompiling requires Bazel 7+ with the Pystar rule implementation enabled. * Mixing rules_python PyInfo with Bazel builtin PyInfo will result in pyc files @@ -111,14 +111,14 @@ Note that any execution requirements values can be specified in the flag. causes the module to be found in the workspace source directory instead of within the binary's runfiles directory (where the pyc files are). This can usually be worked around by removing `sys.path[0]` (or otherwise ensuring the - runfiles directory comes before the repos source directory in `sys.path`). -* The pyc filename does not include the optimization level (e.g. - `foo.cpython-39.opt-2.pyc`). This works fine (it's all byte code), but also + runfiles directory comes before the repo's source directory in `sys.path`). +* The pyc filename does not include the optimization level (e.g., + `foo.cpython-39.opt-2.pyc`). This works fine (it's all bytecode), but also means the interpreter `-O` argument can't be used -- doing so will cause the interpreter to look for the non-existent `opt-N` named files. -* Targets with the same source files and different exec properites will result +* Targets with the same source files and different exec properties will result in action conflicts. This most commonly occurs when a `py_binary` and - `py_library` have the same source files. To fix, modify both targets so + a `py_library` have the same source files. To fix this, modify both targets so they have the same exec properties. If this is difficult because unsupported exec groups end up being passed to the Python rules, please file an issue to have those exec groups added to the Python rules. diff --git a/docs/pypi/circular-dependencies.md b/docs/pypi/circular-dependencies.md index d22f5b36a7..62613f489e 100644 --- a/docs/pypi/circular-dependencies.md +++ b/docs/pypi/circular-dependencies.md @@ -3,8 +3,8 @@ # Circular dependencies -Sometimes PyPi packages contain dependency cycles -- for instance a particular -version `sphinx` (this is no longer the case in the latest version as of +Sometimes PyPI packages contain dependency cycles. For instance, a particular +version of `sphinx` (this is no longer the case in the latest version as of 2024-06-02) depends on `sphinxcontrib-serializinghtml`. When using them as `requirement()`s, ala @@ -47,10 +47,10 @@ simultaneously. ) ``` -`pip_parse` supports fixing multiple cycles simultaneously, however cycles must -be distinct. `apache-airflow` for instance has dependency cycles with a number +`pip_parse` supports fixing multiple cycles simultaneously, however, cycles must +be distinct. `apache-airflow`, for instance, has dependency cycles with a number of its optional dependencies, which means those optional dependencies must all -be a part of the `airflow` cycle. For instance -- +be a part of the `airflow` cycle. For instance: ```starlark ... @@ -67,9 +67,9 @@ be a part of the `airflow` cycle. For instance -- Alternatively, one could resolve the cycle by removing one leg of it. -For example while `apache-airflow-providers-sqlite` is "baked into" the Airflow +For example, while `apache-airflow-providers-sqlite` is "baked into" the Airflow package, `apache-airflow-providers-postgres` is not and is an optional feature. -Rather than listing `apache-airflow[postgres]` in your `requirements.txt` which +Rather than listing `apache-airflow[postgres]` in your `requirements.txt`, which would expose a cycle via the extra, one could either _manually_ depend on `apache-airflow` and `apache-airflow-providers-postgres` separately as requirements. Bazel rules which need only `apache-airflow` can take it as a @@ -77,6 +77,6 @@ dependency, and rules which explicitly want to mix in `apache-airflow-providers-postgres` now can. Alternatively, one could use `rules_python`'s patching features to remove one -leg of the dependency manually. For instance by making +leg of the dependency manually, for instance, by making `apache-airflow-providers-postgres` not explicitly depend on `apache-airflow` or perhaps `apache-airflow-providers-common-sql`. diff --git a/docs/pypi/download-workspace.md b/docs/pypi/download-workspace.md index 48710095a4..5dfb0f257a 100644 --- a/docs/pypi/download-workspace.md +++ b/docs/pypi/download-workspace.md @@ -3,7 +3,7 @@ # Download (WORKSPACE) -This documentation page covers how to download the PyPI dependencies in the legacy `WORKSPACE` setup. +This documentation page covers how to download PyPI dependencies in the legacy `WORKSPACE` setup. To add pip dependencies to your `WORKSPACE`, load the `pip_parse` function and call it to create the central external repo and individual wheel external repos. @@ -27,7 +27,7 @@ install_deps() ## Interpreter selection -Note that pip parse runs before the Bazel before decides which Python toolchain to use, it cannot +Note that because `pip_parse` runs before Bazel decides which Python toolchain to use, it cannot enforce that the interpreter used to invoke `pip` matches the interpreter used to run `py_binary` targets. By default, `pip_parse` uses the system command `"python3"`. To override this, pass in the {attr}`pip_parse.python_interpreter` attribute or {attr}`pip_parse.python_interpreter_target`. @@ -44,9 +44,9 @@ your system `python` interpreter), you can force it to re-execute by running (per-os-arch-requirements)= ## Requirements for a specific OS/Architecture -In some cases you may need to use different requirements files for different OS, Arch combinations. +In some cases, you may need to use different requirements files for different OS and architecture combinations. This is enabled via the {attr}`pip_parse.requirements_by_platform` attribute. The keys of the -dictionary are labels to the file and the values are a list of comma separated target (os, arch) +dictionary are labels to the file, and the values are a list of comma-separated target (os, arch) tuples. For example: @@ -63,8 +63,8 @@ For example: requirements_lock = "requirements_lock.txt", ``` -In case of duplicate platforms, `rules_python` will raise an error as there has -to be unambiguous mapping of the requirement files to the (os, arch) tuples. +In case of duplicate platforms, `rules_python` will raise an error, as there has +to be an unambiguous mapping of the requirement files to the (os, arch) tuples. An alternative way is to use per-OS requirement attributes. ```starlark diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 18d6699ab3..7f4e205d84 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -8,8 +8,8 @@ For WORKSPACE instructions see [here](./download-workspace). ::: To add PyPI dependencies to your `MODULE.bazel` file, use the `pip.parse` -extension, and call it to create the central external repo and individual wheel -external repos. Include in the `MODULE.bazel` the toolchain extension as shown +extension and call it to create the central external repo and individual wheel +external repos. Include the toolchain extension in the `MODULE.bazel` file as shown in the first bzlmod example above. ```starlark @@ -24,7 +24,7 @@ pip.parse( use_repo(pip, "my_deps") ``` -For more documentation, see the bzlmod examples under the {gh-path}`examples` folder or the documentation +For more documentation, see the Bzlmod examples under the {gh-path}`examples` folder or the documentation for the {obj}`@rules_python//python/extensions:pip.bzl` extension. :::note} @@ -42,7 +42,7 @@ difference. ## Interpreter selection -The {obj}`pip.parse` `bzlmod` extension by default uses the hermetic python toolchain for the host +The {obj}`pip.parse` `bzlmod` extension by default uses the hermetic Python toolchain for the host platform, but you can customize the interpreter using {attr}`pip.parse.python_interpreter` and {attr}`pip.parse.python_interpreter_target`. @@ -58,10 +58,10 @@ name]`. (per-os-arch-requirements)= ## Requirements for a specific OS/Architecture -In some cases you may need to use different requirements files for different OS, Arch combinations. -This is enabled via the `requirements_by_platform` attribute in `pip.parse` extension and the -{obj}`pip.parse` tag class. The keys of the dictionary are labels to the file and the values are a -list of comma separated target (os, arch) tuples. +In some cases, you may need to use different requirements files for different OS and architecture combinations. +This is enabled via the `requirements_by_platform` attribute in the `pip.parse` extension and the +{obj}`pip.parse` tag class. The keys of the dictionary are labels to the file, and the values are a +list of comma-separated target (os, arch) tuples. For example: ```starlark @@ -77,8 +77,8 @@ For example: requirements_lock = "requirements_lock.txt", ``` -In case of duplicate platforms, `rules_python` will raise an error as there has -to be unambiguous mapping of the requirement files to the (os, arch) tuples. +In case of duplicate platforms, `rules_python` will raise an error, as there has +to be an unambiguous mapping of the requirement files to the (os, arch) tuples. An alternative way is to use per-OS requirement attributes. ```starlark @@ -98,24 +98,24 @@ the lock file will be evaluated against, consider using the aforementioned ## Multi-platform support -Historically the {obj}`pip_parse` and {obj}`pip.parse` have been only downloading/building +Historically, the {obj}`pip_parse` and {obj}`pip.parse` have only been downloading/building Python dependencies for the host platform that the `bazel` commands are executed on. Over -the years people started needing support for building containers and usually that involves -fetching dependencies for a particular target platform that may be other than the host +the years, people started needing support for building containers, and usually, that involves +fetching dependencies for a particular target platform that may be different from the host platform. -Multi-platform support of cross-building the wheels can be done in two ways: +Multi-platform support for cross-building the wheels can be done in two ways: 1. using {attr}`experimental_index_url` for the {bzl:obj}`pip.parse` bzlmod tag class -2. using {attr}`pip.parse.download_only` setting. +2. using the {attr}`pip.parse.download_only` setting. :::{warning} -This will not for sdists with C extensions, but pure Python sdists may still work using the first +This will not work for sdists with C extensions, but pure Python sdists may still work using the first approach. ::: ### Using `download_only` attribute -Let's say you have 2 requirements files: +Let's say you have two requirements files: ``` # requirements.linux_x86_64.txt --platform=manylinux_2_17_x86_64 @@ -151,9 +151,9 @@ pip.parse( ) ``` -With this, the `pip.parse` will create a hub repository that is going to -support only two platforms - `cp39_osx_aarch64` and `cp39_linux_x86_64` and it -will only use `wheels` and ignore any sdists that it may find on the PyPI +With this, `pip.parse` will create a hub repository that is going to +support only two platforms - `cp39_osx_aarch64` and `cp39_linux_x86_64` - and it +will only use `wheels` and ignore any sdists that it may find on the PyPI- compatible indexes. :::{warning} @@ -162,7 +162,7 @@ multiple times. ::: :::{note} -This will only work for wheel-only setups, i.e. all of your dependencies need to have wheels +This will only work for wheel-only setups, i.e., all of your dependencies need to have wheels available on the PyPI index that you use. ::: @@ -173,9 +173,9 @@ Currently this is disabled by default, but you can turn it on using {envvar}`RULES_PYTHON_ENABLE_PIPSTAR` environment variable. ::: -In order to understand what dependencies to pull for a particular package +In order to understand what dependencies to pull for a particular package, `rules_python` parses the `whl` file [`METADATA`][metadata]. -Packages can express dependencies via `Requires-Dist` and they can add conditions using +Packages can express dependencies via `Requires-Dist`, and they can add conditions using "environment markers", which represent the Python version, OS, etc. While the PyPI integration provides reasonable defaults to support most @@ -198,8 +198,8 @@ additional keys, which become available during dependency evaluation. ### Bazel downloader and multi-platform wheel hub repository. :::{warning} -This is currently still experimental and whilst it has been proven to work in quite a few -environments, the APIs are still being finalized and there may be changes to the APIs for this +This is currently still experimental, and whilst it has been proven to work in quite a few +environments, the APIs are still being finalized, and there may be changes to the APIs for this feature without much notice. The issues that you can subscribe to for updates are: @@ -207,7 +207,7 @@ The issues that you can subscribe to for updates are: * {gh-issue}`1357` ::: -The {obj}`pip` extension supports pulling information from `PyPI` (or a compatible mirror) and it +The {obj}`pip` extension supports pulling information from `PyPI` (or a compatible mirror), and it will ensure that the [bazel downloader][bazel_downloader] is used for downloading the wheels. This provides the following benefits: @@ -222,7 +222,7 @@ To enable the feature specify {attr}`pip.parse.experimental_index_url` as shown the {gh-path}`examples/bzlmod/MODULE.bazel` example. Similar to [uv](https://docs.astral.sh/uv/configuration/indexes/), one can override the -index that is used for a single package. By default we first search in the index specified by +index that is used for a single package. By default, we first search in the index specified by {attr}`pip.parse.experimental_index_url`, then we iterate through the {attr}`pip.parse.experimental_extra_index_urls` unless there are overrides specified via {attr}`pip.parse.experimental_index_url_overrides`. @@ -235,12 +235,12 @@ Loading: 0 packages loaded ``` -This does not mean that `rules_python` is fetching the wheels eagerly, but it -rather means that it is calling the PyPI server to get the Simple API response +This does not mean that `rules_python` is fetching the wheels eagerly; rather, +it means that it is calling the PyPI server to get the Simple API response to get the list of all available source and wheel distributions. Once it has -got all of the available distributions, it will select the right ones depending +gotten all of the available distributions, it will select the right ones depending on the `sha256` values in your `requirements_lock.txt` file. If `sha256` hashes -are not present in the requirements file, we will fallback to matching by version +are not present in the requirements file, we will fall back to matching by version specified in the lock file. Fetching the distribution information from the PyPI allows `rules_python` to @@ -264,10 +264,10 @@ available flags: The [Bazel downloader](#bazel-downloader) usage allows for the Bazel [Credential Helper][cred-helper-design]. -Your python artifact registry may provide a credential helper for you. +Your Python artifact registry may provide a credential helper for you. Refer to your index's docs to see if one is provided. -The simplest form of a credential helper is a bash script that accepts an arg and spits out JSON to +The simplest form of a credential helper is a bash script that accepts an argument and spits out JSON to stdout. For a service like Google Artifact Registry that uses ['Basic' HTTP Auth][rfc7617] and does not provide a credential helper that conforms to the [spec][cred-helper-spec], the script might look like: @@ -285,7 +285,7 @@ echo ' }' echo '}' ``` -Configure Bazel to use this credential helper for your python index `example.com`: +Configure Bazel to use this credential helper for your Python index `example.com`: ``` # .bazelrc diff --git a/docs/pypi/index.md b/docs/pypi/index.md index c300124398..c32bafc609 100644 --- a/docs/pypi/index.md +++ b/docs/pypi/index.md @@ -3,11 +3,11 @@ # Using PyPI -Using PyPI packages (aka "pip install") involves the following main steps. +Using PyPI packages (aka "pip install") involves the following main steps: 1. [Generating requirements file](./lock) -2. Installing third party packages in [bzlmod](./download) or [WORKSPACE](./download-workspace). -3. [Using third party packages as dependencies](./use) +2. Installing third-party packages in [bzlmod](./download) or [WORKSPACE](./download-workspace). +3. [Using third-party packages as dependencies](./use) With the advanced topics covered separately: * Dealing with [circular dependencies](./circular-dependencies). diff --git a/docs/pypi/lock.md b/docs/pypi/lock.md index c9376036fb..db557fe594 100644 --- a/docs/pypi/lock.md +++ b/docs/pypi/lock.md @@ -11,9 +11,14 @@ Currently `rules_python` only supports `requirements.txt` format. ### pip compile -Generally, when working on a Python project, you'll have some dependencies that themselves have other dependencies. You might also specify dependency bounds instead of specific versions. So you'll need to generate a full list of all transitive dependencies and pinned versions for every dependency. - -Typically, you'd have your project dependencies specified in `pyproject.toml` or `requirements.in` and generate the full pinned list of dependencies in `requirements_lock.txt`, which you can manage with the {obj}`compile_pip_requirements`: +Generally, when working on a Python project, you'll have some dependencies that themselves have +other dependencies. You might also specify dependency bounds instead of specific versions. +So you'll need to generate a full list of all transitive dependencies and pinned versions +for every dependency. + +Typically, you'd have your project dependencies specified in `pyproject.toml` or `requirements.in` +and generate the full pinned list of dependencies in `requirements_lock.txt`, which you can +manage with {obj}`compile_pip_requirements`: ```starlark load("@rules_python//python:pip.bzl", "compile_pip_requirements") diff --git a/docs/pypi/patch.md b/docs/pypi/patch.md index f341bd1091..7e3cb41981 100644 --- a/docs/pypi/patch.md +++ b/docs/pypi/patch.md @@ -4,7 +4,7 @@ # Patching wheels Sometimes the wheels have to be patched to: -* Workaround the lack of a standard `site-packages` layout ({gh-issue}`2156`) -* Include certain PRs of your choice on top of wheels and avoid building from sdist, +* Workaround the lack of a standard `site-packages` layout ({gh-issue}`2156`). +* Include certain PRs of your choice on top of wheels and avoid building from sdist. You can patch the wheels by using the {attr}`pip.override.patches` attribute. diff --git a/docs/pypi/use.md b/docs/pypi/use.md index 7a16b7d9e9..6212097f86 100644 --- a/docs/pypi/use.md +++ b/docs/pypi/use.md @@ -3,10 +3,10 @@ # Use in BUILD.bazel files -Once you have setup the dependencies, you are ready to start using them in your `BUILD.bazel` -files. If you haven't done so yet, set it up by following the following docs: +Once you have set up the dependencies, you are ready to start using them in your `BUILD.bazel` +files. If you haven't done so yet, set it up by following these docs: 1. [WORKSPACE](./download-workspace) -1. [bzlmod](./download) +2. [bzlmod](./download) To refer to targets in a hub repo `pypi`, you can do one of two things: ```starlark @@ -29,19 +29,19 @@ py_library( ) ``` -Note, that the usage of the `requirement` helper is not advised and can be problematic. See the +Note that the usage of the `requirement` helper is not advised and can be problematic. See the [notes below](#requirement-helper). -Note, that the hub repo contains the following targets for each package: -* `@pypi//numpy` which is a shorthand for `@pypi//numpy:numpy`. This is an {obj}`alias` to +Note that the hub repo contains the following targets for each package: +* `@pypi//numpy` - shorthand for `@pypi//numpy:numpy`. This is an {obj}`alias` to `@pypi//numpy:pkg`. * `@pypi//numpy:pkg` - the {obj}`py_library` target automatically generated by the repository rules. -* `@pypi//numpy:data` - the {obj}`filegroup` that is for all of the extra files that are included +* `@pypi//numpy:data` - the {obj}`filegroup` for all of the extra files that are included as data in the `pkg` target. -* `@pypi//numpy:dist_info` - the {obj}`filegroup` that is for all of the files in the `.distinfo` directory. -* `@pypi//numpy:whl` - the {obj}`filegroup` that is the `.whl` file itself which includes all of - the transitive dependencies via the {attr}`filegroup.data` attribute. +* `@pypi//numpy:dist_info` - the {obj}`filegroup` for all of the files in the `.distinfo` directory. +* `@pypi//numpy:whl` - the {obj}`filegroup` that is the `.whl` file itself, which includes all + transitive dependencies via the {attr}`filegroup.data` attribute. ## Entry points @@ -52,14 +52,14 @@ which can help you create a `py_binary` target for a particular console script e ## 'Extras' dependencies -Any 'extras' specified in the requirements lock file will be automatically added +Any "extras" specified in the requirements lock file will be automatically added as transitive dependencies of the package. In the example above, you'd just put `requirement("useful_dep")` or `@pypi//useful_dep`. ## Consuming Wheel Dists Directly -If you need to depend on the wheel dists themselves, for instance, to pass them -to some other packaging tool, you can get a handle to them with the +If you need to depend on the wheel dists themselves (for instance, to pass them +to some other packaging tool), you can get a handle to them with the `whl_requirement` macro. For example: ```starlark @@ -77,7 +77,7 @@ filegroup( ## Creating a filegroup of files within a whl The rule {obj}`whl_filegroup` exists as an easy way to extract the necessary files -from a whl file without the need to modify the `BUILD.bazel` contents of the +from a whl file without needing to modify the `BUILD.bazel` contents of the whl repositories generated via `pip_repository`. Use it similarly to the `filegroup` above. See the API docs for more information. @@ -104,16 +104,16 @@ py_library( ) ``` -The reason `requirement()` exists is to insulate from +The reason `requirement()` exists is to insulate users from changes to the underlying repository and label strings. However, those -labels have become directly used, so aren't able to easily change regardless. +labels have become directly used, so they aren't able to easily change regardless. -On the other hand, using `requirement()` helper has several drawbacks: +On the other hand, using the `requirement()` helper has several drawbacks: -- It doesn't work with `buildifier` -- It doesn't work with `buildozer` -- It adds extra layer on top of normal mechanisms to refer to targets. -- It does not scale well as each type of target needs a new macro to be loaded and imported. +- It doesn't work with `buildifier`. +- It doesn't work with `buildozer`. +- It adds an extra layer on top of normal mechanisms to refer to targets. +- It does not scale well, as each type of target needs a new macro to be loaded and imported. If you don't want to use `requirement()`, you can use the library labels directly instead. For `pip_parse`, the labels are of the following form: diff --git a/docs/repl.md b/docs/repl.md index edcf37e811..1434097fdf 100644 --- a/docs/repl.md +++ b/docs/repl.md @@ -1,6 +1,6 @@ # Getting a REPL or Interactive Shell -rules_python provides a REPL to help with debugging and developing. The goal of +`rules_python` provides a REPL to help with debugging and developing. The goal of the REPL is to present an environment identical to what a {bzl:obj}`py_binary` creates for your code. diff --git a/docs/support.md b/docs/support.md index 5e6de57fcb..ad943b3845 100644 --- a/docs/support.md +++ b/docs/support.md @@ -8,7 +8,7 @@ page for information on our development workflow. ## Supported rules_python Versions In general, only the latest version is supported. Backporting changes is -done on a best effort basis based on severity, risk of regressions, and +done on a best-effort basis based on severity, risk of regressions, and the willingness of volunteers. If you want or need particular functionality backported, then the best way @@ -33,24 +33,24 @@ for what versions are the rolling, active, and prior releases. ## Supported Python versions -As a general rule we test all released non-EOL Python versions. Different +As a general rule, we test all released non-EOL Python versions. Different interpreter versions may work but are not guaranteed. We are interested in staying compatible with upcoming unreleased versions, so if you see that things stop working, please create tickets or, more preferably, pull requests. ## Supported Platforms -We only support the platforms that our continuous integration jobs run, which -is Linux, Mac, and Windows. +We only support the platforms that our continuous integration jobs run on, which +are Linux, Mac, and Windows. -In order to better describe different support levels, the below acts as a rough +In order to better describe different support levels, the following acts as a rough guideline for different platform tiers: -* Tier 0 - The platforms that our CI runs: `linux_x86_64`, `osx_x86_64`, `RBE linux_x86_64`. -* Tier 1 - The platforms that are similar enough to what the CI runs: `linux_aarch64`, `osx_arm64`. - What is more, `windows_x86_64` is in this list as we run tests in CI but - developing for Windows is more challenging and features may come later to +* Tier 0 - The platforms that our CI runs on: `linux_x86_64`, `osx_x86_64`, `RBE linux_x86_64`. +* Tier 1 - The platforms that are similar enough to what the CI runs on: `linux_aarch64`, `osx_arm64`. + What is more, `windows_x86_64` is in this list, as we run tests in CI, but + developing for Windows is more challenging, and features may come later to this platform. -* Tier 2 - The rest of the platforms that may have varying level of support, e.g. +* Tier 2 - The rest of the platforms that may have a varying level of support, e.g., `linux_s390x`, `linux_ppc64le`, `windows_arm64`. :::{note} @@ -75,7 +75,7 @@ a series of releases to so users can still incrementally upgrade. See the ## Experimental Features -An experimental features is functionality that may not be ready for general +An experimental feature is functionality that may not be ready for general use and may change quickly and/or significantly. Such features are denoted in their name or API docs as "experimental". They may have breaking changes made at any time. diff --git a/docs/toolchains.md b/docs/toolchains.md index 668a458156..de819cb515 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -4,13 +4,13 @@ (configuring-toolchains)= # Configuring Python toolchains and runtimes -This documents how to configure the Python toolchain and runtimes for different +This document explains how to configure the Python toolchain and runtimes for different use cases. ## Bzlmod MODULE configuration -How to configure `rules_python` in your MODULE.bazel file depends on how and why -you're using Python. There are 4 basic use cases: +How to configure `rules_python` in your `MODULE.bazel` file depends on how and why +you're using Python. There are four basic use cases: 1. A root module that always uses Python. For example, you're building a Python application. @@ -51,7 +51,7 @@ python.toolchain(python_version = "3.12") ### Library modules A library module is a module that can show up in arbitrary locations in the -bzlmod module graph -- it's unknown where in the breadth-first search order the +Bzlmod module graph -- it's unknown where in the breadth-first search order the module will be relative to other modules. For example, `rules_python` is a library module. @@ -84,9 +84,9 @@ used for the Python programs it runs isn't chosen by the module itself. Instead, it's up to the root module to pick an appropriate version of Python. For this case, configuration is simple: just depend on `rules_python` and use -the normal `//python:py_binary.bzl` et al rules. There is no need to call -`python.toolchain` -- rules_python ensures _some_ Python version is available, -but more often the root module will specify some version. +the normal `//python:py_binary.bzl` et al. rules. There is no need to call +`python.toolchain` -- `rules_python` ensures _some_ Python version is available, +but more often, the root module will specify some version. ``` # MODULE.bazel @@ -108,7 +108,7 @@ specific Python version be used with its tools. This has some pros/cons: * It has higher build overhead because additional runtimes and libraries need to be downloaded, and Bazel has to keep additional configuration state. -To configure this, request the Python versions needed in MODULE.bazel and use +To configure this, request the Python versions needed in `MODULE.bazel` and use the version-aware rules for `py_binary`. ``` @@ -132,7 +132,7 @@ is most useful for two cases: 1. For submodules to ensure they run with the appropriate Python version 2. To allow incremental, per-target, upgrading to newer Python versions, - typically in a mono-repo situation. + typically in a monorepo situation. To configure a submodule with the version-aware rules, request the particular version you need when defining the toolchain: @@ -147,7 +147,7 @@ python.toolchain( use_repo(python) ``` -Then use the `@rules_python` repo in your BUILD file to explicity pin the Python version when calling the rule: +Then use the `@rules_python` repo in your `BUILD` file to explicitly pin the Python version when calling the rule: ```starlark # BUILD.bazel @@ -202,29 +202,29 @@ The `python.toolchain()` call makes its contents available under a repo named `python_X_Y`, where X and Y are the major and minor versions. For example, `python.toolchain(python_version="3.11")` creates the repo `@python_3_11`. Remember to call `use_repo()` to make repos visible to your module: -`use_repo(python, "python_3_11")` +`use_repo(python, "python_3_11")`. :::{deprecated} 1.1.0 -The toolchain specific `py_binary` and `py_test` symbols are aliases to the regular rules. -i.e. Deprecated `load("@python_versions//3.11:defs.bzl", "py_binary")` & `load("@python_versions//3.11:defs.bzl", "py_test")` +The toolchain-specific `py_binary` and `py_test` symbols are aliases to the regular rules. +For example, `load("@python_versions//3.11:defs.bzl", "py_binary")` & `load("@python_versions//3.11:defs.bzl", "py_test")` are deprecated. -Usages of them should be changed to load the regular rules directly; -i.e. Use `load("@rules_python//python:py_binary.bzl", "py_binary")` & `load("@rules_python//python:py_test.bzl", "py_test")` and then specify the `python_version` when using the rules corresponding to the python version you defined in your toolchain. {ref}`Library modules with version constraints` +Usages of them should be changed to load the regular rules directly. +For example, use `load("@rules_python//python:py_binary.bzl", "py_binary")` & `load("@rules_python//python:py_test.bzl", "py_test")` and then specify the `python_version` when using the rules corresponding to the Python version you defined in your toolchain. {ref}`Library modules with version constraints` ::: #### Toolchain usage in other rules -Python toolchains can be utilized in other bazel rules, such as `genrule()`, by +Python toolchains can be utilized in other Bazel rules, such as `genrule()`, by adding the `toolchains=["@rules_python//python:current_py_toolchain"]` attribute. You can obtain the path to the Python interpreter using the `$(PYTHON2)` and `$(PYTHON3)` ["Make" Variables](https://bazel.build/reference/be/make-variables). See the {gh-path}`test_current_py_toolchain ` target -for an example. We also make available `$(PYTHON2_ROOTPATH)` and `$(PYTHON3_ROOTPATH)` +for an example. We also make available `$(PYTHON2_ROOTPATH)` and `$(PYTHON3_ROOTPATH)`, which are Make Variable equivalents of `$(PYTHON2)` and `$(PYTHON3)` but for runfiles -locations. These will be helpful if you need to set env vars of binary/test rules +locations. These will be helpful if you need to set environment variables of binary/test rules while using [`--nolegacy_external_runfiles`](https://bazel.build/reference/command-line-reference#flag--legacy_external_runfiles). The original make variables still work in exec contexts such as genrules. @@ -246,9 +246,9 @@ existing attributes: ### Registering custom runtimes Because the python-build-standalone project has _thousands_ of prebuilt runtimes -available, rules_python only includes popular runtimes in its built in +available, `rules_python` only includes popular runtimes in its built-in configurations. If you want to use a runtime that isn't already known to -rules_python then {obj}`single_version_platform_override()` can be used to do +`rules_python`, then {obj}`single_version_platform_override()` can be used to do so. In short, it allows specifying an arbitrary URL and using custom flags to control when a runtime is used. @@ -287,21 +287,21 @@ config_setting( ``` Notes: -- While any URL and archive can be used, it's assumed their content looks how - a python-build-standalone archive looks. -- A "version aware" toolchain is registered, which means the Python version flag - must also match (e.g. `--@rules_python//python/config_settings:python_version=3.13.3` +- While any URL and archive can be used, it's assumed their content looks like + a python-build-standalone archive. +- A "version-aware" toolchain is registered, which means the Python version flag + must also match (e.g., `--@rules_python//python/config_settings:python_version=3.13.3` must be set -- see `minor_mapping` and `is_default` for controls and docs about version matching and selection). - The `target_compatible_with` attribute can be used to entirely specify the - arg of the same name the toolchain uses. + argument of the same name that the toolchain uses. - The labels in `target_settings` must be absolute; `@@` refers to the main repo. - The `target_settings` are `config_setting` targets, which means you can customize how matching occurs. :::{seealso} -See {obj}`//python/config_settings` for flags rules_python already defines -that can be used with `target_settings`. Some particular ones of note are: +See {obj}`//python/config_settings` for flags `rules_python` already defines +that can be used with `target_settings`. Some particular ones of note are {flag}`--py_linux_libc` and {flag}`--py_freethreaded`, among others. ::: @@ -312,7 +312,7 @@ Added support for custom platform names, `target_compatible_with`, and ### Using defined toolchains from WORKSPACE -It is possible to use toolchains defined in `MODULE.bazel` in `WORKSPACE`. For example +It is possible to use toolchains defined in `MODULE.bazel` in `WORKSPACE`. For example, the following `MODULE.bazel` and `WORKSPACE` provides a working {bzl:obj}`pip_parse` setup: ```starlark # File: WORKSPACE @@ -343,16 +343,16 @@ python.toolchain(python_version = "3.10") use_repo(python, "python_3_10", "python_3_10_host") ``` -Note, the user has to import the `*_host` repository to use the python interpreter in the -{bzl:obj}`pip_parse` and `whl_library` repository rules and once that is done +Note, the user has to import the `*_host` repository to use the Python interpreter in the +{bzl:obj}`pip_parse` and `whl_library` repository rules, and once that is done, users should be able to ensure the setting of the default toolchain even during the transition period when some of the code is still defined in `WORKSPACE`. ## Workspace configuration -To import rules_python in your project, you first need to add it to your +To import `rules_python` in your project, you first need to add it to your `WORKSPACE` file, using the snippet provided in the -[release you choose](https://github.com/bazel-contrib/rules_python/releases) +[release you choose](https://github.com/bazel-contrib/rules_python/releases). To depend on a particular unreleased version, you can do the following: @@ -403,15 +403,15 @@ pip_parse( ``` After registration, your Python targets will use the toolchain's interpreter during execution, but a system-installed interpreter -is still used to 'bootstrap' Python targets (see https://github.com/bazel-contrib/rules_python/issues/691). +is still used to "bootstrap" Python targets (see https://github.com/bazel-contrib/rules_python/issues/691). You may also find some quirks while using this toolchain. Please refer to [python-build-standalone documentation's _Quirks_ section](https://gregoryszorc.com/docs/python-build-standalone/main/quirks.html). ## Local toolchain It's possible to use a locally installed Python runtime instead of the regular prebuilt, remotely downloaded ones. A local toolchain contains the Python -runtime metadata (Python version, headers, ABI flags, etc) that the regular -remotely downloaded runtimes contain, which makes it possible to build e.g. C +runtime metadata (Python version, headers, ABI flags, etc.) that the regular +remotely downloaded runtimes contain, which makes it possible to build, e.g., C extensions (unlike the autodetecting and runtime environment toolchains). For simple cases, the {obj}`local_runtime_repo` and @@ -420,10 +420,10 @@ Python installation and create an appropriate Bazel definition from it. To do this, three pieces need to be wired together: 1. Specify a path or command to a Python interpreter (multiple can be defined). -2. Create toolchains for the runtimes in (1) -3. Register the toolchains created by (2) +2. Create toolchains for the runtimes in (1). +3. Register the toolchains created by (2). -The below is an example that will use `python3` from PATH to find the +The following is an example that will use `python3` from `PATH` to find the interpreter, then introspect its installation to generate a full toolchain. ```starlark @@ -474,7 +474,7 @@ Python versions and/or platforms to be configured in a single `MODULE.bazel`. Note that `register_toolchains` will insert the local toolchain earlier in the toolchain ordering, so it will take precedence over other registered toolchains. To better control when the toolchain is used, see [Conditionally using local -toolchains] +toolchains]. ### Conditionally using local toolchains @@ -483,22 +483,22 @@ ordering, which means it will usually be used no matter what. This can be problematic for CI (where it shouldn't be used), expensive for CI (CI must initialize/download the repository to determine its Python version), and annoying for iterative development (enabling/disabling it requires modifying -MODULE.bazel). +`MODULE.bazel`). These behaviors can be mitigated, but it requires additional configuration -to avoid triggering the local toolchain repository to initialize (i.e. run +to avoid triggering the local toolchain repository to initialize (i.e., run local commands and perform downloads). The two settings to change are {obj}`local_runtime_toolchains_repo.target_compatible_with` and {obj}`local_runtime_toolchains_repo.target_settings`, which control how Bazel decides if a toolchain should match. By default, they point to targets *within* -the local runtime repository (trigger repo initialization). We have to override +the local runtime repository (triggering repo initialization). We have to override them to *not* reference the local runtime repository at all. In the example below, we reconfigure the local toolchains so they are only activated if the custom flag `--//:py=local` is set and the target platform -matches the Bazel host platform. The net effect is CI won't use the local +matches the Bazel host platform. The net effect is that CI won't use the local toolchain (nor initialize its repository), and developers can easily enable/disable the local toolchain with a command line flag. @@ -545,9 +545,9 @@ information about Python at build time. In particular, this means it is not able to build C extensions -- doing so requires knowing, at build time, what Python headers to use. -In effect, all it does is generate a small wrapper script that simply calls e.g. +In effect, all it does is generate a small wrapper script that simply calls, e.g., `/usr/bin/env python3` to run a program. This makes it easy to change what -Python is used to run a program, but also makes it easy to use a Python version +Python is used to run a program but also makes it easy to use a Python version that isn't compatible with build-time assumptions. ``` @@ -565,26 +565,26 @@ locally installed Python. ### Autodetecting toolchain The autodetecting toolchain is a deprecated toolchain that is built into Bazel. -**It's name is a bit misleading: it doesn't autodetect anything**. All it does is +**Its name is a bit misleading: it doesn't autodetect anything.** All it does is use `python3` from the environment a binary runs within. This provides extremely limited functionality to the rules (at build time, nothing is knowable about the Python runtime). Bazel itself automatically registers `@bazel_tools//tools/python:autodetecting_toolchain` -as the lowest priority toolchain. For WORKSPACE builds, if no other toolchain -is registered, that toolchain will be used. For bzlmod builds, rules_python +as the lowest priority toolchain. For `WORKSPACE` builds, if no other toolchain +is registered, that toolchain will be used. For Bzlmod builds, `rules_python` automatically registers a higher-priority toolchain; it won't be used unless there is a toolchain misconfiguration somewhere. -To aid migration off the Bazel-builtin toolchain, rules_python provides +To aid migration off the Bazel-builtin toolchain, `rules_python` provides {bzl:obj}`@rules_python//python/runtime_env_toolchains:all`. This is an equivalent -toolchain, but is implemented using rules_python's objects. +toolchain but is implemented using `rules_python`'s objects. ## Custom toolchains -While rules_python provides toolchains by default, it is not required to use +While `rules_python` provides toolchains by default, it is not required to use them, and you can define your own toolchains to use instead. This section -gives an introduction for how to define them yourself. +gives an introduction to how to define them yourself. :::{note} * Defining your own toolchains is an advanced feature. @@ -599,7 +599,7 @@ toolchains a "toolchain suite". One of the underlying design goals of the toolchains is to support complex and bespoke environments. Such environments may use an arbitrary combination of {bzl:obj}`RBE`, cross-platform building, multiple Python versions, -building Python from source, embeding Python (as opposed to building separate +building Python from source, embedding Python (as opposed to building separate interpreters), using prebuilt binaries, or using binaries built from source. To that end, many of the attributes they accept, and fields they provide, are optional. @@ -610,7 +610,7 @@ The target toolchain type is {obj}`//python:toolchain_type`, and it is for _target configuration_ runtime information, e.g., the Python version and interpreter binary that a program will use. -The is typically implemented using {obj}`py_runtime()`, which +This is typically implemented using {obj}`py_runtime()`, which provides the {obj}`PyRuntimeInfo` provider. For historical reasons from the Python 2 transition, `py_runtime` is wrapped in {obj}`py_runtime_pair`, which provides {obj}`ToolchainInfo` with the field `py3_runtime`, which is an @@ -625,7 +625,7 @@ set {external:bzl:obj}`toolchain.exec_compatible_with`. ### Python C toolchain type The Python C toolchain type ("py cc") is {obj}`//python/cc:toolchain_type`, and -it has C/C++ information for the _target configuration_, e.g. the C headers that +it has C/C++ information for the _target configuration_, e.g., the C headers that provide `Python.h`. This is typically implemented using {obj}`py_cc_toolchain()`, which provides @@ -642,7 +642,7 @@ set {external:bzl:obj}`toolchain.exec_compatible_with`. ### Exec tools toolchain type The exec tools toolchain type is {obj}`//python:exec_tools_toolchain_type`, -and it is for supporting tools for _building_ programs, e.g. the binary to +and it is for supporting tools for _building_ programs, e.g., the binary to precompile code at build time. This toolchain type is intended to hold only _exec configuration_ values -- @@ -661,7 +661,7 @@ target configuration (e.g. Python version), then for one to be chosen based on finding one compatible with the available host platforms to run the tool on. However, what `target_compatible_with`/`target_settings` and -`exec_compatible_with` values to use depend on details of the tools being used. +`exec_compatible_with` values to use depends on the details of the tools being used. For example: * If you had a precompiler that supported any version of Python, then putting the Python version in `target_settings` is unnecessary. @@ -672,9 +672,9 @@ This can work because, when the rules invoke these build tools, they pass along all necessary information so that the tool can be entirely independent of the target configuration being built for. -Alternatively, if you had a precompiler that only ran on linux, and only -produced valid output for programs intended to run on linux, then _both_ -`exec_compatible_with` and `target_compatible_with` must be set to linux. +Alternatively, if you had a precompiler that only ran on Linux and only +produced valid output for programs intended to run on Linux, then _both_ +`exec_compatible_with` and `target_compatible_with` must be set to Linux. ### Custom toolchain example @@ -684,9 +684,9 @@ Here, we show an example for a semi-complicated toolchain suite, one that is: * For Python version 3.12.0 * Using an in-build interpreter built from source * That only runs on Linux -* Using a prebuilt precompiler that only runs on Linux, and only produces byte - code valid for 3.12 -* With the exec tools interpreter disabled (unnecessary with a prebuild +* Using a prebuilt precompiler that only runs on Linux and only produces + bytecode valid for 3.12 +* With the exec tools interpreter disabled (unnecessary with a prebuilt precompiler) * Providing C headers and libraries @@ -748,13 +748,13 @@ toolchain( name = "runtime_toolchain", toolchain = "//toolchain_impl:runtime_pair", toolchain_type = "@rules_python//python:toolchain_type", - target_compatible_with = ["@platforms/os:linux"] + target_compatible_with = ["@platforms/os:linux"], ) toolchain( name = "py_cc_toolchain", toolchain = "//toolchain_impl:py_cc_toolchain_impl", toolchain_type = "@rules_python//python/cc:toolchain_type", - target_compatible_with = ["@platforms/os:linux"] + target_compatible_with = ["@platforms/os:linux"], ) toolchain( @@ -764,19 +764,19 @@ toolchain( target_settings = [ "@rules_python//python/config_settings:is_python_3.12", ], - exec_comaptible_with = ["@platforms/os:linux"] + exec_compatible_with = ["@platforms/os:linux"], ) # ----------------------------------------------- # File: MODULE.bazel or WORKSPACE.bazel -# These toolchains will considered before others. +# These toolchains will be considered before others. # ----------------------------------------------- register_toolchains("//toolchains:all") ``` -When registering custom toolchains, be aware of the the [toolchain registration +When registering custom toolchains, be aware of the [toolchain registration order](https://bazel.build/extending/toolchains#toolchain-resolution). In brief, -toolchain order is the BFS-order of the modules; see the bazel docs for a more +toolchain order is the BFS-order of the modules; see the Bazel docs for a more detailed description. :::{note} @@ -796,7 +796,7 @@ Currently the following flags are used to influence toolchain selection: To run the interpreter that Bazel will use, you can use the `@rules_python//python/bin:python` target. This is a binary target with -the executable pointing at the `python3` binary plus its relevent runfiles. +the executable pointing at the `python3` binary plus its relevant runfiles. ```console $ bazel run @rules_python//python/bin:python @@ -838,7 +838,7 @@ targets on its own. Please file a feature request if this is desired. The `//python/bin:python` target provides access to the underlying interpreter without any hermeticity guarantees. -The [`//python/bin:repl` target](repl) provides an environment indentical to +The [`//python/bin:repl` target](repl) provides an environment identical to what `py_binary` provides. That means it handles things like the [`PYTHONSAFEPATH`](https://docs.python.org/3/using/cmdline.html#envvar-PYTHONSAFEPATH) environment variable automatically. The `//python/bin:python` target will not. From 036e8c5af1258cf1a0b318a51c75f88ea4c93f11 Mon Sep 17 00:00:00 2001 From: yushan26 <107004874+yushan26@users.noreply.github.com> Date: Sat, 21 Jun 2025 19:01:39 -0700 Subject: [PATCH 42/43] feat(gazelle): For package mode, resolve dependencies when imports are relative to the package path (#2865) When `# gazelle:python_generation_mode package` is enabled, relative imports are currently not being added to the `deps` field of the generated target. For example, given the following Python code: ``` from .library import add as _add from .library import divide as _divide from .library import multiply as _multiply from .library import subtract as _subtract ``` The expected py_library rule should include a dependency on the local library package: ``` py_library( name = "py_default_library", srcs = ["__init__.py"], visibility = ["//visibility:public"], deps = [ "//example/library:py_default_library", ], ) ``` However, the actual generated rule is missing the deps entry: ``` py_library( name = "py_default_library", srcs = ["__init__.py"], visibility = ["//visibility:public"], ) ``` This change updates file_parser.go to ensure that relative imports (those starting with a .) are parsed and preserved. In `Resolve()`, logic is added to correctly interpret relative paths: A single dot (.) refers to the current package. Multiple dots (.., ..., etc.) traverse up parent directories. The relative import is resolved against the current label.Pkg path that imports the module and converted into an path relative to the root before dependency resolution. As a result, dependencies for relative imports are now correctly added to the deps field in package generation mode. Added a directive `# gazelle:experimental_allow_relative_imports true` to allow this feature to be opt in. --------- Co-authored-by: yushan Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Co-authored-by: Douglas Thor --- CHANGELOG.md | 3 ++ gazelle/README.md | 48 +++++++++++++++-- gazelle/python/configure.go | 8 +++ gazelle/python/file_parser.go | 4 +- gazelle/python/resolve.go | 53 ++++++++++++++++++- .../testdata/relative_imports/README.md | 4 -- .../relative_imports_package_mode/BUILD.in | 2 + .../relative_imports_package_mode/BUILD.out | 15 ++++++ .../relative_imports_package_mode/README.md | 6 +++ .../WORKSPACE | 0 .../relative_imports_package_mode/__main__.py | 5 ++ .../package1}/BUILD.in | 0 .../package1/BUILD.out | 11 ++++ .../package1/__init__.py | 2 + .../package1/module1.py | 0 .../package1/module2.py | 0 .../package1/my_library/BUILD.in | 7 +++ .../package1/my_library/BUILD.out | 7 +++ .../package1/my_library/__init__.py | 2 + .../package1/my_library/foo/BUILD.in | 0 .../package1/my_library/foo/BUILD.out | 7 +++ .../package1/my_library/foo/__init__.py | 2 + .../package1/subpackage1/BUILD.in | 10 ++++ .../package1/subpackage1/BUILD.out | 10 ++++ .../package1/subpackage1/__init__.py | 3 ++ .../package1/subpackage1/some_module.py | 3 ++ .../package1/subpackage1/subpackage2/BUILD.in | 0 .../subpackage1/subpackage2/BUILD.out | 16 ++++++ .../subpackage1/subpackage2/__init__.py | 0 .../subpackage1/subpackage2/library/BUILD.in | 0 .../subpackage1/subpackage2/library/BUILD.out | 7 +++ .../subpackage2/library/other_module.py | 0 .../subpackage1/subpackage2/script.py | 11 ++++ .../package2/BUILD.in | 0 .../package2/BUILD.out | 12 +++++ .../package2/__init__.py | 20 +++++++ .../package2/library/BUILD.in | 0 .../package2/library/BUILD.out | 7 +++ .../package2/library/__init__.py | 14 +++++ .../package2/module3.py | 5 ++ .../package2/module4.py | 2 + .../test.yaml | 0 .../BUILD.in | 1 + .../BUILD.out | 7 +-- .../relative_imports_project_mode/README.md | 5 ++ .../relative_imports_project_mode/WORKSPACE | 1 + .../__main__.py | 0 .../package1/module1.py | 19 +++++++ .../package1/module2.py | 17 ++++++ .../package2/BUILD.in | 0 .../package2/BUILD.out | 0 .../package2/__init__.py | 0 .../package2/module3.py | 0 .../package2/module4.py | 0 .../package2/subpackage1/module5.py | 0 .../relative_imports_project_mode/test.yaml | 15 ++++++ gazelle/pythonconfig/pythonconfig.go | 16 ++++++ 57 files changed, 373 insertions(+), 14 deletions(-) delete mode 100644 gazelle/python/testdata/relative_imports/README.md create mode 100644 gazelle/python/testdata/relative_imports_package_mode/BUILD.in create mode 100644 gazelle/python/testdata/relative_imports_package_mode/BUILD.out create mode 100644 gazelle/python/testdata/relative_imports_package_mode/README.md rename gazelle/python/testdata/{relative_imports => relative_imports_package_mode}/WORKSPACE (100%) create mode 100644 gazelle/python/testdata/relative_imports_package_mode/__main__.py rename gazelle/python/testdata/{relative_imports/package2 => relative_imports_package_mode/package1}/BUILD.in (100%) create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/BUILD.out create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/__init__.py rename gazelle/python/testdata/{relative_imports => relative_imports_package_mode}/package1/module1.py (100%) rename gazelle/python/testdata/{relative_imports => relative_imports_package_mode}/package1/module2.py (100%) create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/my_library/BUILD.in create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/my_library/BUILD.out create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/my_library/__init__.py create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/my_library/foo/BUILD.in create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/my_library/foo/BUILD.out create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/my_library/foo/__init__.py create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/BUILD.in create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/BUILD.out create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/__init__.py create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/some_module.py create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/BUILD.in create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/BUILD.out create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/__init__.py create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/library/BUILD.in create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/library/BUILD.out create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/library/other_module.py create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/script.py create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package2/BUILD.in create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package2/BUILD.out create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package2/__init__.py create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package2/library/BUILD.in create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package2/library/BUILD.out create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package2/library/__init__.py create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package2/module3.py create mode 100644 gazelle/python/testdata/relative_imports_package_mode/package2/module4.py rename gazelle/python/testdata/{relative_imports => relative_imports_package_mode}/test.yaml (100%) rename gazelle/python/testdata/{relative_imports => relative_imports_project_mode}/BUILD.in (61%) rename gazelle/python/testdata/{relative_imports => relative_imports_project_mode}/BUILD.out (70%) create mode 100644 gazelle/python/testdata/relative_imports_project_mode/README.md create mode 100644 gazelle/python/testdata/relative_imports_project_mode/WORKSPACE rename gazelle/python/testdata/{relative_imports => relative_imports_project_mode}/__main__.py (100%) create mode 100644 gazelle/python/testdata/relative_imports_project_mode/package1/module1.py create mode 100644 gazelle/python/testdata/relative_imports_project_mode/package1/module2.py create mode 100644 gazelle/python/testdata/relative_imports_project_mode/package2/BUILD.in rename gazelle/python/testdata/{relative_imports => relative_imports_project_mode}/package2/BUILD.out (100%) rename gazelle/python/testdata/{relative_imports => relative_imports_project_mode}/package2/__init__.py (100%) rename gazelle/python/testdata/{relative_imports => relative_imports_project_mode}/package2/module3.py (100%) rename gazelle/python/testdata/{relative_imports => relative_imports_project_mode}/package2/module4.py (100%) rename gazelle/python/testdata/{relative_imports => relative_imports_project_mode}/package2/subpackage1/module5.py (100%) create mode 100644 gazelle/python/testdata/relative_imports_project_mode/test.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index f2fa98f73f..ecdc129502 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,9 @@ END_UNRELEASED_TEMPLATE {#v0-0-0-changed} ### Changed +* (gazelle) For package mode, resolve dependencies when imports are relative + to the package path. This is enabled via the + `# gazelle:experimental_allow_relative_imports` true directive ({gh-issue}`2203`). * (gazelle) Types for exposed members of `python.ParserOutput` are now all public. {#v0-0-0-fixed} diff --git a/gazelle/README.md b/gazelle/README.md index 89ebaef4cd..58ec55eb11 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -121,12 +121,12 @@ gazelle_python_manifest( requirements = "//:requirements_lock.txt", # include_stub_packages: bool (default: False) # If set to True, this flag automatically includes any corresponding type stub packages - # for the third-party libraries that are present and used. For example, if you have + # for the third-party libraries that are present and used. For example, if you have # `boto3` as a dependency, and this flag is enabled, the corresponding `boto3-stubs` # package will be automatically included in the BUILD file. # - # Enabling this feature helps ensure that type hints and stubs are readily available - # for tools like type checkers and IDEs, improving the development experience and + # Enabling this feature helps ensure that type hints and stubs are readily available + # for tools like type checkers and IDEs, improving the development experience and # reducing manual overhead in managing separate stub packages. include_stub_packages = True ) @@ -220,6 +220,8 @@ Python-specific directives are as follows: | Defines the format of the distribution name in labels to third-party deps. Useful for using Gazelle plugin with other rules with different repository conventions (e.g. `rules_pycross`). Full label is always prepended with (pip) repository name, e.g. `@pip//numpy`. | | `# gazelle:python_label_normalization` | `snake_case` | | Controls how distribution names in labels to third-party deps are normalized. Useful for using Gazelle plugin with other rules with different label conventions (e.g. `rules_pycross` uses PEP-503). Can be "snake_case", "none", or "pep503". | +| `# gazelle:experimental_allow_relative_imports` | `false` | +| Controls whether Gazelle resolves dependencies for import statements that use paths relative to the current package. Can be "true" or "false".| #### Directive: `python_root`: @@ -468,7 +470,7 @@ def py_test(name, main=None, **kwargs): name = "__test__", deps = ["@pip_pytest//:pkg"], # change this to the pytest target in your repo. ) - + deps.append(":__test__") main = ":__test__.py" @@ -581,6 +583,44 @@ deps = [ ] ``` +#### Directive: `experimental_allow_relative_imports` +Enables experimental support for resolving relative imports in +`python_generation_mode package`. + +By default, when `# gazelle:python_generation_mode package` is enabled, +relative imports (e.g., from .library import foo) are not added to the +deps field of the generated target. This results in incomplete py_library +rules that lack required dependencies on sibling packages. + +Example: +Given this Python file import: +```python +from .library import add as _add +from .library import subtract as _subtract +``` + +Expected BUILD file output: +```starlark +py_library( + name = "py_default_library", + srcs = ["__init__.py"], + deps = [ + "//example/library:py_default_library", + ], + visibility = ["//visibility:public"], +) +``` + +Actual output without this annotation: +```starlark +py_library( + name = "py_default_library", + srcs = ["__init__.py"], + visibility = ["//visibility:public"], +) +``` +If the directive is set to `true`, gazelle will resolve imports +that are relative to the current package. ### Libraries diff --git a/gazelle/python/configure.go b/gazelle/python/configure.go index a00b0ba0ba..ae0f7ee1d1 100644 --- a/gazelle/python/configure.go +++ b/gazelle/python/configure.go @@ -68,6 +68,7 @@ func (py *Configurer) KnownDirectives() []string { pythonconfig.TestFilePattern, pythonconfig.LabelConvention, pythonconfig.LabelNormalization, + pythonconfig.ExperimentalAllowRelativeImports, } } @@ -222,6 +223,13 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) { default: config.SetLabelNormalization(pythonconfig.DefaultLabelNormalizationType) } + case pythonconfig.ExperimentalAllowRelativeImports: + v, err := strconv.ParseBool(strings.TrimSpace(d.Value)) + if err != nil { + log.Printf("invalid value for gazelle:%s in %q: %q", + pythonconfig.ExperimentalAllowRelativeImports, rel, d.Value) + } + config.SetExperimentalAllowRelativeImports(v) } } diff --git a/gazelle/python/file_parser.go b/gazelle/python/file_parser.go index 3f8363fbdf..cb82cb93b4 100644 --- a/gazelle/python/file_parser.go +++ b/gazelle/python/file_parser.go @@ -165,7 +165,9 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool { } } else if node.Type() == sitterNodeTypeImportFromStatement { from := node.Child(1).Content(p.code) - if strings.HasPrefix(from, ".") { + // If the import is from the current package, we don't need to add it to the modules i.e. from . import Class1. + // If the import is from a different relative package i.e. from .package1 import foo, we need to add it to the modules. + if from == "." { return true } for j := 3; j < int(node.ChildCount()); j++ { diff --git a/gazelle/python/resolve.go b/gazelle/python/resolve.go index 996cbbadc0..413e69b289 100644 --- a/gazelle/python/resolve.go +++ b/gazelle/python/resolve.go @@ -148,12 +148,61 @@ func (py *Resolver) Resolve( modules := modulesRaw.(*treeset.Set) it := modules.Iterator() explainDependency := os.Getenv("EXPLAIN_DEPENDENCY") + // Resolve relative paths for package generation + isPackageGeneration := !cfg.PerFileGeneration() && !cfg.CoarseGrainedGeneration() hasFatalError := false MODULES_LOOP: for it.Next() { mod := it.Value().(Module) - moduleParts := strings.Split(mod.Name, ".") - possibleModules := []string{mod.Name} + moduleName := mod.Name + // Transform relative imports `.` or `..foo.bar` into the package path from root. + if strings.HasPrefix(mod.From, ".") { + if !cfg.ExperimentalAllowRelativeImports() || !isPackageGeneration { + continue MODULES_LOOP + } + + // Count number of leading dots in mod.From (e.g., ".." = 2, "...foo.bar" = 3) + relativeDepth := strings.IndexFunc(mod.From, func(r rune) bool { return r != '.' }) + if relativeDepth == -1 { + relativeDepth = len(mod.From) + } + + // Extract final symbol (e.g., "some_function") from mod.Name + imported := mod.Name + if idx := strings.LastIndex(mod.Name, "."); idx >= 0 { + imported = mod.Name[idx+1:] + } + + // Optional subpath in 'from' clause, e.g. "from ...my_library.foo import x" + fromPath := strings.TrimLeft(mod.From, ".") + var fromParts []string + if fromPath != "" { + fromParts = strings.Split(fromPath, ".") + } + + // Current Bazel package as path segments + pkgParts := strings.Split(from.Pkg, "/") + + if relativeDepth-1 > len(pkgParts) { + log.Printf("ERROR: Invalid relative import %q in %q: exceeds package root.", mod.Name, mod.Filepath) + continue MODULES_LOOP + } + + // Go up relativeDepth - 1 levels + baseParts := pkgParts + if relativeDepth > 1 { + baseParts = pkgParts[:len(pkgParts)-(relativeDepth-1)] + } + // Build absolute module path + absParts := append([]string{}, baseParts...) // base path + absParts = append(absParts, fromParts...) // subpath from 'from' + absParts = append(absParts, imported) // actual imported symbol + + moduleName = strings.Join(absParts, ".") + } + + moduleParts := strings.Split(moduleName, ".") + possibleModules := []string{moduleName} for len(moduleParts) > 1 { // Iterate back through the possible imports until // a match is found. diff --git a/gazelle/python/testdata/relative_imports/README.md b/gazelle/python/testdata/relative_imports/README.md deleted file mode 100644 index 1937cbcf4a..0000000000 --- a/gazelle/python/testdata/relative_imports/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Relative imports - -This test case asserts that the generated targets handle relative imports in -Python correctly. diff --git a/gazelle/python/testdata/relative_imports_package_mode/BUILD.in b/gazelle/python/testdata/relative_imports_package_mode/BUILD.in new file mode 100644 index 0000000000..78ef0a7863 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/BUILD.in @@ -0,0 +1,2 @@ +# gazelle:python_generation_mode package +# gazelle:experimental_allow_relative_imports true diff --git a/gazelle/python/testdata/relative_imports_package_mode/BUILD.out b/gazelle/python/testdata/relative_imports_package_mode/BUILD.out new file mode 100644 index 0000000000..f51b516cab --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/BUILD.out @@ -0,0 +1,15 @@ +load("@rules_python//python:defs.bzl", "py_binary") + +# gazelle:python_generation_mode package +# gazelle:experimental_allow_relative_imports true + +py_binary( + name = "relative_imports_package_mode_bin", + srcs = ["__main__.py"], + main = "__main__.py", + visibility = ["//:__subpackages__"], + deps = [ + "//package1", + "//package2", + ], +) diff --git a/gazelle/python/testdata/relative_imports_package_mode/README.md b/gazelle/python/testdata/relative_imports_package_mode/README.md new file mode 100644 index 0000000000..eb9f8c096c --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/README.md @@ -0,0 +1,6 @@ +# Resolve deps for relative imports + +This test case verifies that the generated targets correctly handle relative imports in +Python. Specifically, when the Python generation mode is set to "package," it ensures +that relative import statements such as from .foo import X are properly resolved to +their corresponding modules. diff --git a/gazelle/python/testdata/relative_imports/WORKSPACE b/gazelle/python/testdata/relative_imports_package_mode/WORKSPACE similarity index 100% rename from gazelle/python/testdata/relative_imports/WORKSPACE rename to gazelle/python/testdata/relative_imports_package_mode/WORKSPACE diff --git a/gazelle/python/testdata/relative_imports_package_mode/__main__.py b/gazelle/python/testdata/relative_imports_package_mode/__main__.py new file mode 100644 index 0000000000..4fb887a803 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/__main__.py @@ -0,0 +1,5 @@ +from package1.module1 import function1 +from package2.module3 import function3 + +print(function1()) +print(function3()) diff --git a/gazelle/python/testdata/relative_imports/package2/BUILD.in b/gazelle/python/testdata/relative_imports_package_mode/package1/BUILD.in similarity index 100% rename from gazelle/python/testdata/relative_imports/package2/BUILD.in rename to gazelle/python/testdata/relative_imports_package_mode/package1/BUILD.in diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/BUILD.out b/gazelle/python/testdata/relative_imports_package_mode/package1/BUILD.out new file mode 100644 index 0000000000..c562ff07de --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package1/BUILD.out @@ -0,0 +1,11 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "package1", + srcs = [ + "__init__.py", + "module1.py", + "module2.py", + ], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/__init__.py b/gazelle/python/testdata/relative_imports_package_mode/package1/__init__.py new file mode 100644 index 0000000000..11ffb98647 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package1/__init__.py @@ -0,0 +1,2 @@ +def some_function(): + pass diff --git a/gazelle/python/testdata/relative_imports/package1/module1.py b/gazelle/python/testdata/relative_imports_package_mode/package1/module1.py similarity index 100% rename from gazelle/python/testdata/relative_imports/package1/module1.py rename to gazelle/python/testdata/relative_imports_package_mode/package1/module1.py diff --git a/gazelle/python/testdata/relative_imports/package1/module2.py b/gazelle/python/testdata/relative_imports_package_mode/package1/module2.py similarity index 100% rename from gazelle/python/testdata/relative_imports/package1/module2.py rename to gazelle/python/testdata/relative_imports_package_mode/package1/module2.py diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/BUILD.in b/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/BUILD.in new file mode 100644 index 0000000000..80a4a22348 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/BUILD.in @@ -0,0 +1,7 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "my_library", + srcs = ["__init__.py"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/BUILD.out b/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/BUILD.out new file mode 100644 index 0000000000..80a4a22348 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/BUILD.out @@ -0,0 +1,7 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "my_library", + srcs = ["__init__.py"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/__init__.py b/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/__init__.py new file mode 100644 index 0000000000..aaa161cd59 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/__init__.py @@ -0,0 +1,2 @@ +def some_function(): + return "some_function" diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/foo/BUILD.in b/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/foo/BUILD.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/foo/BUILD.out b/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/foo/BUILD.out new file mode 100644 index 0000000000..58498ee3b3 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/foo/BUILD.out @@ -0,0 +1,7 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "foo", + srcs = ["__init__.py"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/foo/__init__.py b/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/foo/__init__.py new file mode 100644 index 0000000000..aaa161cd59 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package1/my_library/foo/__init__.py @@ -0,0 +1,2 @@ +def some_function(): + return "some_function" diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/BUILD.in b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/BUILD.in new file mode 100644 index 0000000000..0a5b665c8d --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/BUILD.in @@ -0,0 +1,10 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "subpackage1", + srcs = [ + "__init__.py", + "some_module.py", + ], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/BUILD.out b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/BUILD.out new file mode 100644 index 0000000000..0a5b665c8d --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/BUILD.out @@ -0,0 +1,10 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "subpackage1", + srcs = [ + "__init__.py", + "some_module.py", + ], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/__init__.py b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/__init__.py new file mode 100644 index 0000000000..02feaeb848 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/__init__.py @@ -0,0 +1,3 @@ + +def some_init(): + return "some_init" diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/some_module.py b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/some_module.py new file mode 100644 index 0000000000..3cae706242 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/some_module.py @@ -0,0 +1,3 @@ + +def some_function(): + return "some_function" diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/BUILD.in b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/BUILD.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/BUILD.out b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/BUILD.out new file mode 100644 index 0000000000..8c34081210 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/BUILD.out @@ -0,0 +1,16 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "subpackage2", + srcs = [ + "__init__.py", + "script.py", + ], + visibility = ["//:__subpackages__"], + deps = [ + "//package1/my_library", + "//package1/my_library/foo", + "//package1/subpackage1", + "//package1/subpackage1/subpackage2/library", + ], +) diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/__init__.py b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/library/BUILD.in b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/library/BUILD.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/library/BUILD.out b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/library/BUILD.out new file mode 100644 index 0000000000..9fe2e3d1d7 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/library/BUILD.out @@ -0,0 +1,7 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "library", + srcs = ["other_module.py"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/library/other_module.py b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/library/other_module.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/script.py b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/script.py new file mode 100644 index 0000000000..e93f07719a --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package1/subpackage1/subpackage2/script.py @@ -0,0 +1,11 @@ +from ...my_library import ( + some_function, +) # Import path should be package1.my_library.some_function +from ...my_library.foo import ( + some_function, +) # Import path should be package1.my_library.foo.some_function +from .library import ( + other_module, +) # Import path should be package1.subpackage1.subpackage2.library.other_module +from .. import some_module # Import path should be package1.subpackage1.some_module +from .. import some_function # Import path should be package1.subpackage1.some_function diff --git a/gazelle/python/testdata/relative_imports_package_mode/package2/BUILD.in b/gazelle/python/testdata/relative_imports_package_mode/package2/BUILD.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/relative_imports_package_mode/package2/BUILD.out b/gazelle/python/testdata/relative_imports_package_mode/package2/BUILD.out new file mode 100644 index 0000000000..bd78108159 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package2/BUILD.out @@ -0,0 +1,12 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "package2", + srcs = [ + "__init__.py", + "module3.py", + "module4.py", + ], + visibility = ["//:__subpackages__"], + deps = ["//package2/library"], +) diff --git a/gazelle/python/testdata/relative_imports_package_mode/package2/__init__.py b/gazelle/python/testdata/relative_imports_package_mode/package2/__init__.py new file mode 100644 index 0000000000..3d19d80e21 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package2/__init__.py @@ -0,0 +1,20 @@ +from .library import add as _add +from .library import divide as _divide +from .library import multiply as _multiply +from .library import subtract as _subtract + + +def add(a, b): + return _add(a, b) + + +def divide(a, b): + return _divide(a, b) + + +def multiply(a, b): + return _multiply(a, b) + + +def subtract(a, b): + return _subtract(a, b) diff --git a/gazelle/python/testdata/relative_imports_package_mode/package2/library/BUILD.in b/gazelle/python/testdata/relative_imports_package_mode/package2/library/BUILD.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/relative_imports_package_mode/package2/library/BUILD.out b/gazelle/python/testdata/relative_imports_package_mode/package2/library/BUILD.out new file mode 100644 index 0000000000..d704b7fe93 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package2/library/BUILD.out @@ -0,0 +1,7 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "library", + srcs = ["__init__.py"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/relative_imports_package_mode/package2/library/__init__.py b/gazelle/python/testdata/relative_imports_package_mode/package2/library/__init__.py new file mode 100644 index 0000000000..5f8fc62492 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package2/library/__init__.py @@ -0,0 +1,14 @@ +def add(a, b): + return a + b + + +def divide(a, b): + return a / b + + +def multiply(a, b): + return a * b + + +def subtract(a, b): + return a - b diff --git a/gazelle/python/testdata/relative_imports_package_mode/package2/module3.py b/gazelle/python/testdata/relative_imports_package_mode/package2/module3.py new file mode 100644 index 0000000000..6b955cfda6 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package2/module3.py @@ -0,0 +1,5 @@ +from .library import function5 + + +def function3(): + return "function3 " + function5() diff --git a/gazelle/python/testdata/relative_imports_package_mode/package2/module4.py b/gazelle/python/testdata/relative_imports_package_mode/package2/module4.py new file mode 100644 index 0000000000..6e69699985 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_package_mode/package2/module4.py @@ -0,0 +1,2 @@ +def function4(): + return "function4" diff --git a/gazelle/python/testdata/relative_imports/test.yaml b/gazelle/python/testdata/relative_imports_package_mode/test.yaml similarity index 100% rename from gazelle/python/testdata/relative_imports/test.yaml rename to gazelle/python/testdata/relative_imports_package_mode/test.yaml diff --git a/gazelle/python/testdata/relative_imports/BUILD.in b/gazelle/python/testdata/relative_imports_project_mode/BUILD.in similarity index 61% rename from gazelle/python/testdata/relative_imports/BUILD.in rename to gazelle/python/testdata/relative_imports_project_mode/BUILD.in index c04b5e5434..1059942bfb 100644 --- a/gazelle/python/testdata/relative_imports/BUILD.in +++ b/gazelle/python/testdata/relative_imports_project_mode/BUILD.in @@ -1 +1,2 @@ # gazelle:resolve py resolved_package //package2:resolved_package +# gazelle:python_generation_mode project diff --git a/gazelle/python/testdata/relative_imports/BUILD.out b/gazelle/python/testdata/relative_imports_project_mode/BUILD.out similarity index 70% rename from gazelle/python/testdata/relative_imports/BUILD.out rename to gazelle/python/testdata/relative_imports_project_mode/BUILD.out index bf9524480a..acdc914541 100644 --- a/gazelle/python/testdata/relative_imports/BUILD.out +++ b/gazelle/python/testdata/relative_imports_project_mode/BUILD.out @@ -1,9 +1,10 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_library") # gazelle:resolve py resolved_package //package2:resolved_package +# gazelle:python_generation_mode project py_library( - name = "relative_imports", + name = "relative_imports_project_mode", srcs = [ "package1/module1.py", "package1/module2.py", @@ -12,12 +13,12 @@ py_library( ) py_binary( - name = "relative_imports_bin", + name = "relative_imports_project_mode_bin", srcs = ["__main__.py"], main = "__main__.py", visibility = ["//:__subpackages__"], deps = [ - ":relative_imports", + ":relative_imports_project_mode", "//package2", ], ) diff --git a/gazelle/python/testdata/relative_imports_project_mode/README.md b/gazelle/python/testdata/relative_imports_project_mode/README.md new file mode 100644 index 0000000000..3c95a36e62 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_project_mode/README.md @@ -0,0 +1,5 @@ +# Relative imports + +This test case asserts that the generated targets handle relative imports in +Python correctly. This tests that if python generation mode is project, +the relative paths are included in the subdirectories. diff --git a/gazelle/python/testdata/relative_imports_project_mode/WORKSPACE b/gazelle/python/testdata/relative_imports_project_mode/WORKSPACE new file mode 100644 index 0000000000..4959898cdd --- /dev/null +++ b/gazelle/python/testdata/relative_imports_project_mode/WORKSPACE @@ -0,0 +1 @@ +# This is a test data Bazel workspace. diff --git a/gazelle/python/testdata/relative_imports/__main__.py b/gazelle/python/testdata/relative_imports_project_mode/__main__.py similarity index 100% rename from gazelle/python/testdata/relative_imports/__main__.py rename to gazelle/python/testdata/relative_imports_project_mode/__main__.py diff --git a/gazelle/python/testdata/relative_imports_project_mode/package1/module1.py b/gazelle/python/testdata/relative_imports_project_mode/package1/module1.py new file mode 100644 index 0000000000..28502f1f84 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_project_mode/package1/module1.py @@ -0,0 +1,19 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .module2 import function2 + + +def function1(): + return "function1 " + function2() diff --git a/gazelle/python/testdata/relative_imports_project_mode/package1/module2.py b/gazelle/python/testdata/relative_imports_project_mode/package1/module2.py new file mode 100644 index 0000000000..0cbc5f0be0 --- /dev/null +++ b/gazelle/python/testdata/relative_imports_project_mode/package1/module2.py @@ -0,0 +1,17 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def function2(): + return "function2" diff --git a/gazelle/python/testdata/relative_imports_project_mode/package2/BUILD.in b/gazelle/python/testdata/relative_imports_project_mode/package2/BUILD.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/relative_imports/package2/BUILD.out b/gazelle/python/testdata/relative_imports_project_mode/package2/BUILD.out similarity index 100% rename from gazelle/python/testdata/relative_imports/package2/BUILD.out rename to gazelle/python/testdata/relative_imports_project_mode/package2/BUILD.out diff --git a/gazelle/python/testdata/relative_imports/package2/__init__.py b/gazelle/python/testdata/relative_imports_project_mode/package2/__init__.py similarity index 100% rename from gazelle/python/testdata/relative_imports/package2/__init__.py rename to gazelle/python/testdata/relative_imports_project_mode/package2/__init__.py diff --git a/gazelle/python/testdata/relative_imports/package2/module3.py b/gazelle/python/testdata/relative_imports_project_mode/package2/module3.py similarity index 100% rename from gazelle/python/testdata/relative_imports/package2/module3.py rename to gazelle/python/testdata/relative_imports_project_mode/package2/module3.py diff --git a/gazelle/python/testdata/relative_imports/package2/module4.py b/gazelle/python/testdata/relative_imports_project_mode/package2/module4.py similarity index 100% rename from gazelle/python/testdata/relative_imports/package2/module4.py rename to gazelle/python/testdata/relative_imports_project_mode/package2/module4.py diff --git a/gazelle/python/testdata/relative_imports/package2/subpackage1/module5.py b/gazelle/python/testdata/relative_imports_project_mode/package2/subpackage1/module5.py similarity index 100% rename from gazelle/python/testdata/relative_imports/package2/subpackage1/module5.py rename to gazelle/python/testdata/relative_imports_project_mode/package2/subpackage1/module5.py diff --git a/gazelle/python/testdata/relative_imports_project_mode/test.yaml b/gazelle/python/testdata/relative_imports_project_mode/test.yaml new file mode 100644 index 0000000000..fcea77710f --- /dev/null +++ b/gazelle/python/testdata/relative_imports_project_mode/test.yaml @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- diff --git a/gazelle/pythonconfig/pythonconfig.go b/gazelle/pythonconfig/pythonconfig.go index 866339d449..e0a2b8a469 100644 --- a/gazelle/pythonconfig/pythonconfig.go +++ b/gazelle/pythonconfig/pythonconfig.go @@ -91,6 +91,9 @@ const ( // names of labels to third-party dependencies are normalized. Supported values // are 'none', 'pep503' and 'snake_case' (default). See LabelNormalizationType. LabelNormalization = "python_label_normalization" + // ExperimentalAllowRelativeImports represents the directive that controls + // whether relative imports are allowed. + ExperimentalAllowRelativeImports = "experimental_allow_relative_imports" ) // GenerationModeType represents one of the generation modes for the Python @@ -177,6 +180,7 @@ type Config struct { testFilePattern []string labelConvention string labelNormalization LabelNormalizationType + experimentalAllowRelativeImports bool } type LabelNormalizationType int @@ -212,6 +216,7 @@ func New( testFilePattern: strings.Split(DefaultTestFilePatternString, ","), labelConvention: DefaultLabelConvention, labelNormalization: DefaultLabelNormalizationType, + experimentalAllowRelativeImports: false, } } @@ -244,6 +249,7 @@ func (c *Config) NewChild() *Config { testFilePattern: c.testFilePattern, labelConvention: c.labelConvention, labelNormalization: c.labelNormalization, + experimentalAllowRelativeImports: c.experimentalAllowRelativeImports, } } @@ -520,6 +526,16 @@ func (c *Config) LabelNormalization() LabelNormalizationType { return c.labelNormalization } +// SetExperimentalAllowRelativeImports sets whether relative imports are allowed. +func (c *Config) SetExperimentalAllowRelativeImports(allowRelativeImports bool) { + c.experimentalAllowRelativeImports = allowRelativeImports +} + +// ExperimentalAllowRelativeImports returns whether relative imports are allowed. +func (c *Config) ExperimentalAllowRelativeImports() bool { + return c.experimentalAllowRelativeImports +} + // FormatThirdPartyDependency returns a label to a third-party dependency performing all formating and normalization. func (c *Config) FormatThirdPartyDependency(repositoryName string, distributionName string) label.Label { conventionalDistributionName := strings.ReplaceAll(c.labelConvention, distributionNameLabelConventionSubstitution, distributionName) From f6feca1e00d9ae768243f05e677b7b636b9ad7ba Mon Sep 17 00:00:00 2001 From: armandomontanez Date: Sat, 21 Jun 2025 19:18:35 -0700 Subject: [PATCH 43/43] fix: Fix bazel vendor support for requirements with environment markers (#2997) Fixes `bazel vendor` support for requirements files that contain environment markers. During a vendored `bazel build`, when evaluate_markers_py() is run it needs PYTHONHOME set to properly find the home of the vendored libraries. Resolves #2996 --------- Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> --- .bazelci/presubmit.yml | 9 +++++++++ CHANGELOG.md | 4 +++- examples/bzlmod/.bazelignore | 1 + examples/bzlmod/.gitignore | 1 + python/private/pypi/evaluate_markers.bzl | 13 ++++++++----- 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 01af217924..07ffa4eaac 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -272,6 +272,15 @@ tasks: working_directory: examples/bzlmod platform: debian11 bazel: 7.x + integration_test_bzlmod_ubuntu_vendor: + <<: *reusable_build_test_all + name: "examples/bzlmod: bazel vendor" + working_directory: examples/bzlmod + platform: ubuntu2004 + shell_commands: + - "bazel vendor --vendor_dir=./vendor //..." + - "bazel build --vendor_dir=./vendor //..." + - "rm -rf ./vendor" integration_test_bzlmod_macos: <<: *reusable_build_test_all <<: *coverage_targets_example_bzlmod diff --git a/CHANGELOG.md b/CHANGELOG.md index ecdc129502..4facff4917 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,7 +61,9 @@ END_UNRELEASED_TEMPLATE {#v0-0-0-fixed} ### Fixed -* Nothing fixed. +* (pypi) Fixes an issue where builds using a `bazel vendor` vendor directory + would fail if the constraints file contained environment markers. Fixes + [#2996](https://github.com/bazel-contrib/rules_python/issues/2996). {#v0-0-0-added} ### Added diff --git a/examples/bzlmod/.bazelignore b/examples/bzlmod/.bazelignore index 3927f8e910..536ded93a6 100644 --- a/examples/bzlmod/.bazelignore +++ b/examples/bzlmod/.bazelignore @@ -1,2 +1,3 @@ other_module py_proto_library/foo_external +vendor diff --git a/examples/bzlmod/.gitignore b/examples/bzlmod/.gitignore index ac51a054d2..0f6c6316dd 100644 --- a/examples/bzlmod/.gitignore +++ b/examples/bzlmod/.gitignore @@ -1 +1,2 @@ bazel-* +vendor/ diff --git a/python/private/pypi/evaluate_markers.bzl b/python/private/pypi/evaluate_markers.bzl index 58a29a9181..2b805c33e6 100644 --- a/python/private/pypi/evaluate_markers.bzl +++ b/python/private/pypi/evaluate_markers.bzl @@ -78,14 +78,16 @@ def evaluate_markers_py(mrctx, *, requirements, python_interpreter, python_inter out_file = mrctx.path("requirements_with_markers.out.json") mrctx.file(in_file, json.encode(requirements)) + interpreter = pypi_repo_utils.resolve_python_interpreter( + mrctx, + python_interpreter = python_interpreter, + python_interpreter_target = python_interpreter_target, + ) + pypi_repo_utils.execute_checked( mrctx, op = "ResolveRequirementEnvMarkers({})".format(in_file), - python = pypi_repo_utils.resolve_python_interpreter( - mrctx, - python_interpreter = python_interpreter, - python_interpreter_target = python_interpreter_target, - ), + python = interpreter, arguments = [ "-m", "python.private.pypi.requirements_parser.resolve_target_platforms", @@ -94,6 +96,7 @@ def evaluate_markers_py(mrctx, *, requirements, python_interpreter, python_inter ], srcs = srcs, environment = { + "PYTHONHOME": str(interpreter.dirname), "PYTHONPATH": [ Label("@pypi__packaging//:BUILD.bazel"), Label("//:BUILD.bazel"),