8000 feat(bzlmod): Allowing multiple python.toolchain extension calls (#1230) · bazel-contrib/rules_python@60c61e5 · GitHub
[go: up one dir, main page]
Skip to content

Commit 60c61e5

Browse files
chrislovecnmchrislovecnm
authored
feat(bzlmod): Allowing multiple python.toolchain extension calls (#1230)
We do this work for two reasons. First, we must support Module dependencies and sub-modules using `python.toolchain`. There are already two known instances of sub-modules setting up a Python toolchain and colliding with another module (nanobind and rules_testing both run into this). Second, the upcoming multi-version support is going to work by having each `python.toolchain()` call register its particular version with the extra toolchain constraint. This also helps unify the version-aware and non-version-aware code paths (the non-version aware paths are just version-aware with a single version registered as the default) This commit implements various business logic in the toolchain class. Toolchains in Sub Modules It will create a toolchain in a sub-module if the toolchain of the same name does not exist in the root module. The extension stops name clashing between toolchains in the root module and sub-modules. You cannot configure more than one toolchain as the default toolchain. Toolchain set as the default version. This extension will not create a toolchain in a sub-module if the sub-module toolchain is marked as the default version. If you have more than one toolchain in your root module, you need to set one of the toolchains as the default version. If there is only one toolchain, it is set as the default toolchain. See #1229 for more information
1 parent 693a158 commit 60c61e5

File tree

12 files changed

+320
-40
lines changed

12 files changed

+320
-40
lines changed

‎examples/bzlmod/.bazelrc

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
11
common --experimental_enable_bzlmod
22

33
coverage --java_runtime_version=remotejdk_11
4+
5+
test --test_output=errors --enable_runfiles
6+
7+
# Windows requires these for multi-python support:
8+
build --enable_runfiles
9+
10+
startup --windows_enable_symlinks

‎examples/bzlmod/BUILD.bazel

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,21 @@
1+
# Load various rules so that we can have bazel download
2+
# various rulesets and dependencies.
3+
# The `load` statement imports the symbol for the rule, in the defined
4+
# ruleset. When the symbol is loaded you can use the rule.
5+
6+
# The names @pip and @python_39 are values that are repository
7+
# names. Those names are defined in the MODULES.bazel file.
18
load("@bazel_skylib//rules:build_test.bzl", "build_test")
29
load("@pip//:requirements.bzl", "all_requirements", "all_whl_requirements", "requirement")
3-
load("@python3_9//:defs.bzl", py_test_with_transition = "py_test")
10+
load("@python_39//:defs.bzl", py_test_with_transition = "py_test")
11+
12+
# This is not working yet till the toolchain hub registration is working
13+
# load("@python_310//:defs.bzl", py_binary_310 = "py_binary")
414
load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
515
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
616

17+
# This stanza calls a rule that generates targets for managing pip dependencies
18+
# with pip-compile.
719
compile_pip_requirements(
820
name = "requirements",
921
extra_args = ["--allow-unsafe"],
@@ -12,6 +24,11 @@ compile_pip_requirements(
1224
requirements_windows = "requirements_windows.txt",
1325
)
1426

27+
# The rules below are language specific rules defined in
28+
# rules_python. See
29+
# https://bazel.build/reference/be/python
30+
31+
# see https://bazel.build/reference/be/python#py_library
1532
py_library(
1633
name = "lib",
1734
srcs = ["lib.py"],
@@ -22,6 +39,7 @@ py_library(
2239
],
2340
)
2441

42+
# see https://bazel.build/reference/be/python#py_binary
2543
py_binary(
2644
name = "bzlmod",
2745
srcs = ["__main__.py"],
@@ -32,6 +50,23 @@ py_binary(
3250
],
3351
)
3452

53+
# This is still WIP. Not working till we have the toolchain
54+
# registration functioning.
55+
56+
# This is used for testing mulitple versions of Python. This is
57+
# used only when you need to support multiple versions of Python
58+
# in the same project.
59+
# py_binary_310(
60+
# name = "main_310",
61+
# srcs = ["__main__.py"],
62+
# main = "__main__.py",
63+
# visibility = ["//:__subpackages__"],
64+
# deps = [
65+
# ":lib",
66+
# ],
67+
# )
68+
69+
# see https://bazel.build/reference/be/python#py_test
3570
py_test(
3671
name = "test",
3772
srcs = ["test.py"],
@@ -46,6 +81,11 @@ py_test_with_transition(
4681
deps = [":lib"],
4782
)
4883

84+
# This example is also used for integration tests within
85+
# rules_python. We are using
86+
# https://github.com/bazelbuild/bazel-skylib
87+
# to run some of the tests.
88+
# See: https://github.com/bazelbuild/bazel-skylib/blob/main/docs/build_test_doc.md
4989
build_test(
5090
name = "all_wheels",
5191
targets = all_whl_requirements,

‎examples/bzlmod/MODULE.bazel

Lines changed: 66 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,32 +11,89 @@ local_path_override(
1111
path = "../..",
1212
)
1313

14+
# This name is passed into python.toolchain and it's use_repo statement.
15+
# We also use the same value in the python.host_python_interpreter call.
16+
PYTHON_NAME_39 = "python_39"
17+
18+
PYTHON_39_TOOLCHAINS = PYTHON_NAME_39 + "_toolchains"
19+
20+
INTERPRETER_NAME_39 = "interpreter_39"
21+
22+
PYTHON_NAME_310 = "python_310"
23+
24+
PYTHON_310_TOOLCHAINS = PYTHON_NAME_310 + "_toolchains"
25+
26+
INTERPRETER_NAME_310 = "interpreter_310"
27+
28+
# We next initialize the python toolchain using the extension.
29+
# You can set different Python versions in this block.
1430
python = use_extension("@rules_python//python/extensions:python.bzl", "python")
1531
python.toolchain(
16-
name = "python3_9",
32+
# This name is used in the various use_repo statements
33+
# below, and in the local extension that is in this
34+
# example.
35+
name = PYTHON_NAME_39,
1736
configure_coverage_tool = True,
37+
# Only set when you have mulitple toolchain versions.
38+
is_default = True,
1839
python_version = "3.9",
1940
)
20-
use_repo(python, "python3_9")
21-
use_repo(python, "python3_9_toolchains")
2241

42+
# We are also using a second version of Python in this project.
43+
# Typically you will only need a single version of Python, but
44+
# If you need a different vesion we support more than one.
45+
# Note: we do not supporting using multiple pip extensions, this is
46+
# work in progress.
47+
python.toolchain(
48+
name = PYTHON_NAME_310,
49+
configure_coverage_tool = True,
50+
python_version = "3.10",
51+
)
52+
53+
# use_repo imports one or more repos generated by the given module extension
54+
# into the scope of the current module. We are importing the various repos
55+
# created by the above python.toolchain calls.
56+
use_repo(
57+
python,
58+
PYTHON_NAME_39,
59+
PYTHON_39_TOOLCHAINS,
60+
PYTHON_NAME_310,
61+
PYTHON_310_TOOLCHAINS,
62+
)
63+
64+
# This call registers the Python toolchains.
65+
# Note: there is work under way to move this code to within
66+
# rules_python, and the user won't have to make this call,
67+
# unless they are registering custom toolchains.
2368
register_toolchains(
24-
"@python3_9_toolchains//:all",
69+
"@{}//:all".format(PYTHON_39_TOOLCHAINS),
70+
"@{}//:all".format(PYTHON_310_TOOLCHAINS),
2571
)
2672

73+
# The interpreter extension discovers the platform specific Python binary.
74+
# It creates a symlink to the binary, and we pass the label to the following
75+
# pip.parse call.
2776
interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter")
2877
interpreter.install(
29-
name = "interpreter_python3_9",
30-
python_name = "python3_9",
78+
name = INTERPRETER_NAME_39,
79+
python_name = PYTHON_NAME_39,
80+
)
81+
82+
# This second call is only needed if you are using mulitple different
83+
# Python versions/interpreters.
84+
interpreter.install(
85+
name = INTERPRETER_NAME_310,
86+
python_name = PYTHON_NAME_310,
3187
)
32-
use_repo(interpreter, "interpreter_python3_9")
88+
use_repo(interpreter, INTERPRETER_NAME_39, INTERPRETER_NAME_310)
3389

3490
pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
3591
pip.parse(
3692
name = "pip",
37-
# Intentionally set it false because the "true" case is already covered by examples/bzlmod_build_file_generation
93+
# Intentionally set it false because the "true" case is already
94+
# covered by examples/bzlmod_build_file_generation
3895
incompatible_generate_aliases = False,
39-
python_interpreter_target = "@interpreter_python3_9//:python",
96+
python_interpreter_target = "@{}//:python".format(INTERPRETER_NAME_39),
4097
requirements_lock = "//:requirements_lock.txt",
4198
requirements_windows = "//:requirements_windows.txt",
4299
)

‎examples/bzlmod/__main__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
# limitations under the License.
1414

1515
from lib import main
16+
import sys
1617

1718
if __name__ == "__main__":
1819
print(main([["A", 1], ["B", 2]]))
20+
print(sys.version)

‎examples/bzlmod/other_module/MODULE.bazel

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,56 @@ module(
22
name = "other_module",
33
)
44

5+
# This module is using the same version of rules_python
6+
# that the parent module uses.
57
bazel_dep(name = "rules_python", version = "")
8+
9+
# It is not best practice to use a python.toolchian in
10+
# a submodule. This code only exists to test that
11+
# we support doing this. This code is only for rules_python
12+
# testing purposes.
13+
PYTHON_NAME_39 = "python_39"
14+
15+
PYTHON_39_TOOLCHAINS = PYTHON_NAME_39 + "_toolchains"
16+
17+
PYTHON_NAME_311 = "python_311"
18+
19+
PYTHON_311_TOOLCHAINS = PYTHON_NAME_311 + "_toolchains"
20+
21+
python = use_extension("@rules_python//python/extensions:python.bzl", "python")
22+
python.toolchain(
23+
# This name is used in the various use_repo statements
24+
# below, and in the local extension that is in this
25+
# example.
26+
name = PYTHON_NAME_39,
27+
configure_coverage_tool = True,
28+
python_version = "3.9",
29+
)
30+
python.toolchain(
31+
# This name is used in the various use_repo statements
32+
# below, and in the local extension that is in this
33+
# example.
34+
name = PYTHON_NAME_311,
35+
configure_coverage_tool = True,
36+
# In a submodule this is ignored
37+
is_default = True,
38+
python_version = "3.11",
39+
)
40+
41+
# created by the above python.toolchain calls.
42+
use_repo(
43+
python,
44+
PYTHON_NAME_39,
45+
PYTHON_39_TOOLCHAINS,
46+
PYTHON_NAME_311,
47+
PYTHON_311_TOOLCHAINS,
48+
)
49+
50+
# This call registers the Python toolchains.
51+
# Note: there is work under way to move this code to within
52+
# rules_python, and the user won't have to make this call,
53+
# unless they are registering custom toolchains.
54+
register_toolchains(
55+
"@{}//:all".format(PYTHON_39_TOOLCHAINS),
56+
"@{}//:all".format(PYTHON_311_TOOLCHAINS),
57+
)

‎examples/bzlmod/other_module/other_module/pkg/BUILD.bazel

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
load("@python_311//:defs.bzl", py_binary_311 = "py_binary")
12
load("@rules_python//python:defs.bzl", "py_library")
23

34
py_library(
@@ -8,4 +9,15 @@ py_library(
89
deps = ["@rules_python//python/runfiles"],
910
)
1011

12+
# This is used for testing mulitple versions of Python. This is
13+
# used only when you need to support multiple versions of Python
14+
# in the same project.
15+
py_binary_311(
16+
name = "lib_311",
17+
srcs = ["lib.py"],
18+
data = ["data/data.txt"],
19+
visibility = ["//visibility:public"],
20+
deps = ["@rules_python//python/runfiles"],
21+
)
22+
1123
exports_files(["data/data.txt"])

‎examples/bzlmod/runfiles/BUILD.bazel

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
load("@rules_python//python:defs.bzl", "py_test")
22

3-
# gazelle:ignore
43
py_test(
54
name = "runfiles_test",
65
srcs = ["runfiles_test.py"],

‎examples/bzlmod_build_file_generation/BUILD.bazel

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
# requirements.
88
load("@bazel_gazelle//:def.bzl", "gazelle")
99
load("@pip//:requirements.bzl", "all_whl_requirements")
10-
load("@python3//:defs.bzl", py_test_with_transition = "py_test")
10+
load("@python//:defs.bzl", py_test_with_transition = "py_test")
1111
load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
1212
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
1313
load("@rules_python_gazelle_plugin//:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS")

‎examples/bzlmod_build_file_generation/MODULE.bazel

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,11 @@ python = use_extension("@rules_python//python/extensions:python.bzl", "python")
4646

4747
# This name is passed into python.toolchain and it's use_repo statement.
4848
# We also use the same name for python.host_python_interpreter.
49-
PYTHON_NAME = "python3"
49+
PYTHON_NAME = "python"
50+
51+
PYTHON_TOOLCHAINS = PYTHON_NAME + "_toolchains"
52+
53+
INTERPRETER_NAME = "interpreter"
5054

5155
# We next initialize the python toolchain using the extension.
5256
# You can set different Python versions in this block.
@@ -56,30 +60,35 @@ python.toolchain(
5660
# example.
5761
name = PYTHON_NAME,
5862
configure_coverage_tool = True,
63+
is_default = True,
5964
python_version = "3.9",
6065
)
6166

6267
# Import the python repositories generated by the given module extension
6368
# into the scope of the current module.
6469
# All of the python3 repositories use the PYTHON_NAME as there prefix. They
6570
# are not catenated for ease of reading.
66-
use_repo(python, PYTHON_NAME, "python3_toolchains")
71+
use_repo(
72+
python,
73+
PYTHON_NAME,
74+
PYTHON_TOOLCHAINS,
75+
)
6776

6877
# Register an already-defined toolchain so that Bazel can use it during
6978
# toolchain resolution.
7079
register_toolchains(
71-
"@python3_toolchains//:all",
80+
"@{}//:all".format(PYTHON_TOOLCHAINS),
7281
)
7382

7483
# The interpreter extension discovers the platform specific Python binary.
7584
# It creates a symlink to the binary, and we pass the label to the following
7685
# pip.parse call.
7786
interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter")
7887
interpreter.install(
79-
name = "interpreter_python3",
88+
name = INTERPRETER_NAME,
8089
python_name = PYTHON_NAME,
8190
)
82-
use_repo(interpreter, "interpreter_python3")
91+
use_repo(interpreter, INTERPRETER_NAME)
8392

8493
# Use the extension, pip.parse, to call the `pip_repository` rule that invokes
8594
# `pip`, with `incremental` set. The pip call accepts a locked/compiled
@@ -102,7 +111,7 @@ pip.parse(
102111
# is used for both resolving dependencies and running tests/binaries.
103112
# If this isn't specified, then you'll get whatever is locally installed
104113
# on your system.
105-
python_interpreter_target = "@interpreter_python3//:python",
114+
python_interpreter_target = "@{}//:python".format(INTERPRETER_NAME),
106115
requirements_lock = "//:requirements_lock.txt",
107116
requirements_windows = "//:requirements_windows.txt",
108117
)

‎python/extensions/private/interpreter_hub.bzl

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,9 @@ load("//python/private:toolchains_repo.bzl", "get_host_os_arch", "get_host_platf
1919

2020
_build_file_for_hub_template = """
2121
INTERPRETER_LABELS = {{
22-
{lines}
22+
{interpreter_labels}
2323
}}
24+
DEFAULT_TOOLCHAIN_NAME = "{default}"
2425
"""
2526

2627
_line_for_hub_template = """\
@@ -35,13 +36,19 @@ def _hub_repo_impl(rctx):
3536
is_windows = (os == WINDOWS_NAME)
3637
path = "python.exe" if is_windows else "bin/python3"
3738

38-
lines = "\n".join([_line_for_hub_template.format(
39+
interpreter_labels = "\n".join([_line_for_hub_template.format(
3940
name = name,
4041
platform = platform,
4142
path = path,
4243
) for name in rctx.attr.toolchains])
4344

44-
rctx.file("interpreters.bzl", _build_file_for_hub_template.format(lines = lines))
45+
rctx.file(
46+
"interpreters.bzl",
47+
_build_file_for_hub_template.format(
48+
interpreter_labels = interpreter_labels,
49+
default = rctx.attr.default_toolchain,
50+
),
51+
)
4552

4653
hub_repo = repository_rule(
4754
doc = """\
@@ -50,6 +57,10 @@ and the labels to said interpreters. This map is used to by the interpreter hub
5057
""",
5158
implementation = _hub_repo_impl,
5259
attrs = {
60+
"default_toolchain": attr.string(
61+
doc = "Name of the default toolchain",
62+
mandatory = True,
63+
),
5364
"toolchains": attr.string_list(
5465
doc = "List of the base names the toolchain repo defines.",
5566
mandatory = True,

0 commit comments

Comments
 (0)
0