10000 feat(bzlmod): Allowing multiple python.toolchain extension calls · bazel-contrib/rules_python@e516a59 · GitHub
[go: up one dir, main page]
Skip to content

Commit e516a59

Browse files
chrislovecnmchrislovecnm
committed
feat(bzlmod): Allowing multiple python.toolchain extension calls
We do this work for two reasons. First, we must support Module dependencies and sub-modules using python.toolchain. Second, we needed this commit to supporting using multiple toolchains with bzlmod. This commit modifies the python.toolchain extension to handle being called multiple times. We are modeling how the multiple Python toolchains work. This commit implements various business logic in the toolchain class as follows. Toolchains in Sub Modules It will create a toolchain in a sub-module if the toolchain of the same name does not exist in the root module. The extension stops name clashing between toolchains in the root module and sub-modules. You cannot configure more than one toolchain as the default toolchain. Toolchain set as the default version. This extension will not create a toolchain in a sub-module if the sub-module toolchain is marked as the default version. If you have more than one toolchain in your root module, you need to set one of the toolchains as the default version. If there is only one toolchain, it is set as the default toolchain.
1 parent 16126d0 commit e516a59

File tree

4 files changed

+130
-25
lines changed

4 files changed

+130
-25
lines changed

‎examples/bzlmod/MODULE.bazel

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,22 +10,34 @@ local_path_override(
1010
path = "../..",
1111
)
1212

13+
PYTHON_NAME = "python3_9"
14+
15+
PYTHON_TOOLCHAINS = PYTHON_NAME + "_toolchains"
16+
1317
python = use_extension("@rules_python//python/extensions:python.bzl", "python")
1418
python.toolchain(
15-
name = "python3_9",
19+
name = PYTHON_NAME,
1620
configure_coverage_tool = True,
1721
python_version = "3.9",
1822
)
19-
use_repo(python, "python3_9")
20-
use_repo(python, "python3_9_toolchains")
23+
use_repo(python, PYTHON_NAME)
24+
use_repo(python, PYTHON_TOOLCHAINS)
2125

2226
register_toolchains(
23-
"@python3_9_toolchains//:all",
27+
"@{}//:all".format(PYTHON_TOOLCHAINS),
28+
)
29+
30+
interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter")
31+
interpreter.install(
32+
name = "interpreter_python_3_9",
33+
python_name = PYTHON_NAME,
2434
)
35+
use_repo(interpreter, "interpreter_python_3_9")
2536

2637
pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
2738
pip.parse(
2839
name = "pip",
40+
python_interpreter_target = "@interpreter_python_3_9//:python",
2941
requirements_lock = "//:requirements_lock.txt",
3042
requirements_windows = "//:requirements_windows.txt",
3143
)

‎examples/bzlmod_build_file_generation/MODULE.bazel

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ python = use_extension("@rules_python//python/extensions:python.bzl", "python")
4848
# We also use the same name for python.host_python_interpreter.
4949
PYTHON_NAME = "python3"
5050

51+
PYTHON_TOOLCHAINS = PYTHON_NAME + "_toolchains"
52+
5153
# We next initialize the python toolchain using the extension.
5254
# You can set different Python versions in this block.
5355
python.toolchain(
@@ -63,12 +65,12 @@ python.toolchain(
6365
# into the scope of the current module.
6466
# All of the python3 repositories use the PYTHON_NAME as there prefix. They
6567
# are not catenated for ease of reading.
66-
use_repo(python, PYTHON_NAME, "python3_toolchains")
68+
use_repo(python, PYTHON_NAME, PYTHON_TOOLCHAINS)
6769

6870
# Register an already-defined toolchain so that Bazel can use it during
6971
# toolchain resolution.
7072
register_toolchains(
71-
"@python3_toolchains//:all",
73+
"@{}//:all".format(PYTHON_TOOLCHAINS),
7274
)
7375

7476
# The interpreter extension discovers the platform specific Python binary.

‎python/extensions/python.bzl

Lines changed: 103 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,40 +17,126 @@
1717
load("@rules_python//python:repositories.bzl", "python_register_toolchains")
1818
load("@rules_python//python/extensions/private:interpreter_hub.bzl", "hub_repo")
1919

20+
def _python_register_toolchains(toolchain_attr, version_constraint):
21+
python_register_toolchains(
22+
name = toolchain_attr.name,
23+
python_version = toolchain_attr.python_version,
24+
register_coverage_tool = toolchain_attr.configure_coverage_tool,
25+
ignore_root_user_error = toolchain_attr.ignore_root_user_error,
26+
set_python_version_constraint = version_constraint,
27+
)
28+
2029
def _python_impl(module_ctx):
21-
toolchains = []
30+
# We collect all of the toolchain names to create
31+
# the INTERPRETER_LABELS map. This is used
32+
# by interpreter_extensions.bzl via the hub_repo call below.
33+
toolchain_names = []
34+
35+
# Used to store the default toolchain so we can create it last.
36+
default_toolchain = None
37+
38+
# Used to store toolchains that are in sub modules.
39+
sub_toolchains_map = {}
40+
2241
for mod in module_ctx.modules:
2342
for toolchain_attr in mod.tags.toolchain:
24-
python_register_toolchains(
25-
name = toolchain_attr.name,
26-
python_version = toolchain_attr.python_version,
27-
bzlmod = True,
28-
# Toolchain registration in bzlmod is done in MODULE file
29-
register_toolchains = False,
30-
register_coverage_tool = toolchain_attr.configure_coverage_tool,
31-
ignore_root_user_error = toolchain_attr.ignore_root_user_error,
32-
)
33-
34-
# We collect all of the toolchain names to create
35-
# the INTERPRETER_LABELS map. This is used
36-
# by interpreter_extensions.bzl
37-
toolchains.append(toolchain_attr.name)
43+
# If we are in the root module we always register the toolchain.
44+
# We wait to register the default toolchain till the end.
45+
if mod.is_root:
46+
toolchain_names.append(toolchain_attr.name)
47+
48+
# If we have the default version or we only have one toolchain
49+
# in the root module we set the toolchain as the default toolchain.
50+
if toolchain_attr.default_version or len(mod.tags.toolchain) == 1:
51+
# We have already found one default toolchain, and we can
52+
# only have one.
53+
if default_toolchain != None:
54+
fail("""We found more than one toolchain that is marked
55+
as the default version. Only set one toolchain with default_version set as
56+
True.""")
57+
default_toolchain = toolchain_attr
58+
continue
59+
60+
# Always register toolchains that are in the root module.
61+
_python_register_toolchains(toolchain_attr, True)
62+
else:
63+
# We add the toolchain to a map, and we later create the
64+
# toolchain if the root module does not have a toolchain with
65+
# the same name. We have to loop through all of the modules to
66+
# ensure that we get a full list of the root toolchains.
67+
sub_toolchains_map[toolchain_attr.name] = toolchain_attr
68+
69+
# We did not find a default toolchain so we fail.
70+
if default_toolchain == None:
71+
fail("""Unable to find a default toolchain in the root module.
72+
Please define a toolchain that has default_version set to True.""")
73+
74+
# Create the toolchains in the submodule(s).
75+
for name, toolchain_attr in sub_toolchains_map:
76+
# A sub module cannot have a toolchain that is marked as the
77+
# default version. TODO: should we create the toolchain anyways,
78+
# but set the default version to False?
79+
if toolchain_attr.default_version:
80+
fail("""Not able to create toolchain named: {}. This toolchain exists
81+
in a sub module and defalult_version is set to True.""".format(name))
82+
83+
# We cannot have a toolchain in a sub module that has the same name of
84+
# a toolchain in the root module. This will cause name clashing.
85+
if name in toolchain_names:
86+
print("""Not creating the toolchain from sub module, with the name {}. The root
87+
modhas a toolchain of the same name.""".format(toolchain_attr.name))
88+
continue
89+
toolchain_names.append(name)
90+
_python_register_toolchains(toolchain_attr, True)
3891

92+
# We register the default toolchain last.
93+
_python_register_toolchains(default_toolchain, False)
94+
95+
# Create the hub for the different interpreter versions.
3996
hub_repo(
4097
name = "pythons_hub",
41-
toolchains = toolchains,
98+
toolchains = toolchain_names,
4299
)
43100

44101
python = module_extension(
45-
doc = "Bzlmod extension that is used to register a Python toolchain.",
102+
doc = """Bzlmod extension that is used to register Python toolchains.
103+
""",
46104
implementation = _python_impl,
47105
tag_classes = {
48106
"toolchain": tag_class(
107+
doc = """Tag class used to register Python toolchains.
108+
Use this tag class to register one of more Python toolchains. This class
109+
is also potentially called by sub modules. The following covers different
110+
business rules and use cases.
111+
112+
Toolchains in the Root Module
113+
114+
This class registers all toolchains in the root module.
115+
116+
Toolchains in Sub Modules
117+
118+
It will create a toolchain that is in a sub module, if the toolchain
119+
of the same name does not exist in the root module. The extension stops name
120+
clashing between toolchains in the root module and toolchains in sub modules.
121+
You cannot configure more than one toolchain as the default toolchain.
122+
123+
Toolchain set as the default versions
124+
125+
This extension will not create a toolchain that exists in a sub module,
126+
if the sub module toolchain is marked as the default version. If you have
127+
more than one toolchain in your root module, you need to set one of the
128+
toolchains as the default version. If there is only one toolchain it
129+
is set as the default toolchain.
130+
""",
49131
attrs = {
50132
"configure_coverage_tool": attr.bool(
51133
mandatory = False,
52134
doc = "Whether or not to configure the default coverage tool for the toolchains.",
53135
),
136+
"default_version": attr.bool(
137+
mandatory = False,
138+
doc = "Whether the toolchain is the default version",
139+
),
54140
"ignore_root_user_error": attr.bool(
55141
default = False,
56142
doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.",

‎python/repositories.bzl

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -457,7 +457,6 @@ def python_register_toolchains(
457457
register_coverage_tool = False,
458458
set_python_version_constraint = False,
459459
tool_versions = TOOL_VERSIONS,
460-
bzlmod = False,
461460
**kwargs):
462461
"""Convenience macro for users which does typical setup.
463462
@@ -480,9 +479,15 @@ def python_register_toolchains(
480479
set_python_version_constraint: When set to true, target_compatible_with for the toolchains will include a version constraint.
481480
tool_versions: a dict containing a mapping of version with SHASUM and platform info. If not supplied, the defaults
482481
in python/versions.bzl will be used.
483-
bzlmod: Whether this rule is being run under a bzlmod module extension.
484482
**kwargs: passed to each python_repositories call.
485483
"""
484+
485+
# If we have @@ we have bzlmod
486+
bzlmod = str(Label("//:distribution")).startswith("@@")
487+
if bzlmod:
488+
# you cannot used native.register_toolchains when using bzlmod.
489+
register_toolchains = False
490+
486491
base_url = kwargs.pop("base_url", DEFAULT_RELEASE_BASE_URL)
487492

488493
if python_version in MINOR_MAPPING:

0 commit comments

Comments
 (0)
0