8000 feat(bzlmod): Allowing multiple python.toolchain extension calls · bazel-contrib/rules_python@19fe4b1 · GitHub
[go: up one dir, main page]
Skip to content

Commit 19fe4b1

Browse files
chrislovecnmchrislovecnm
committed
feat(bzlmod): Allowing multiple python.toolchain extension calls
We do this work for two reasons. First, we must support Module dependencies and sub-modules using python.toolchain. Second, we needed this commit to supporting using multiple toolchains with bzlmod. This commit modifies the python.toolchain extension to handle being called multiple times. We are modeling how the multiple Python toolchains work. This commit implements various business logic in the toolchain class as follows. Toolchains in Sub Modules It will create a toolchain in a sub-module if the toolchain of the same name does not exist in the root module. The extension stops name clashing between toolchains in the root module and sub-modules. You cannot configure more than one toolchain as the default toolchain. Toolchain set as the default version. This extension will not create a toolchain in a sub-module if the sub-module toolchain is marked as the default version. If you have more than one toolchain in your root module, you need to set one of the toolchains as the default version. If there is only one toolchain, it is set as the default toolchain.
1 parent 1383bd4 commit 19fe4b1

File tree

4 files changed

+138
-26
lines changed

4 files changed

+138
-26
lines changed

‎examples/bzlmod/MODULE.bazel

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,24 +11,37 @@ local_path_override(
1111
path = "../..",
1212
)
1313

14+
PYTHON_NAME = "python3_9"
15+
16+
PYTHON_TOOLCHAINS = PYTHON_NAME + "_toolchains"
17+
1418
python = use_extension("@rules_python//python/extensions:python.bzl", "python")
1519
python.toolchain(
16-
name = "python3_9",
20+
name = PYTHON_NAME,
1721
configure_coverage_tool = True,
1822
python_version = "3.9",
1923
)
20-
use_repo(python, "python3_9")
21-
use_repo(python, "python3_9_toolchains")
24+
use_repo(python, PYTHON_NAME)
25+
use_repo(python, PYTHON_TOOLCHAINS)
2226

2327
register_toolchains(
24-
"@python3_9_toolchains//:all",
28+
"@{}//:all".format(PYTHON_TOOLCHAINS),
29+
)
30+
31+
interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter")
32+
interpreter.install(
33+
name = "interpreter_python_3_9",
34+
python_name = PYTHON_NAME,
2535
)
36+
use_repo(interpreter, "interpreter_python_3_9")
2637

2738
pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
2839
pip.parse(
2940
name = "pip",
30-
# Intentionally set it false because the "true" case is already covered by examples/bzlmod_build_file_generation
41+
# Intentionally set it false because the "true" case is already
42+
# covered by examples/bzlmod_build_file_generation
3143
incompatible_generate_aliases = False,
44+
python_interpreter_target = "@interpreter_python_3_9//:python",
3245
requirements_lock = "//:requirements_lock.txt",
3346
requirements_windows = "//:requirements_windows.txt",
3447
)

‎examples/bzlmod_build_file_generation/MODULE.bazel

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ python = use_extension("@rules_python//python/extensions:python.bzl", "python")
4848
# We also use the same name for python.host_python_interpreter.
4949
PYTHON_NAME = "python3"
5050

51+
PYTHON_TOOLCHAINS = PYTHON_NAME + "_toolchains"
52+
5153
# We next initialize the python toolchain using the extension.
5254
# You can set different Python versions in this block.
5355
python.toolchain(
@@ -63,12 +65,12 @@ python.toolchain(
6365
# into the scope of the current module.
6466
# All of the python3 repositories use the PYTHON_NAME as there prefix. They
6567
# are not catenated for ease of reading.
66-
use_repo(python, PYTHON_NAME, "python3_toolchains")
68+
use_repo(python, PYTHON_NAME, PYTHON_TOOLCHAINS)
6769

6870
# Register an already-defined toolchain so that Bazel can use it during
6971
# toolchain resolution.
7072
register_toolchains(
71-
"@python3_toolchains//:all",
73+
"@{}//:all".format(PYTHON_TOOLCHAINS),
7274
)
7375

7476
# The interpreter extension discovers the platform specific Python binary.

‎python/extensions/python.bzl

Lines changed: 109 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,40 +17,132 @@
1717
load("@rules_python//python:repositories.bzl", "python_register_toolchains")
1818
load("@rules_python//python/extensions/private:interpreter_hub.bzl", "hub_repo")
1919

20+
# Printing a warning msg not debugging, so we have to disable
21+
# the buildifier check.
22+
# buildifier: disable=print
23+
def _print_warn(msg):
24+
print(msg)
25+
26+
def _python_register_toolchains(toolchain_attr, version_constraint):
27+
python_register_toolchains(
28+
name = toolchain_attr.name,
29+
python_version = toolchain_attr.python_version,
30+
register_coverage_tool = toolchain_attr.configure_coverage_tool,
31+
ignore_root_user_error = toolchain_attr.ignore_root_user_error,
32+
set_python_version_constraint = version_constraint,
33+
)
34+
2035
def _python_impl(module_ctx):
21-
toolchains = []
36+
# We collect all of the toolchain names to create
37+
# the INTERPRETER_LABELS map. This is used
38+
# by interpreter_extensions.bzl via the hub_repo call below.
39+
toolchain_names = []
40+
41+
# Used to store the default toolchain so we can create it last.
42+
default_toolchain = None
43+
44+
# Used to store toolchains that are in sub modules.
45+
sub_toolchains_map = {}
46+
2247
for mod in module_ctx.modules:
2348
for toolchain_attr in mod.tags.toolchain:
24-
python_register_toolchains(
25-
name = toolchain_attr.name,
26-
python_version = toolchain_attr.python_version,
27-
bzlmod = True,
28-
# Toolchain registration in bzlmod is done in MODULE file
29-
register_toolchains = False,
30-
register_coverage_tool = toolchain_attr.configure_coverage_tool,
31-
ignore_root_user_error = toolchain_attr.ignore_root_user_error,
32-
)
33-
34-
# We collect all of the toolchain names to create
35-
# the INTERPRETER_LABELS map. This is used
36-
# by interpreter_extensions.bzl
37-
toolchains.append(toolchain_attr.name)
49+
# If we are in the root module we always register the toolchain.
50+
# We wait to register the default toolchain till the end.
51+
if mod.is_root:
52+
toolchain_names.append(toolchain_attr.name)
53+
54+
# If we have the default version or we only have one toolchain
55+
# in the root module we set the toolchain as the default toolchain.
56+
if toolchain_attr.default_version or len(mod.tags.toolchain) == 1:
57+
# We have already found one default toolchain, and we can
58+
# only have one.
59+
if default_toolchain != None:
60+
fail("""We found more than one toolchain that is marked
61+
as the default version. Only set one toolchain with default_version set as
62+
True.""")
63+
default_toolchain = toolchain_attr
64+
continue
65+
66+
# Always register toolchains that are in the root module.
67+
_python_register_toolchains(toolchain_attr, True)
68+
else:
69+
# We add the toolchain to a map, and we later create the
70+
# toolchain if the root module does not have a toolchain with
71+
# the same name. We have to loop through all of the modules to
72+
# ensure that we get a full list of the root toolchains.
73+
sub_toolchains_map[toolchain_attr.name] = toolchain_attr
74+
75+
# We did not find a default toolchain so we fail.
76+
if default_toolchain == None:
77+
fail("""Unable to find a default toolchain in the root module.
78+
Please define a toolchain that has default_version set to True.""")
79+
80+
# Create the toolchains in the submodule(s).
81+
for name, toolchain_attr in sub_toolchains_map:
82+
# A sub module cannot have a toolchain that is marked as the
83+
# default version. TODO: should we create the toolchain anyways,
84+
# but set the default version to False?
85+
if toolchain_attr.default_version:
86+
fail("""Not able to create toolchain named: {}. This toolchain exists
87+
in a sub module and defalult_version is set to True.""".format(name))
3888

89+
# We cannot have a toolchain in a sub module that has the same name of
90+
# a toolchain in the root module. This will cause name clashing.
91+
if name in toolchain_names:
92+
_print_warn("""Not creating the toolchain from sub module, with the name {}. The root
93+
modhas a toolchain of the same name.""".format(toolchain_attr.name))
94+
continue
95+
toolchain_names.append(name)
96+
_python_register_toolchains(toolchain_attr, True)
97+
98+
# We register the default toolchain last.
99+
_python_register_toolchains(default_toolchain, False)
100+
101+
# Create the hub for the different interpreter versions.
39102
hub_repo(
40103
name = "pythons_hub",
41-
toolchains = toolchains,
104+
toolchains = toolchain_names,
42105
)
43106

44107
python = module_extension(
45-
doc = "Bzlmod extension that is used to register a Python toolchain.",
108+
doc = """Bzlmod extension that is used to register Python toolchains.
109+
""",
46110
implementation = _python_impl,
47111
tag_classes = {
48112
"toolchain": tag_class(
113+
doc = """Tag class used to register Python toolchains.
114+
Use this tag class to register one of more Python toolchains. This class
115+
is also potentially called by sub modules. The following covers different
116+
business rules and use cases.
117+
118+
Toolchains in the Root Module
119+
120+
This class registers all toolchains in the root module.
121+
122+
Toolchains in Sub Modules
123+
124+
It will create a toolchain that is in a sub module, if the toolchain
125+
of the same name does not exist in the root module. The extension stops name
126+
clashing between toolchains in the root module and toolchains in sub modules.
127+
You cannot configure more than one toolchain as the default toolchain.
128+
129+
Toolchain set as the default versions
130+
131+
This extension will not create a toolchain that exists in a sub module,
132+
if the sub module toolchain is marked as the default version. If you have
133+
more than one toolchain in your root module, you need to set one of the
134+
toolchains as the default version. If there is only one toolchain it
135+
is set as the default toolchain.
136+
""",
49137
attrs = {
50138
"configure_coverage_tool": attr.bool(
51139
mandatory = False,
52140
doc = "Whether or not to configure the default coverage tool for the toolchains.",
53141
),
142+
"default_version": attr.bool(
143+
mandatory = False,
144+
doc = "Whether the toolchain is the default version",
145+
),
54146
"ignore_root_user_error": attr.bool(
55147
default = False,
56148
doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.",

‎python/repositories.bzl

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -463,7 +463,6 @@ def python_register_toolchains(
463463
register_coverage_tool = False,
464464
set_python_version_constraint = False,
465465
tool_versions = TOOL_VERSIONS,
466-
bzlmod = False,
467466
**kwargs):
468467
"""Convenience macro for users which does typical setup.
469468
@@ -486,9 +485,15 @@ def python_register_toolchains(
486485
set_python_version_constraint: When set to true, target_compatible_with for the toolchains will include a version constraint.
487486
tool_versions: a dict containing a mapping of version with SHASUM and platform info. If not supplied, the defaults
488487
in python/versions.bzl will be used.
489-
bzlmod: Whether this rule is being run under a bzlmod module extension.
490488
**kwargs: passed to each python_repositories call.
491489
"""
490+
491+
# If we have @@ we have bzlmod
492+
bzlmod = str(Label("//:distribution")).startswith("@@")
493+
if bzlmod:
494+
# you cannot used native.register_toolchains when using bzlmod.
495+
register_toolchains = False
496+
492497
base_url = kwargs.pop("base_url", DEFAULT_RELEASE_BASE_URL)
493498

494499
if python_version in MINOR_MAPPING:

0 commit comments

Comments
 (0)
0