8000 feat(bzlmod): Cleaning up interpreter resolution (#1218) · Rasrack/rules_python@ccea92a · GitHub
[go: up one dir, main page]
Skip to content

Commit ccea92a

Browse files
chrislovecnmchrislovecnm
authored
feat(bzlmod): Cleaning up interpreter resolution (bazel-contrib#1218)
This commit cleans up the use of "canonical resolution" of the Python interpreter. When the extension toolchains run it collects a list of the interpreters and then uses the hub_repo rule to create a map of names and the interpreter labels. Next, we then use the interpreter_extension that, creates reports that have symlinks pointing to the different interpreter binaries. The user can then pass in a label to the pip call for the specific hermetic interpreter.
1 parent d434f10 commit ccea92a

File tree

5 files changed

+168
-106
lines changed

5 files changed

+168
-106
lines changed

‎MODULE.bazel

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,6 @@ use_repo(
4646
"pypi__coverage_cp39_x86_64-apple-darwin",
4747
"pypi__coverage_cp39_x86_64-unknown-linux-gnu",
4848
)
49+
50+
python = use_extension("@rules_python//python:extensions.bzl", "python")
51+
use_repo(python, "pythons_hub")

‎examples/bzlmod_build_file_generation/MODULE.bazel

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,6 @@ python = use_extension("@rules_python//python:extensions.bzl", "python")
4848
# We also use the same name for python.host_python_interpreter.
4949
PYTHON_NAME = "python3"
5050

51-
# This is the name that is used for the host interpreter
52-
PYTHON_INTERPRETER = PYTHON_NAME + "_host_interpreter"
53-
5451
# We next initialize the python toolchain using the extension.
5552
# You can set different Python versions in this block.
5653
python.toolchain(
@@ -66,37 +63,46 @@ python.toolchain(
6663
# into the scope of the current module.
6764
# All of the python3 repositories use the PYTHON_NAME as there prefix. They
6865
# are not catenated for ease of reading.
69-
use_repo(python, PYTHON_NAME)
70-
use_repo(python, "python3_toolchains")
71-
use_repo(python, PYTHON_INTERPRETER)
66+
use_repo(python, PYTHON_NAME, "python3_toolchains")
7267

73-
# Register an already-defined toolchain so that Bazel can use it during toolchain resolution.
68+
# Register an already-defined toolchain so that Bazel can use it during
69+
# toolchain resolution.
7470
register_toolchains(
7571
"@python3_toolchains//:all",
7672
)
7773

78-
# Use the pip extension
79-
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
74+
# The interpreter extension discovers the platform specific Python binary.
75+
# It creates a symlink to the binary, and we pass the label to the following
76+
# pip.parse call.
77+
interpreter = use_extension("@rules_python//python:interpreter_extension.bzl", "interpreter")
78+
interpreter.install(
79+
name = "interpreter_python3",
80+
python_name = PYTHON_NAME,
81+
)
82+
use_repo(interpreter, "interpreter_python3")
8083

81-
# Use the extension to call the `pip_repository` rule that invokes `pip`, with `incremental` set.
82-
# Accepts a locked/compiled requirements file and installs the dependencies listed within.
84+
# Use the extension, pip.parse, to call the `pip_repository` rule that invokes
85+
# `pip`, with `incremental` set. The pip call accepts a locked/compiled
86+
# requirements file and installs the dependencies listed within.
8387
# Those dependencies become available in a generated `requirements.bzl` file.
8488
# You can instead check this `requirements.bzl` file into your repo.
8589
# Because this project has different requirements for windows vs other
8690
# operating systems, we have requirements for each.
91+
pip = use_extension("@rules_python//python:extensions.bzl", "pip")
8792
pip.parse(
8893
name = "pip",
8994
# When using gazelle you must use set the following flag
9095
# in order for the generation of gazelle dependency resolution.
9196
incompatible_generate_aliases = True,
92-
# The interpreter attribute points to the interpreter to use for running
93-
# pip commands to download the packages in the requirements file.
97+
# The interpreter_target attribute points to the interpreter to
98+
# use for running pip commands to download the packages in the
99+
# requirements file.
94100
# As a best practice, we use the same interpreter as the toolchain
95101
# that was configured above; this ensures the same Python version
96102
# is used for both resolving dependencies and running tests/binaries.
97103
# If this isn't specified, then you'll get whatever is locally installed
98104
# on your system.
99-
python_interpreter_target = "@" + PYTHON_INTERPRETER + "//:python",
105+
python_interpreter_target = "@interpreter_python3//:python",
100106
requirements_lock = "//:requirements_lock.txt",
101107
requirements_windows = "//:requirements_windows.txt",
102108
)

‎python/extensions.bzl

Lines changed: 12 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,10 @@ load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirement
1919
load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
2020
load("@rules_python//python/pip_install:requirements_parser.bzl", parse_requirements = "parse")
2121
load("@rules_python//python/private:coverage_deps.bzl", "install_coverage_deps")
22-
load("@rules_python//python/private:toolchains_repo.bzl", "get_host_os_arch", "get_host_platform")
22+
load("@rules_python//python/private:interpreter_hub.bzl", "hub_repo")
2323

2424
def _python_impl(module_ctx):
25+
toolchains = []
2526
for mod in module_ctx.modules:
2627
for toolchain_attr in mod.tags.toolchain:
2728
python_register_toolchains(
@@ -33,11 +34,16 @@ def _python_impl(module_ctx):
3334
register_coverage_tool = toolchain_attr.configure_coverage_tool,
3435
ignore_root_user_error = toolchain_attr.ignore_root_user_error,
3536
)
36-
host_hub_name = toolchain_attr.name + "_host_interpreter"
37-
_host_hub(
38-
name = host_hub_name,
39-
user_repo_prefix = toolchain_attr.name,
40-
)
37+
38+
# We collect all of the toolchain names to create
39+
# the INTERPRETER_LABELS map. This is used
40+
# by interpreter_extensions.bzl
41+
toolchains.append(toolchain_attr.name)
42+
43+
hub_repo(
44+
name = "pythons_hub",
45+
toolchains = toolchains,
46+
)
4147

4248
python = module_extension(
4349
implementation = _python_impl,
@@ -133,89 +139,3 @@ pip = module_extension(
133139
"parse": tag_class(attrs = _pip_parse_ext_attrs()),
134140
},
135141
)
136-
137-
# This function allows us to build the label name of a label
138-
# that is not passed into the current context.
139-
# The module_label is the key element that is passed in.
140-
# This value provides the root location of the labels
141-
# See https://bazel.build/external/extension#repository_names_and_visibility
142-
def _repo_mapped_label(module_label, extension_name, apparent):
143-
"""Construct a canonical repo label accounting for repo mapping.
144-
145-
Args:
146-
module_label: Label object of the module hosting the extension; see
147-
"_module" implicit attribute.
148-
extension_name: str, name of the extension that created the repo in `apparent`.
149-
apparent: str, a repo-qualified target string, but without the "@". e.g.
150-
"python38_x86_linux//:python". The repo name should use the apparent
151-
name used by the extension named by `ext_name` (i.e. the value of the
152-
`name` arg the extension passes to repository rules)
153-
"""
154-
return Label("@@{module}~{extension_name}~{apparent}".format(
155-
module = module_label.workspace_name,
156-
extension_name = extension_name,
157-
apparent = apparent,
158-
))
159-
160-
# We are doing some bazel stuff here that could use an explanation.
161-
# The basis of this function is that we need to create a symlink to
162-
# the python binary that exists in a different repo that we know is
163-
# setup by rules_python.
164-
#
165-
# We are building a Label like
166-
# @@rules_python~override~python~python3_x86_64-unknown-linux-gnu//:python
167-
# and then the function creates a symlink named python to that Label.
168-
# The tricky part is the "~override~" part can't be known in advance
169-
# and will change depending on how and what version of rules_python
170-
# is used. To figure that part out, an implicit attribute is used to
171-
# resolve the module's current name (see "_module" attribute)
172-
#
173-
# We are building the Label name dynamically, and can do this even
174-
# though the Label is not passed into this function. If we choose
175-
# not do this a user would have to write another 16 lines
176-
# of configuration code, but we are able to save them that work
177-
# because we know how rules_python works internally. We are using
178-
# functions from private:toolchains_repo.bzl which is where the repo
179-
# is being built. The repo name differs between host OS and platforms
180-
# and the functions from toolchains_repo gives us this functions that
181-
# information.
182-
def _host_hub_impl(repo_ctx):
183-
# Intentionally empty; this is only intended to be used by repository
184-
# rules, which don't process build file contents.
185-
repo_ctx.file("BUILD.bazel", "")
186-
187-
# The two get_ functions we use are also utilized when building
188-
# the repositories for the different interpreters.
189-
(os, arch) = get_host_os_arch(repo_ctx)
190-
host_platform = "{}_{}//:python".format(
191-
repo_ctx.attr.user_repo_prefix,
192-
get_host_platform(os, arch),
193-
)
194-
195-
# the attribute is set to attr.label(default = "//:_"), which
196-
# provides us the resolved, canonical, prefix for the module's repos.
197-
# The extension_name "python" is determined by the
198-
# name bound to the module_extension() call.
199-
# We then have the OS and platform specific name of the python
200-
# interpreter.
201-
label = _repo_mapped_label(repo_ctx.attr._module, "python", host_platform)
202-
203-
# create the symlink in order to set the interpreter for pip.
204-
repo_ctx.symlink(label, "python")
205-
206-
# We use this rule to set the pip interpreter target when using different operating
207-
# systems with the same project
208-
_host_hub = repository_rule(
209-
implementation = _host_hub_impl,
210-
local = True,
211-
attrs = {
212-
"user_repo_prefix": attr.string(
213-
mandatory = True,
214-
doc = """\
215-
The prefix to create the repository name. Usually the name you used when you created the
216-
Python toolchain.
217-
""",
218-
),
219-
"_module": attr.label(default = "//:_"),
220-
},
221-
)

‎python/interpreter_extension.bzl

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
# Copyright 2023 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"Module extension that finds the current toolchain Python binary and creates a symlink to it."
16+
17+
load("@pythons_hub//:interpreters.bzl", "INTERPRETER_LABELS")
18+
19+
def _interpreter_impl(mctx):
20+
for mod in mctx.modules:
21+
for install_attr in mod.tags.install:
22+
_interpreter_repo(
23+
name = install_attr.name,
24+
python_name = install_attr.python_name,
25+
)
26+
27+
interpreter = module_extension(
28+
doc = """\
29+
This extension is used to expose the underlying platform-specific
30+
interpreter registered as a toolchain. It is used by users to get
31+
a label to the interpreter for use with pip.parse
32+
in the MODULES.bazel file.
33+
""",
34+
implementation = _interpreter_impl,
35+
tag_classes = {
36+
"install": tag_class(
37+
attrs = {
38+
"name": attr.string(
39+
doc = "Name of the interpreter, we use this name to set the interpreter for pip.parse",
40+
mandatory = True,
41+
),
42+
"python_name": attr.string(
43+
doc = "The name set in the previous python.toolchain call.",
44+
mandatory = True,
45+
),
46+
},
47+
),
48+
},
49+
)
50+
51+
def _interpreter_repo_impl(rctx):
52+
rctx.file("BUILD.bazel", "")
53+
54+
actual_interpreter_label = INTERPRETER_LABELS.get(rctx.attr.python_name)
55+
if actual_interpreter_label == None:
56+
fail("Unable to find interpreter with name {}".format(rctx.attr.python_name))
57+
58+
rctx.symlink(actual_interpreter_label, "python")
59+
60+
_interpreter_repo = repository_rule(
61+
doc = """\
62+
Load the INTERPRETER_LABELS map. This map contain of all of the Python binaries
63+
by name and a label the points to the interpreter binary. The
64+
binaries are downloaded as part of the python toolchain setup.
65+
The rule finds the label and creates a symlink named "python" to that
66+
label. This symlink is then used by pip.
67+
""",
68+
implementation = _interpreter_repo_impl,
69+
attrs = {
70+
"python_name": attr.string(
71+
mandatory = True,
72+
doc = "Name of the Python toolchain",
73+
),
74+
},
75+
)

‎python/private/interpreter_hub.bzl

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Copyright 2023 The Bazel Authors. All rights reserved
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"Repo rule used by bzlmod extension to create a repo that has a map of Python interpreters and their labels"
16+
17+
load("//python:versions.bzl", "WINDOWS_NAME")
18+
load("//python/private:toolchains_repo.bzl", "get_host_os_arch", "get_host_platform")
19+
20+
_build_file_for_hub_template = """
21+
INTERPRETER_LABELS = {{
22+
{lines}
23+
}}
24+
"""
25+
26+
_line_for_hub_template = """\
27+
"{name}": Label("@{name}_{platform}//:{path}"),
28+
"""
29+
30+
def _hub_repo_impl(rctx):
31+
(os, arch) = get_host_os_arch(rctx)
32+
platform = get_host_platform(os, arch)
33+
34+
rctx.file("BUILD.bazel", "")
35+
is_windows = (os == WINDOWS_NAME)
36+
path = "python.exe" if is_windows else "bin/python3"
37+
38+
lines = "\n".join([_line_for_hub_template.format(
< A584 /code>
39+
name = name,
40+
platform = platform,
41+
path = path,
42+
) for name in rctx.attr.toolchains])
43+
44+
rctx.file("interpreters.bzl", _build_file_for_hub_template.format(lines = lines))
45+
46+
hub_repo = repository_rule(
47+
doc = """\
48+
This private rule create a repo with a BUILD file that contains a map of interpreter names
49+
and the labels to said interpreters. This map is used to by the interpreter hub extension.
50+
""",
51+
implementation = _hub_repo_impl,
52+
attrs = {
53+
"toolchains": attr.string_list(
54+
doc = "List of the base names the toolchain repo defines.",
55+
mandatory = True,
56+
),
57+
},
58+
)

0 commit comments

Comments
 (0)
0