8000 feat: add alias generation for pip parse · bazel-contrib/rules_python@d4ecd4c · GitHub
[go: up one dir, main page]
Skip to content

Commit d4ecd4c

Browse files
corypaikcorypaik
committed
feat: add alias generation for pip parse
Generate alias packages and targets for `pip_parse` under the parent repository. This serves two purposes. It allows for users to reference packages using a simpler `@pypi//package_name` without the import issues associated with putting the contents of the package there (as may be the case with `pip_install` following this naming pattern). Additionally, this allows users to override individual libraries either by default or based on select statements.
1 parent 0054574 commit d4ecd4c

File tree

12 files changed

+221
-12
lines changed

12 files changed

+221
-12
lines changed

‎docs/pip.md

Lines changed: 22 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎docs/pip_repository.md

Lines changed: 27 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎examples/pip_parse/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ py_test(
8181
":yamllint",
8282
data_requirement("s3cmd"),
8383
dist_info_requirement("requests"),
84+
"@pypi//yamllint",
8485
],
8586
env = {
8687
"WHEEL_DATA_CONTENTS": "$(rootpaths {})".format(data_requirement("s3cmd")),

‎examples/pip_parse/WORKSPACE

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ python_register_toolchains(
1313
)
1414

1515
load("@python39//:defs.bzl", "interpreter")
16-
load("@rules_python//python:pip.bzl", "pip_parse")
16+
load("@rules_python//python:pip.bzl", "package_overrides", "pip_parse")
1717

1818
pip_parse(
1919
# (Optional) You can set an environment in the pip process to control its
@@ -22,6 +22,20 @@ pip_parse(
2222
# can be passed
2323
# environment = {"HTTPS_PROXY": "http://my.proxy.fun/"},
2424
name = "pypi",
25+
26+
# Pip parse uses alias packages to create targets configurable aliases such as @pypi//package.
27+
# You can then override these packages, either by default or in certain cases (under the hood, this uses select).
28+
# Note that this will override it for everyone who uses @pypi//mypackage, including other pip packages. This
29+
# will not affect callers who use @pypi_mypackage//:pkg directly.
30+
library_overrides = package_overrides({
31+
# Shorthand to override the default.
32+
# "pip_package_1" : "//:my_pip_package_1",
33+
# # Any valid select would work two (default not required).
34+
# "pip_package_2" : {
35+
# "//:mycondition": "//:my_pip_package_2",
36+
# }
37+
}),
38+
2539
# (Optional) You can provide extra parameters to pip.
2640
# Here, make pip output verbose (this is usable with `quiet = False`).
2741
# extra_pip_args = ["-v"],

‎examples/pip_parse_vendored/requirements.bzl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ from //:requirements.txt
77
load("@python39//:defs.bzl", "interpreter")
88
load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library")
99

10-
all_requirements = ["@pip_certifi//:pkg", "@pip_charset_normalizer//:pkg", "@pip_idna//:pkg", "@pip_requests//:pkg", "@pip_urllib3//:pkg"]
10+
all_requirements = ["@pip//certifi", "@pip//charset_normalizer", "@pip//idna", "@pip//requests", "@pip//urllib3"]
1111

1212
all_whl_requirements = ["@pip_certifi//:whl", "@pip_charset_normalizer//:whl", "@pip_idna//:whl", "@pip_requests//:whl", "@pip_urllib3//:whl"]
1313

‎python/pip.bzl

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,13 @@
1313
# limitations under the License.
1414
"""Import pip requirements into Bazel."""
1515

16-
load("//python/pip_install:pip_repository.bzl", "pip_repository", _package_annotation = "package_annotation")
16+
load("//python/pip_install:pip_repository.bzl", "pip_repository", _package_annotation = "package_annotation", _package_overrides = "package_overrides")
1717
load("//python/pip_install:repositories.bzl", "pip_install_dependencies")
1818
load("//python/pip_install:requirements.bzl", _compile_pip_requirements = "compile_pip_requirements")
1919

2020
compile_pip_requirements = _compile_pip_requirements
2121
package_annotation = _package_annotation
22+
package_overrides = _package_overrides
2223

2324
def pip_install(requirements = None, name = "pip", **kwargs):
2425
"""Accepts a locked/compiled requirements file and installs the dependencies listed within.

‎python/pip_install/extract_wheels/bazel.py

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,10 @@ def sanitised_repo_file_label(whl_name: str, repo_prefix: str) -> str:
326326
)
327327

328328

329+
def sanitised_alias_repo_library_label(repo: str, name: str) -> str:
330+
return '"@{}//{}"'.format(sanitise_name(repo, ""), sanitise_name(name, ""))
331+
332+
329333
def extract_wheel(
330334
wheel_file: str,
331335
extras: Dict[str, Set[str]],
@@ -335,6 +339,7 @@ def extract_wheel(
335339
incremental: bool = False,
336340
incremental_dir: Path = Path("."),
337341
annotation: Optional[annotation.Annotation] = None,
342+
parent_repo_name: Optional[str] = None,
338343
) -> Optional[str]:
339344
"""Extracts wheel into given directory and creates py_library and filegroup targets.
340345
@@ -347,6 +352,7 @@ def extract_wheel(
347352
effects the names of libraries and their dependencies, which point to other external repositories.
348353
incremental_dir: An optional override for the working directory of incremental builds.
349354
annotation: An optional set of annotations to apply to the BUILD contents of the wheel.
355+
parent_repo_name: The parent repo name, required when `incremental=True` and ignored otherwise.
350356
351357
Returns:
352358
The Bazel label for the extracted wheel, in the form '//path/to/wheel'.
@@ -373,8 +379,12 @@ def extract_wheel(
373379
whl_deps = sorted(whl.dependencies(extras_requested) - self_edge_dep)
374380

375381
if incremental:
382+
assert (
383+
parent_repo_name is not None
384+
), '"parent_repo_name" is required when incremental=True.'
376385
sanitised_dependencies = [
377-
sanitised_repo_library_label(d, repo_prefix=repo_prefix) for d in whl_deps
386+
sanitised_alias_repo_library_label(repo=parent_repo_name, name=d)
387+
for d in whl_deps
378388
]
379389
sanitised_wheel_file_dependencies = [
380390
sanitised_repo_file_label(d, repo_prefix=repo_prefix) for d in whl_deps

‎python/pip_install/extract_wheels/extract_single_wheel.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ def main() -> None:
9898
incremental=True,
9999
repo_prefix=args.repo_prefix,
100100
annotation=args.annotation,
101+
parent_repo_name=args.repo,
101102
)
102103

103104

‎python/pip_install/extract_wheels/parse_requirements_to_bzl.py

Lines changed: 100 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
import argparse
22
import json
3+
import os
34
import shlex
45
import sys
56
import textwrap
7+
import warnings
68
from pathlib import Path
79
from typing import Any, Dict, List, TextIO, Tuple
810

@@ -86,17 +88,93 @@ def parse_whl_library_args(args: argparse.Namespace) -> Dict[str, Any]:
8688
"requirements_lock_label",
8789
"annotations",
8890
"bzlmod",
91+
"library_overrides",
8992
):
9093
if arg in whl_library_args:
9194
whl_library_args.pop(arg)
9295

9396
return whl_library_args
9497

9598

99+
def _create_alias_package(
100+
alias_name: str,
101+
py_library_selection: Dict[str, str],
102+
):
103+
# The directory name is just the sanitized name.
104+
os.mkdir(alias_name)
105+
106+
py_library_selection = json.dumps(py_library_selection)
107+
108+
build_file_content = textwrap.dedent(
109+
"""\
110+
111+
package(default_visibility = ["//visibility:public"])
112+
113+
alias(
114+
name = "{py_library_label}",
115+
actual = select({py_library_selection}),
116+
)
117+
""".format(
118+
py_library_label=alias_name,
119+
py_library_selection=py_library_selection,
120+
)
121+
)
122+
123+
with open(os.path.join(alias_name, "BUILD.bazel"), "w", encoding="utf-8") as f:
124+
f.write(build_file_content)
125+
126+
127+
def create_alias_packages(
128+
repo_prefix: str,
129+
install_requirements: List[Tuple[InstallRequirement, str]],
130+
library_overrides: Dict[str, Dict[str, str]],
131+
):
132+
"""Create alias packages and targets for each requirement."""
133+
for ir, _ in install_requirements:
134+
# The 'actual' library is the one in the incrementally fetched repo.
135+
# We need to strip the quotes here as we will encode w/ json.
136+
actual_library = bazel.sanitised_repo_library_label(
137+
ir.name, repo_prefix=repo_prefix
138+
)
139+
actual_library = actual_library.replace('"', "")
140+
141+
alias_name = bazel.sanitise_name(ir.name, "")
142+
143+
# Apply any overrides on top. We pop the keys here so we can report
144+
# unused libraries to the user. Currently this only accepts overrides
145+
# which use the alias (sanitized) name.
146+
py_library_selection = {
147+
"//conditions:default": actual_library,
148+
**library_overrides.pop(alias_name, {}),
149+
}
150+
151+
_create_alias_package(
152+
alias_name=alias_name, py_library_selection=py_library_selection
153+
)
154+
155+
# By default, warn about overrides which aren't present in the requirements
156+
# file, but create repos for these overrides anyway. This is useful in cases
157+
# where the actual combination of libraries is unsupported but it works with
158+
# the user's custom override(s).
159+
# TODO(corypaik): Parameterize this behavior so that the users can whitelist
160+
# specific libraries or throw an error instead.
161+
if len(library_overrides) > 0:
162+
warnings.warn(
163+
"Ignoring library overrides for packages not present in the "
164+
f"requirements file: {library_overrides}"
165+
)
166+
167+
for alias_name, py_library_selection in library_overrides.items():
168+
_create_alias_package(
169+
alias_name=alias_name, py_library_selection=py_library_selection
170+
)
171+
172+
96173
def generate_parsed_requirements_contents(
97174
requirements_lock: Path,
98175
repo: str,
99176
repo_prefix: str,
177+
parent_repo_name: str,
100178
whl_library_args: Dict[str, Any],
101179
annotations: Dict[str, str] = dict(),
102180
bzlmod: bool = False,
@@ -114,10 +192,12 @@ def generate_parsed_requirements_contents(
114192
repo_names_and_reqs = repo_names_and_requirements(
115193
install_req_and_lines, repo_prefix
116194
)
117-
195+
# Use the alias targets for `all_requirements`.
118196
all_requirements = ", ".join(
119197
[
120-
bazel.sanitised_repo_library_label(ir.name, repo_prefix=repo_prefix)
198+
bazel.sanitised_alias_repo_library_label(
199+
repo=parent_repo_name, name=ir.name
200+
)
121201
for ir, _ in install_req_and_lines
122202
]
123203
)
@@ -212,6 +292,11 @@ def coerce_to_bool(option):
212292
return str(option).lower() == "true"
213293

214294

295+
def parse_json_from_file(option):
296+
content = Path(option).read_text()
297+
return json.loads(content) if content.strip() != "" else {}
298+
299+
215300
def main(output: TextIO) -> None:
216301
"""Args:
217302
@@ -266,6 +351,11 @@ def main(output: TextIO) -> None:
266351
default=False,
267352
help="Whether this script is run under bzlmod. Under bzlmod we don't generate the install_deps() macro as it isn't needed.",
268353
)
354+
parser.add_argument(
355+
"--library_overrides",
356+
type=parse_json_from_file,
357+
help="A json encoded file containing library overrides for packages.",
358+
)
269359
arguments.parse_common_args(parser)
270360
args = parser.parse_args()
271361

@@ -278,6 +368,13 @@ def main(output: TextIO) -> None:
278368
req_names = sorted([req.name for req, _ in install_requirements])
279369
annotations = args.annotations.collect(req_names) if args.annotations else {}
280370

371+
# Generate build files for each library.
372+
create_alias_packages(
373+
repo_prefix=args.repo_prefix,
374+
install_requirements=install_requirements,
375+
library_overrides=args.library_overrides,
376+
)
377+
281378
# Write all rendered annotation files and generate a list of the labels to write to the requirements file
282379
annotated_requirements = dict()
283380
for name, content in annotations.items():
@@ -310,6 +407,7 @@ def main(output: TextIO) -> None:
310407
requirements_lock=args.requirements_lock,
311408
repo=args.repo,
312409
repo_prefix=args.repo_prefix,
410+
parent_repo_name=args.repo,
313411
whl_library_args=whl_library_args,
314412
annotations=annotated_requirements,
315413
bzlmod=args.bzlmod,

‎python/pip_install/extract_wheels/parse_requirements_to_bzl_test.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ def test_generated_requirements_bzl(self) -> None:
2727
)
2828
args = argparse.Namespace()
2929
args.requirements_lock = str(requirements_lock.resolve())
30-
args.repo = ("pip_parsed_deps_pypi__",)
30+
args.repo = "pip_parsed_deps_pypi"
3131
args.repo_prefix = "pip_parsed_deps_pypi__"
3232
extra_pip_args = ["--index-url=pypi.org/simple"]
3333
pip_data_exclude = ["**.foo"]
@@ -42,8 +42,9 @@ def test_generated_requirements_bzl(self) -> None:
4242
repo=args.repo,
4343
repo_prefix=args.repo_prefix,
4444
whl_library_args=whl_library_args,
45+
parent_repo_name=args.repo,
4546
)
46-
library_target = "@pip_parsed_deps_pypi__foo//:pkg"
47+
library_target = "@pip_parsed_deps_pypi//foo"
4748
whl_target = "@pip_parsed_deps_pypi__foo//:whl"
4849
all_requirements = 'all_requirements = ["{library_target}"]'.format(
4950
library_target=library_target

0 commit comments

Comments
 (0)
0