8000 feat: add alias generation for pip parse · bazel-contrib/rules_python@db232db · GitHub
[go: up one dir, main page]
Skip to content

Commit db232db

Browse files
corypaikcorypaik
committed
feat: add alias generation for pip parse
Generate alias packages and targets for `pip_parse` under the parent repository. This serves two purposes. It allows for users to reference packages using a simpler `@pypi//package_name` without the import issues associated with putting the contents of the package there (as may be the case with `pip_install` following this naming pattern). Additionally, this allows users to override individual libraries either by default or based on select statements.
1 parent 6d9080f commit db232db

File tree

12 files changed

+193
-9
lines changed

12 files changed

+193
-9
lines changed

‎docs/pip.md

Lines changed: 18 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎docs/pip_repository.md

Lines changed: 20 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎examples/pip_parse/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ py_test(
8181
":yamllint",
8282
data_requirement("s3cmd"),
8383
dist_info_requirement("requests"),
84+
"@pypi//yamllint",
8485
],
8586
env = {
8687
"WHEEL_DATA_CONTENTS": "$(rootpaths {})".format(data_requirement("s3cmd")),

‎examples/pip_parse/WORKSPACE

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ python_register_toolchains(
1313
)
1414

1515
load("@python39//:defs.bzl", "interpreter")
16-
load("@rules_python//python:pip.bzl", "pip_parse")
16+
load("@rules_python//python:pip.bzl", "package_overrides", "pip_parse")
1717

1818
pip_parse(
1919
# (Optional) You can set an environment in the pip process to control its
@@ -22,6 +22,20 @@ pip_parse(
2222
# can be passed
2323
# environment = {"HTTPS_PROXY": "http://my.proxy.fun/"},
2424
name = "pypi",
25+
26+
# Pip parse uses alias packages to create targets configurable aliases such as @pypi//package.
27+
# You can then override these packages, either by default or in certain cases (under the hood, this uses select).
28+
# Note that this will override it for everyone who uses @pypi//mypackage, including other pip packages. This
29+
# will not affect callers who use @pypi_mypackage//:pkg directly.
30+
library_overrides = package_overrides({
31+
# Shorthand to override the default.
32+
# "pip_package_1" : "//:my_pip_package_1",
33+
# # Any valid select would work two (default not required).
34+
# "pip_package_2" : {
35+
# "//:mycondition": "//:my_pip_package_2",
36+
# }
37+
}),
38+
2539
# (Optional) You can provide extra parameters to pip.
2640
# Here, make pip output verbose (this is usable with `quiet = False`).
2741
# extra_pip_args = ["-v"],

‎examples/pip_parse_vendored/requirements.bzl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ from //:requirements.txt
77
load("@python39//:defs.bzl", "interpreter")
88
load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library")
99

10-
all_requirements = ["@pip_certifi//:pkg", "@pip_charset_normalizer//:pkg", "@pip_idna//:pkg", "@pip_requests//:pkg", "@pip_urllib3//:pkg"]
10+
all_requirements = ["@pip//certifi", "@pip//charset_normalizer", "@pip//idna", "@pip//requests", "@pip//urllib3"]
1111

1212
all_whl_requirements = ["@pip_certifi//:whl", "@pip_charset_normalizer//:whl", "@pip_idna//:whl", "@pip_requests//:whl", "@pip_urllib3//:whl"]
1313

‎python/pip.bzl

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,13 @@
1313
# limitations under the License.
1414
"""Import pip requirements into Bazel."""
1515

16-
load("//python/pip_install:pip_repository.bzl", "pip_repository", _package_annotation = "package_annotation")
16+
load("//python/pip_install:pip_repository.bzl", "pip_repository", _package_annotation = "package_annotation", _package_overrides = "package_overrides")
1717
load("//python/pip_install:repositories.bzl", "pip_install_dependencies")
1818
load("//python/pip_install:requirements.bzl", _compile_pip_requirements = "compile_pip_requirements")
1919

2020
compile_pip_requirements = _compile_pip_requirements
2121
package_annotation = _package_annotation
22+
package_overrides = _package_overrides
2223

2324
def pip_install(requirements = None, name = "pip", **kwargs):
2425
"""Accepts a locked/compiled requirements file and installs the dependencies listed within.

‎python/pip_install/extract_wheels/bazel.py

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,10 @@ def sanitised_repo_file_label(whl_name: str, repo_prefix: str) -> str:
326326
)
327327

328328

329+
def sanitised_alias_repo_library_label(repo: str, name: str) -> str:
330+
return '"@{}//{}"'.format(sanitise_name(repo, ""), sanitise_name(name, ""))
331+
332+
329333
def extract_wheel(
330334
wheel_file: str,
331335
extras: Dict[str, Set[str]],
@@ -335,6 +339,7 @@ def extract_wheel(
335339
incremental: bool = False,
336340
incremental_dir: Path = Path("."),
337341
annotation: Optional[annotation.Annotation] = None,
342+
parent_repo_name: Optional[str] = None,
338343
) -> Optional[str]:
339344
"""Extracts wheel into given directory and creates py_library and filegroup targets.
340345
@@ -347,6 +352,7 @@ def extract_wheel(
347352
effects the names of libraries and their dependencies, which point to other external repositories.
348353
incremental_dir: An optional override for the working directory of incremental builds.
349354
annotation: An optional set of annotations to apply to the BUILD contents of the wheel.
355+
parent_repo_name: The parent repo name, required when `incremental=True` and ignored otherwise.
350356
351357
Returns:
352358
The Bazel label for the extracted wheel, in the form '//path/to/wheel'.
@@ -373,8 +379,12 @@ def extract_wheel(
373379
whl_deps = sorted(whl.dependencies(extras_requested) - self_edge_dep)
374380

375381
if incremental:
382+
assert (
383+
parent_repo_name is not None
384+
), '"parent_repo_name" is required when incremental=True.'
376385
sanitised_dependencies = [
377-
sanitised_repo_library_label(d, repo_prefix=repo_prefix) for d in whl_deps
386+
sanitised_alias_repo_library_label(repo=parent_repo_name, name 10000 =d)
387+
for d in whl_deps
378388
]
379389
sanitised_wheel_file_dependencies = [
380390
sanitised_repo_file_label(d, repo_prefix=repo_prefix) for d in whl_deps

‎python/pip_install/extract_wheels/extract_single_wheel.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ def main() -> None:
9898
incremental=True,
9999
repo_prefix=args.repo_prefix,
100100
annotation=args.annotation,
101+
parent_repo_name=args.repo,
101102
)
102103

103104

‎python/pip_install/extract_wheels/parse_requirements_to_bzl.py

Lines changed: 84 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
import argparse
22
import json
3+
import os
34
import shlex
45
import sys
56
import textwrap
7+
import warnings
68
from pathlib import Path
79
from typing import Any, Dict, List, TextIO, Tuple
810

@@ -81,16 +83,75 @@ def parse_whl_library_args(args: argparse.Namespace) -> Dict[str, Any]:
8183
whl_library_args.setdefault("python_interpreter", sys.executable)
8284

8385
# These arguments are not used by `whl_library`
84-
for arg in ("requirements_lock", "requirements_lock_label", "annotations"):
86+
for arg in (
87+
"requirements_lock",
88+
"requirements_lock_label",
89+
"annotations",
90+
"library_overrides",
91+
):
8592
if arg in whl_library_args:
8693
whl_library_arg F987 s.pop(arg)
8794

8895
return whl_library_args
8996

9097

98+
def create_alias_packages(
99+
repo_prefix: str,
100+
install_requirements: List[Tuple[InstallRequirement, str]],
101+
library_overrides: Dict[str, Dict[str, str]],
102+
):
103+
"""Create alias packages and targets for each requirement."""
104+
for ir, _ in install_requirements:
105+
# The 'actual' library is the one in the incrementally fetched repo.
106+
# We need to strip the quotes here as we will encode w/ json.
107+
actual_library = bazel.sanitised_repo_library_label(
108+
ir.name, repo_prefix=repo_prefix
109+
)
110+
actual_library = actual_library.replace('"', "")
111+
112+
# The directory name is just the sanitized name.
113+
alias_name = bazel.sanitise_name(ir.name, "")
114+
os.mkdir(alias_name)
115+
116+
# Apply any overrides on top. We pop the keys here so we can report
117+
# unused libraries to the user. Currently this only accepts overrides
118+
# which use the alias (sanitized) name.
119+
py_library_selection = {
120+
"//conditions:default": actual_library,
121+
**library_overrides.pop(alias_name, {}),
122+
}
123+
124+
py_library_selection = json.dumps(py_library_selection)
125+
126+
build_file_content = textwrap.dedent(
127+
"""\
128+
129+
package(default_visibility = ["//visibility:public"])
130+
131+
alias(
132+
name = "{py_library_label}",
133+
actual = select({py_library_selection}),
134+
)
135+
""".format(
136+
py_library_label=alias_name,
137+
py_library_selection=py_library_selection,
138+
)
139+
)
140+
141+
with open(os.path.join(alias_name, "BUILD.bazel"), "w", encoding="utf-8") as f:
142+
f.write(build_file_content)
143+
144+
if len(library_overrides) > 0:
145+
warnings.warn(
146+
"Ignoring library overrides for packages not present in the "
147+
f"requirements file: {library_overrides}"
148+
)
149+
150+
91151
def generate_parsed_requirements_contents(
92152
requirements_lock: Path,
93153
repo_prefix: str,
154+
parent_repo_name: str,
94155
whl_library_args: Dict[str, Any],
95156
annotations: Dict[str, str] = dict(),
96157
) -> str:
@@ -107,9 +168,12 @@ def generate_parsed_requirements_contents(
107168
repo_names_and_reqs = repo_names_and_requirements(
108169
install_req_and_lines, repo_prefix
109170
)
171+
# Use the alias targets for `all_requirements`.
110172
all_requirements = ", ".join(
111173
[
112-
bazel.sanitised_repo_library_label(ir.name, repo_prefix=repo_prefix)
174+
bazel.sanitised_alias_repo_library_label(
175+
repo=parent_repo_name, name=ir.name
176+
)
113177
for ir, _ in install_req_and_lines
114178
]
115179
)
@@ -188,6 +252,11 @@ def coerce_to_bool(option):
188252
return str(option).lower() == "true"
189253

190254

255+
def parse_json_from_file(option):
256+
content = Path(option).read_text()
257+
return json.loads(content) if content.strip() != "" else {}
258+
259+
191260
def main(output: TextIO) -> None:
192261
"""Args:
193262
@@ -236,6 +305,11 @@ def main(output: TextIO) -> None:
236305
type=annotation.annotations_map_from_str_path,
237306
help="A json encoded file containing annotations for rendered packages.",
238307
)
308+
parser.add_argument(
309+
"--library_overrides",
310+
type=parse_json_from_file,
311+
help="A json encoded file containing library overrides for packages.",
312+
)
239313
arguments.parse_common_args(parser)
240314
args = parser.parse_args()
241315

@@ -248,6 +322,13 @@ def main(output: TextIO) -> None:
248322
req_names = sorted([req.name for req, _ in install_requirements])
249323
annotations = args.annotations.collect(req_names) if args.annotations else {}
250324

325+
# Generate build files for each library.
326+
create_alias_packages(
327+
repo_prefix=args.repo_prefix,
328+
install_requirements=install_requirements,
329+
library_overrides=args.library_overrides,
330+
)
331+
251332
# Write all rendered annotation files and generate a list of the labels to write to the requirements file
252333
annotated_requirements = dict()
253334
for name, content in annotations.items():
@@ -278,6 +359,7 @@ def main(output: TextIO) -> None:
278359
generate_parsed_requirements_contents(
279360
requirements_lock=args.requirements_lock,
280361
repo_prefix=args.repo_prefix,
362+
parent_repo_name=args.repo,
281363
whl_library_args=whl_library_args,
282364
annotations=annotated_requirements,
283365
)

‎python/pip_install/extract_wheels/parse_requirements_to_bzl_test.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ def test_generated_requirements_bzl(self) -> None:
2727
)
2828
args = argparse.Namespace()
2929
args.requirements_lock = str(requirements_lock.resolve())
30+
args.repo = "pip_parsed_deps_pypi"
3031
args.repo_prefix = "pip_parsed_deps_pypi__"
3132
extra_pip_args = ["--index-url=pypi.org/simple"]
3233
pip_data_exclude = ["**.foo"]
@@ -40,8 +41,9 @@ def test_generated_requirements_bzl(self) -> None:
4041
requirements_lock=args.requirements_lock,
4142
repo_prefix=args.repo_prefix,
4243
whl_library_args=whl_library_args,
44+
parent_repo_name=args.repo,
4345
)
44-
library_target = "@pip_parsed_deps_pypi__foo//:pkg"
46+
library_target = "@pip_parsed_deps_pypi//foo"
4547
whl_target = "@pip_parsed_deps_pypi__foo//:whl"
4648
all_requirements = 'all_requirements = ["{library_target}"]'.format(
4749
library_target=library_target

‎python/pip_install/extract_wheels/whl_filegroup_test.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import tempfile
44
import unittest
55
from pathlib import Path
6+
from typing import Optional
67

78
from python.pip_install.extract_wheels import bazel
89

@@ -21,13 +22,15 @@ def _run(
2122
self,
2223
repo_prefix: str,
2324
incremental: bool = False,
25+
repo_name: Optional[str] = None,
2426
) -> None:
2527
generated_bazel_dir = bazel.extract_wheel(
2628
self.wheel_path,
2729
extras={},
2830
pip_data_exclude=[],
2931
enable_implicit_namespace_pkgs=False,
3032
incremental=incremental,
33+
parent_repo_name=repo_name,
3134
repo_prefix=repo_prefix,
3235
incremental_dir=Path(self.wheel_dir),
3336
)
@@ -46,7 +49,7 @@ def test_nonincremental(self) -> None:
4649
self._run(repo_prefix="prefix_")
4750

4851
def test_incremental(self) -> None:
49-
self._run(incremental=True, repo_prefix="prefix_")
52+
self._run(incremental=True, repo_prefix="prefix_", repo_name="prefix")
5053

5154

5255
if __name__ == "__main__":

0 commit comments

Comments
 (0)
0