8000 Incrementally download wheels at workspace time. (#432) · bazel-contrib/rules_python@7aaf762 · GitHub
[go: up one dir, main page]
Skip to content

Commit 7aaf762

Browse files
hrfullerhrfuller
authored
Incrementally download wheels at workspace time. (#432)
* Create support for lazily fetched repo's. Refactor pip_repository rule to invoke different scripts based on the value of the incremental attribute to the rule. Create a new macro in repositories.bzl which will instantiate all the child repos representing individual python packages. Refactor code which is repeated between the parse_requirements_to_bzl scripts and the extract_wheels script.
1 parent c37ba22 commit 7aaf762

File tree

26 files changed

+765
-94
lines changed

26 files changed

+765
-94
lines changed

‎.bazelrc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# This lets us glob() up all the files inside the examples to make them inputs to tests
44
# (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
55
# To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh
6-
build --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install
7-
query --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install
6+
build --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse
7+
query --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse
88

99
test --test_output=errors

‎.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,7 @@ bazel-bin
3737
bazel-genfiles
3838
bazel-out
3939
bazel-testlogs
40+
41+
# vim swap files
42+
*.swp
43+
*.swo

‎README.md

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ target in the appropriate wheel repo.
105105

106106
### Importing `pip` dependencies
107107

108-
To add pip dependencies to your `WORKSPACE` is you load
108+
To add pip dependencies to your `WORKSPACE` load
109109
the `pip_install` function, and call it to create the
110110
individual wheel repos.
111111

@@ -136,6 +136,40 @@ re-executed in order to pick up a non-hermetic change to your environment (e.g.,
136136
updating your system `python` interpreter), you can completely flush out your
137137
repo cache with `bazel clean --expunge`.
138138

139+
### Fetch `pip` dependencies lazily (experimental)
140+
141+
One pain point with `pip_install` is the need to download all dependencies resolved by
142+
your requirements.txt before the bazel analysis phase can start. For large python monorepos
143+
this can take a long time, especially on slow connections.
144+
145+
`pip_parse` provides a solution to this problem. If you can provide a lock
146+
file of all your python dependencies `pip_parse` will translate each requirement into its own external repository.
147+
Bazel will only fetch/build wheels for the requirements in the subgraph of your build target.
148+
149+
There are API differences between `pip_parse` and `pip_install`:
150+
1. `pip_parse` requires a fully resolved lock file of your python dependencies. You can generate this using
151+
`pip-compile`, or a virtualenv and `pip freeze`. `pip_parse` uses a label argument called `requirements_lock` instead of `requirements`
152+
to make this distinction clear.
153+
2. `pip_parse` translates your requirements into a starlark macro called `install_deps`. You must call this macro in your WORKSPACE to
154+
declare your dependencies.
155+
156+
157+
```python
158+
load("@rules_python//python:pip.bzl", "pip_parse")
159+
160+
# Create a central repo that knows about the dependencies needed from
161+
# requirements_lock.txt.
162+
pip_parse(
163+
name = "my_deps",
164+
requirements_lock = "//path/to:requirements_lock.txt",
165+
)
166+
167+
# Load the starlark macro which will define your dependencies.
168+
load("@my_deps//:requirements.bzl", "install_deps")
169+
# Call it to define repos for your requirements.
170+
install_deps()
171+
```
172+
139173
### Importing `pip` dependencies with `pip_import` (legacy)
140174

141175
The deprecated `pip_import` can still be used if needed.

‎examples/BUILD

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,8 @@ bazel_integration_test(
2626
name = "pip_install_example",
2727
timeout = "long",
2828
)
29+
30+
bazel_integration_test(
31+
name = "pip_parse_example",
32+
timeout = "long",
33+
)

‎examples/pip_parse/BUILD

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
load("@pip_parsed_deps//:requirements.bzl", "requirement")
2+
load("@rules_python//python:defs.bzl", "py_binary", "py_test")
3+
4+
# Toolchain setup, this is optional.
5+
# Demonstrate that we can use the same python interpreter for the toolchain and executing pip in pip install (see WORKSPACE).
6+
#
7+
#load("@rules_python//python:defs.bzl", "py_runtime_pair")
8+
#
9+
#py_runtime(
10+
# name = "python3_runtime",
11+
# files = ["@python_interpreter//:files"],
12+
# interpreter = "@python_interpreter//:python_bin",
13+
# python_version = "PY3",
14+
# visibility = ["//visibility:public"],
15+
#)
16+
#
17+
#py_runtime_pair(
18+
# name = "my_py_runtime_pair",
19+
# py2_runtime = None,
20+
# py3_runtime = ":python3_runtime",
21+
#)
22+
#
23+
#toolchain(
24+
# name = "my_py_toolchain",
25+
# toolchain = ":my_py_runtime_pair",
26+
# toolchain_type = "@bazel_tools//tools/python:toolchain_type",
27+
#)
28+
# End of toolchain setup.
29+
30+
py_binary(
31+
name = "main",
32+
srcs = ["main.py"],
33+
deps = [
34+
requirement("requests"),
35+
],
36+
)
37+
38+
py_test(
39+
name = "test",
40+
srcs = ["test.py"],
41+
deps = [":main"],
42+
)

‎examples/pip_parse/WORKSPACE

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
workspace(name = "example_repo")
2+
3+
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
4+
5+
http_archive(
6+
name = "rules_python",
7+
url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz",
8+
sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0",
9+
)
10+
11+
load("@rules_python//python:pip.bzl", "pip_parse")
12+
13+
pip_parse(
14+
# (Optional) You can provide extra parameters to pip.
15+
# Here, make pip output verbose (this is usable with `quiet = False`).
16+
# extra_pip_args = ["-v"],
17+
18+
# (Optional) You can exclude custom elements in the data section of the generated BUILD files for pip packages.
19+
# Exclude directories with spaces in their names in this example (avoids build errors if there are such directories).
20+
#pip_data_exclude = ["**/* */**"],
21+
22+
# (Optional) You can provide a python_interpreter (path) or a python_interpreter_target (a Bazel target, that
23+
# acts as an executable). The latter can be anything that could be used as Python interpreter. E.g.:
24+
# 1. Python interpreter that you compile in the build file (as above in @python_interpreter).
25+
# 2. Pre-compiled python interpreter included with http_archive
26+
# 3. Wrapper script, like in the autodetecting python toolchain.
27+
#python_interpreter_target = "@python_interpreter//:python_bin",
28+
29+
# (Optional) You can set quiet to False if you want to see pip output.
30+
#quiet = False,
31+
32+
# Uses the default repository name "pip_incremental"
33+
requirements_lock = "//:requirements_lock.txt",
34+
)
35+
36+
load("@pip_parsed_deps//:requirements.bzl", "install_deps")
37+
38+
# Initialize repositories for all packages in requirements_lock.txt.
39+
install_deps()

‎examples/pip_parse/main.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
import requests
2+
3+
4+
def version():
5+
return requests.__version__

‎examples/pip_parse/requirements.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
requests==2.24.0

‎examples/pip_parse/requirements_lock.txt

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#
2+
# This file is autogenerated by pip-compile
3+
# To update, run:
4+
#
5+
# pip-compile --output-file=requirements_lock.txt requirements.txt
6+
#
7+
certifi==2020.12.5
8+
# via requests
9+
chardet==3.0.4
10+
# via requests
11+
idna==2.10
12+
# via requests
13+
requests==2.24.0
14+
# via -r requirements.txt
15+
urllib3==1.25.11
16+
# via requests

‎examples/pip_parse/test.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import unittest
2+
import main
3+
4+
5+
class ExampleTest(unittest.TestCase):
6+
def test_main(self):
7+
self.assertEqual("2.24.0", main.version())
8+
9+
10+
if __name__ == '__main__':
11+
unittest.main()

‎python/pip.bzl

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,17 @@ def pip_install(requirements, name = "pip", **kwargs):
5656
**kwargs
5757
)
5858

59+
def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs):
60+
# Just in case our dependencies weren't already fetched
61+
pip_install_dependencies()
62+
63+
pip_repository(
64+
name = name,
65+
requirements_lock = requirements_lock,
66+
incremental = True,
67+
**kwargs
68+
)
69+
5970
def pip_repositories():
6071
# buildifier: disable=print
6172
print("DEPRECATED: the pip_repositories rule has been replaced with pip_install, please see rules_python 0.1 release notes")

‎python/pip_install/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ filegroup(
33
srcs = glob(["*.bzl"]) + [
44
"BUILD",
55
"//python/pip_install/extract_wheels:distribution",
6+
"//python/pip_install/parse_requirements_to_bzl:distribution",
67
],
78
visibility = ["//:__pkg__"],
89
)

‎python/pip_install/extract_wheels/__init__.py

Lines changed: 6 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
import sys
1313
import json
1414

15-
from python.pip_install.extract_wheels.lib import bazel, requirements
15+
from python.pip_install.extract_wheels.lib import bazel, requirements, arguments
1616

1717

1818
def configure_reproducible_wheels() -> None:
@@ -58,25 +58,7 @@ def main() -> None:
5858
required=True,
5959
help="Path to requirements.txt from where to install dependencies",
6060
)
61-
parser.add_argument(
62-
"--repo",
63-
action="store",
64-
required=True,
65-
help="The external repo name to install dependencies. In the format '@{REPO_NAME}'",
66-
)
67-
parser.add_argument(
68-
"--extra_pip_args", action="store", help="Extra arguments to pass down to pip.",
69-
)
70-
parser.add_argument(
71-
"--pip_data_exclude",
72-
action="store",
73-
help="Additional data exclusion parameters to add to the pip packages BUILD file.",
74-
)
75-
parser.add_argument(
76-
"--enable_implicit_namespace_pkgs",
77-
action="store_true",
78-
help="Disables conversion of implicit namespace packages into pkg-util style packages.",
79-
)
61+
arguments.parse_common_args(parser)
8062
args = parser.parse_args()
8163

8264
pip_args = [sys.executable, "-m", "pip", "--isolated", "wheel", "-r", args.requirements]
@@ -93,10 +75,12 @@ def main() -> None:
9375
else:
9476
pip_data_exclude = []
9577

78+
repo_label = "@%s" % args.repo
79+
9680
targets = [
9781
'"%s%s"'
9882
% (
99-
args.repo,
83+
repo_label,
10084
bazel.extract_wheel(
10185
whl, extras, pip_data_exclude, args.enable_implicit_namespace_pkgs
10286
),
@@ -106,5 +90,5 @@ def main() -> None:
10690

10791
with open("requirements.bzl", "w") as requirement_file:
10892
requirement_file.write(
109-
bazel.generate_requirements_file_contents(args.repo, targets)
93+
bazel.generate_requirements_file_contents(repo_label, targets)
11094
)

‎python/pip_install/extract_wheels/lib/BUILD

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,12 @@ py_library(
99
"purelib.py",
1010
"requirements.py",
1111
"wheel.py",
12+
"arguments.py",
13+
],
14+
visibility = [
15+
"//python/pip_install/extract_wheels:__subpackages__",
16+
"//python/pip_install/parse_requirements_to_bzl:__subpackages__",
1217
],
13-
visibility = ["//python/pip_install/extract_wheels:__subpackages__"],
1418
deps = [
1519
requirement("pkginfo"),
1620
requirement("setuptools"),
@@ -41,6 +45,19 @@ py_test(
4145
],
4246
)
4347

48+
py_test(
49+
name = "arguments_test",
50+
size = "small",
51+
srcs = [
52+
"arguments_test.py",
53+
],
54+
tags = ["unit"],
55+
deps = [
56+
":lib",
57+
"//python/pip_install/parse_requirements_to_bzl:lib",
58+
],
59+
)
60+
4461
py_test(
4562
name = "whl_filegroup_test",
4663
size = "small",

‎python/pip_install/extract_wheels/lib/arguments.py

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
from argparse import ArgumentParser
2+
3+
4+
def parse_common_args(parser: ArgumentParser) -> ArgumentParser:
5+
parser.add_argument(
6+
"--repo",
7+
action="store",
8+
required=True,
9+
help="The external repo name to install dependencies. In the format '@{REPO_NAME}'",
10+
)
11+
parser.add_argument(
12+
"--extra_pip_args", action="store", help="Extra arguments to pass down to pip.",
13+
)
14+
parser.add_argument(
15+
"--pip_data_exclude",
16+
action="store",
17+
help="Additional data exclusion parameters to add to the pip packages BUILD file.",
18+
)
19+
parser.add_argument(
20+
"--enable_implicit_namespace_pkgs",
21+
action="store_true",
22+
help="Disables conversion of implicit namespace packages into pkg-util style packages.",
23+
)
24+
return parser

‎python/pip_install/extract_wheels/lib/arguments_test.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import argparse
2+
import json
3+
import unittest
4+
5+
from python.pip_install.extract_wheels.lib import arguments
6+
from python.pip_install.parse_requirements_to_bzl import deserialize_structured_args
7+
8+
9+
class ArgumentsTestCase(unittest.TestCase):
10+
def test_arguments(self) -> None:
11+
parser = argparse.ArgumentParser()
12+
parser = arguments.parse_common_args(parser)
13+
repo_name = "foo"
14+
index_url = "--index_url=pypi.org/simple"
15+
args_dict = vars(parser.parse_args(
16+
args=["--repo", repo_name, "--extra_pip_args={index_url}".format(index_url=json.dumps({"args": index_url}))]))
17+
args_dict = deserialize_structured_args(args_dict)
18+
self.assertIn("repo", args_dict)
19+
self.assertIn("extra_pip_args", args_dict)
20+
self.assertEqual(args_dict["pip_data_exclude"], None)
21+
self.assertEqual(args_dict["enable_implicit_namespace_pkgs"], False)
22+
self.assertEqual(args_dict["repo"], repo_name)
23+
self.assertEqual(args_dict["extra_pip_args"], index_url)
24+
25+
26+
if __name__ == "__main__":
27+
unittest.main()

0 commit comments

Comments
 (0)
0