10000 Introduce compile_pip_requirements rule · bazel-contrib/rules_python@d6984b1 · GitHub
[go: up one dir, main page]
Skip to content

Commit d6984b1

Browse files
author
Alex Eagle
committed
Introduce compile_pip_requirements rule
This uses pip-tools to compile a requirements.in file to a requirements.txt file, allowing transitive dependency versions to be pinned so that builds are reproducible. Fixes #176
1 parent 5be1f76 commit d6984b1

File tree

12 files changed

+271
-8
lines changed

12 files changed

+271
-8
lines changed

‎examples/pip_install/.bazelrc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
test --test_output=errors

‎examples/pip_install/BUILD

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
load("@pip//:requirements.bzl", "requirement")
22
load("@rules_python//python:defs.bzl", "py_binary", "py_test")
3+
load("@rules_python//python/pip_install:requirements.bzl", "compile_pip_requirements")
34

45
# Toolchain setup, this is optional.
56
# Demonstrate that we can use the same python interpreter for the toolchain and executing pip in pip install (see WORKSPACE).
@@ -40,3 +41,6 @@ py_test(
4041
srcs = ["test.py"],
4142
deps = [":main"],
4243
)
44+
45+
# Check that our compiled requirements are up-to-date
46+
compile_pip_requirements()

‎examples/pip_install/WORKSPACE

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,17 @@ http_archive(
88
sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0",
99
)
1010

11+
# Should not be here, workaround for
12+
# https://github.com/bazelbuild/rules_python/issues/372
13+
http_archive(
14+
name = "bazel_skylib",
15+
urls = [
16+
"https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
17+
"https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
18+
],
19+
sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
20+
)
21+
1122
load("@rules_python//python:pip.bzl", "pip_install")
1223

1324
pip_install(

‎examples/pip_install/requirements.in

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
boto3==1.14.51

‎examples/pip_install/requirements.txt

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,39 @@
1-
boto3==1.14.51
1+
#
2+
# This file is autogenerated by pip-compile
3+
# To update, run:
4+
#
5+
# bazel run //:requirements.update
6+
#
7+
boto3==1.14.51 \
8+
--hash=sha256:a6bdb808e948bd264af135af50efb76253e85732c451fa605b7a287faf022432 \
9+
--hash=sha256:f9dbccbcec916051c6588adbccae86547308ac4cd154f1eb7cf6422f0e391a71 \
10+
# via -r ./requirements.in
11+
botocore==1.17.63 \
12+
--hash=sha256:40f13f6c9c29c307a9dc5982739e537ddce55b29787b90c3447b507e3283bcd6 \
13+
--hash=sha256:aa88eafc6295132f4bc606f1df32b3248e0fa611724c0a216aceda767948ac75 \
14+
# via boto3, s3transfer
15+
docutils==0.15.2 \
16+
--hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \
17+
--hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \
18+
--hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99 \
19+
# via botocore
20+
jmespath==0.10.0 \
21+
--hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \
22+
--hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f \
23+
# via boto3, botocore
24+
python-dateutil==2.8.1 \
25+
--hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
26+
--hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \
27+
# via botocore
28+
s3transfer==0.3.3 \
29+
--hash=sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13 \
30+
--hash=sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db \
31+
# via boto3
32+
six==1.15.0 \
33+
--hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
34+
--hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
35+
# via python-dateutil
36+
urllib3==1.25.11 \
37+
--hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \
38+
--hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e \
39+
# via botocore

‎python/BUILD

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
In an ideal renaming, we'd move the packaging rules to a different package so
2424
that @rules_python//python is only concerned with the core rules.
2525
"""
26+
load("//python/pip_install:requirements.bzl", "compile_pip_requirements")
2627

2728
package(default_visibility = ["//visibility:public"])
2829

@@ -130,3 +131,5 @@ exports_files([
130131
"pip.bzl",
131132
"whl.bzl",
132133
])
134+
135+
compile_pip_requirements(extra_args = ["--allow-unsafe"])

‎python/pip_install/BUILD

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
22

3+
exports_files(["pip_compile.py"])
4+
35
filegroup(
46
name = "distribution",
57
srcs = glob(["*.bzl"]) + [
68
"BUILD",
9+
"pip_compile.py",
710
"//python/pip_install/extract_wheels:distribution",
811
],
912
visibility = ["//:__pkg__"],

‎python/pip_install/pip_compile.py

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
"Set defaults for the pip-compile command to run it under Bazel"
2+
3+
import os
4+
import sys
5+
from shutil import copyfile
6+
7+
from piptools.scripts.compile import cli
8+
9+
if len(sys.argv) < 4:
10+
print(
11+
"Expected at least two arguments: requirements_in requirements_out",
12+
file=sys.stderr,
13+
)
14+
sys.exit(1)
15+
16+
requirements_in = sys.argv.pop(1)
17+
requirements_txt = sys.argv.pop(1)
18+
update_target_name = sys.argv.pop(1)
19+
20+
# Before loading click, set the locale for its parser.
21+
# If it leaks through to the system setting, it may fail:
22+
# RuntimeError: Click will abort further execution because Python 3 was configured to use ASCII
23+
# as encoding for the environment. Consult https://click.palletsprojects.com/python3/ for
24+
# mitigation steps.
25+
os.environ["LC_ALL"] = "C.UTF-8"
26+
os.environ["LANG"] = "C.UTF-8"
27+
28+
UPDATE = True
29+
# Detect if we are running under `bazel test`
30+
if "TEST_TMPDIR" in os.environ:
31+
UPDATE = False
32+
# pip-compile wants the cache files to be writeable, but if we point
33+
# to the real user cache, Bazel sandboxing makes the file read-only
34+
# and we fail.
35+
# In theory this makes the test more hermetic as well.
36+
sys.argv.append("--cache-dir")
37+
sys.argv.append(os.environ["TEST_TMPDIR"])
38+
# Make a copy for pip-compile to read and mutate
39+
requirements_out = os.path.join(
40+
os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out"
41+
)
42+
copyfile(requirements_txt, requirements_out)
43+
44+
elif "BUILD_WORKING_DIRECTORY" in os.environ:
45+
os.chdir(os.environ['BUILD_WORKING_DIRECTORY'])
46+
else:
47+
print(
48+
"Expected to find BUILD_WORKING_DIRECTORY in environment",
49+
file=sys.stderr,
50+
)
51+
sys.exit(1)
52+
53+
update_target_pkg = "/".join(requirements_in.split('/')[:-1])
54+
# $(rootpath) in the workspace root gives ./requirements.in
55+
if update_target_pkg == ".":
56+
update_target_pkg = ""
57+
update_command = "bazel run //%s:%s" % (update_target_pkg, update_target_name)
58+
59+
os.environ["CUSTOM_COMPILE_COMMAND"] = update_command
60+
61+
sys.argv.append("--generate-hashes")
62+
sys.argv.append("--output-file")
63+
sys.argv.append(requirements_txt if UPDATE else requirements_out)
64+
sys.argv.append(requirements_in)
65+
66+
if UPDATE:
67+
print("Updating " + requirements_txt)
68+
cli()
69+
else:
70+
# cli will exit(0) on success
71+
try:
72+
print("Checking " + requirements_txt)
73+
cli()
74+
print("cl() should exit", file=sys.stderr)
75+
sys.exit(1)
76+ except SystemExit:
77+
golden = open(requirements_txt).readlines()
78+
out = open(requirements_out).readlines()
79+
if golden != out:
80+
import difflib
81+
82+
print(''.join(difflib.unified_diff(golden, out)), file=sys.stderr)
83+
print(
84+
"Lock file out of date. Run '"
85+
+ update_command
86+
+ "' to update.",
87+
file=sys.stderr,
88+
)
89+
sys.exit(1)

‎python/pip_install/repositories.bzl

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,20 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
44
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
55

66
_RULE_DEPS = [
7+
(
8+
"pypi__click",
9+
"https://files.pythonhosted.org/packages/d2/3d/fa76db83bf75c4f8d338c2fd15c8d33fdd7ad23a9b5e57eb6c5de26b430e/click-7.1.2-py2.py3-none-any.whl",
10+
"dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc",
11+
),
712
(
813
"pypi__pip",
9-
"https://files.pythonhosted.org/packages/00/b6/9cfa56b4081ad13874b0c6f96af8ce16cfbc1cb06bedf8e9164ce5551ec1/pip-19.3.1-py2.py3-none-any.whl",
10-
"6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7",
14+
"https://files.pythonhosted.org/packages/43/84/23ed6a1796480a6f1a2d38f2802901d078266bda38388954d01d3f2e821d/pip-20.1.1-py2.py3-none-any.whl",
15+
"b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4",
16+
),
17+
(
18+
"pypi__pip_tools",
19+
"https://files.pythonhosted.org/packages/a1/d5/c0f282060c483e6fd4635ed8f4f88aff02c5490d0fc588edec52b0cb9d7b/pip_tools-5.3.1-py2.py3-none-any.whl",
20+
"73787e23269bf8a9230f376c351297b9037ed0d32ab0f9bef4a187d976acc054",
1121
),
1222
(
1323
"pypi__pkginfo",

‎python/pip_install/requirements.bzl

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
"Rules to verify and update pip-compile locked requirements.txt"
2+
3+
load("//python:defs.bzl", "py_binary", "py_test")
4+
load("//python/pip_install:repositories.bzl", "requirement")
5+
6+
def compile_pip_requirements(
7+
name = "requirements",
8+
extra_args = [],
9+
visibility = ["//visibility:private"],
10+
**kwargs):
11+
"""
12+
Produce two targets for checking pip-compile:
13+
14+
- validate with `bazel test <name>_test`
15+
- update with `bazel run <name>.update`
16+
17+
By default requirements in file is expected to be <name>.in and output
18+
requirements txt lock file is expected to be <name>.txt. These may be customized
19+
with `requirements_in` and `requirements_locked` params.
20+
21+
Args:
22+
name: string
23+
extra_args: passed to pip-compile
24+
visibility: passed to both the _test and .update rules
25+
**kwargs: other bazel attributes passed to the "_test" rule
26+
"""
27+
requirements_in = kwargs.pop("requirements_in", name + ".in")
28+
requirements_txt = kwargs.pop("requirements_locked", name + ".txt")
29+
30+
data = kwargs.pop("data", []) + [requirements_in, requirements_txt]
31+
32+
loc = "$(rootpath %s)"
33+
34+
# Use the Label constructor so this is expanded in the context of the file
35+
# where it appears, which is to say, in @rules_python
36+
pip_compile = Label("//python/pip_install:pip_compile.py")
37+
38+
args = [
39+
loc % requirements_in,
40+
loc % requirements_txt,
41+
name + ".update",
42+
] + extra_args
43+
44+
py_binary(
45+
name = name + ".update",
46+
srcs = [pip_compile],
47+
main = pip_compile,
48+
args = args,
49+
visibility = visibility,
50+
deps = [
51+
requirement("click"),
52+
requirement("pip"),
53+
requirement("pip_tools"),
54+
],
55+
data = data,
56+
)
57+
58+
py_test(
59+
name = name + "_test",
60+
srcs = [pip_compile],
61+
main = pip_compile,
62+
args = args,
63+
visibility = visibility,
64+
deps = [
65+
requirement("click"),
66+
requirement("pip"),
67+
requirement("pip_tools"),
68+
],
69+
data = data,
70+
**kwargs
71+
)

‎python/requirements.in

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pip==9.0.3
2+
setuptools==44.0.0
3+
wheel==0.30.0a0
4+
5+
# For tests
6+
mock==2.0.0

‎python/requirements.txt

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,32 @@
1-
pip==9.0.3
2-
setuptools==44.0.0
3-
wheel==0.30.0a0
1+
#
2+
# This file is autogenerated by pip-compile
3+
# To update, run:
4+
#
5+
# bazel run //python:requirements.update
6+
#
7+
mock==2.0.0 \
8+
--hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
9+
--hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba \
10+
# via -r python/requirements.in
11+
pbr==5.5.1 \
12+
--hash=sha256:5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9 \
13+
--hash=sha256:b236cde0ac9a6aedd5e3c34517b423cd4fd97ef723849da6b0d2231142d89c00 \
14+
# via mock
15+
six==1.15.0 \
16+
--hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
17+
--hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
18+
# via mock
19+
wheel==0.30.0a0 \
20+
--hash=sha256:98f3e09b4ad7f5649a7e3d00e0e005ec1824ddcd6ec16c5086c05b1d91ada6da \
21+
--hash=sha256:cd19aa9325d3af1c641b0a23502b12696159171d2a2f4b84308df9a075c2a4a0 \
22+
# via -r python/requirements.in
423

5-
# For tests
6-
mock==2.0.0
24+
# The following packages are considered to be unsafe in a requirements file:
25+
pip==9.0.3 \
26+
--hash=sha256:7bf48f9a693be1d58f49f7af7e0ae9fe29fd671cde8a55e6edca3581c4ef5796 \
27+
--hash=sha256:c3ede34530e0e0b2381e7363aded78e0c33291654937e7373032fda04e8803e5 \
28+
# via -r python/requirements.in
29+
setuptools==44.0.0 \
30+
--hash=sha256:180081a244d0888b0065e18206950d603f6550721bd6f8c0a10221ed467dd78e \
31+
--hash=sha256:e5baf7723e5bb8382fc146e33032b241efc63314211a3a120aaa55d62d2bb008 \
32+
# via -r python/requirements.in

0 commit comments

Comments
 (0)
0