8000 Parse requirements files for global pip flags. (#456) · lapointexavier/rules_python@37f04a4 · GitHub
[go: up one dir, main page]
Skip to content

Commit 37f04a4

Browse files
hrfulle 10000 rhrfuller
authored
Parse requirements files for global pip flags. (bazel-contrib#456)
Parse requirements files for global pip flags, then add them to the extra_pip_args array. Pass a copy of each cleaned up line (comments and line-breaks removed) to each whl_library repo verbatim. Then link based requirements or requirement lines with requirement specific flags work. Due to the pip restriction on using requirement specific flags in requirements.txt files only, write each requirement line to a temp file before invoking pip in each whl_library repo. Fixes bazel-contrib#438 bazel-contrib#447
1 parent 0ab06a2 commit 37f04a4

File tree

6 files changed

+68
-26
lines changed

6 files changed

+68
-26
lines changed

‎examples/pip_parse/WORKSPACE

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ pip_parse(
2929
# (Optional) You can set quiet to False if you want to see pip output.
3030
#quiet = False,
3131

32-
# Uses the default repository name "pip_incremental"
32+
# Uses the default repository name "pip_parsed_deps"
3333
requirements_lock = "//:requirements_lock.txt",
3434
)
3535

‎examples/pip_parse/requirements_lock.txt

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,27 @@
22
# This file is autogenerated by pip-compile
33
# To update, run:
44
#
5-
# pip-compile --output-file=requirements_lock.txt requirements.txt
5+
# pip-compile --generate-hashes --output-file=requirements_lock.txt requirements.txt
66
#
7-
certifi==2020.12.5
7+
--extra-index-url https://pypi.org/simple
8+
9+
certifi==2020.12.5 \
10+
--hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \
11+
--hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 \
812
# via requests
9-
chardet==3.0.4
13+
chardet==3.0.4 \
14+
--hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
15+
--hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
1016
# via requests
11-
idna==2.10
17+
idna==2.10 \
18+
--hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \
19+
--hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 \
1220
# via requests
13-
requests==2.24.0
21+
requests==2.24.0 \
22+
--hash=sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b \
23+
--hash=sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898 \
1424
# via -r requirements.txt
15-
urllib3==1.25.11
25+
urllib3==1.25.11 \
26+
--hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \
27+
--hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e \
1628
# via requests

‎python/pip_install/extract_wheels/lib/arguments_test.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ def test_arguments(self) -> None:
1717
args_dict = deserialize_structured_args(args_dict)
1818
self.assertIn("repo", args_dict)
1919
self.assertIn("extra_pip_args", args_dict)
20-
self.assertEqual(args_dict["pip_data_exclude"], None)
20+
self.assertEqual(args_dict["pip_data_exclude"], [])
2121
self.assertEqual(args_dict["enable_implicit_namespace_pkgs"], False)
2222
self.assertEqual(args_dict["repo"], repo_name)
2323
self.assertEqual(args_dict["extra_pip_args"], index_url)

‎python/pip_install/parse_requirements_to_bzl/__init__.py

Lines changed: 33 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,28 +2,47 @@
22
import json
33
import textwrap
44
import sys
5+
import shlex
56
from typing import List, Tuple
67

78
from python.pip_install.extract_wheels.lib import bazel, arguments
89
from pip._internal.req import parse_requirements, constructors
910
from pip._internal.req.req_install import InstallRequirement
11+
from pip._internal.req.req_file import get_file_content, preprocess, handle_line, get_line_parser, RequirementsFileParser
1012
from pip._internal.network.session import PipSession
1113

1214

13-
def parse_install_requirements(requirements_lock: str) -> List[InstallRequirement]:
14-
return [
15-
constructors.install_req_from_parsed_requirement(pr)
16-
for pr in parse_requirements(requirements_lock, session=PipSession())
17-
]
15+
def parse_install_requirements(requirements_lock: str, extra_pip_args: List[str]) -> List[Tuple[InstallRequirement, str]]:
16+
ps = PipSession()
17+
# This is roughly taken from pip._internal.req.req_file.parse_requirements
18+
# (https://github.com/pypa/pip/blob/21.0.1/src/pip/_internal/req/req_file.py#L127) in order to keep
19+
# the original line (sort-of, its preprocessed) from the requirements_lock file around, to pass to sub repos
20+
# as the requirement.
21+
line_parser = get_line_parser(finder=None)
22+
parser = RequirementsFileParser(ps, line_parser)
23+
install_req_and_lines: List[Tuple[InstallRequirement, str]] = []
24+
_, content = get_file_content(requirements_lock, ps)
25+
for parsed_line, (_, line) in zip(parser.parse(requirements_lock, constraint=False), preprocess(content)):
26+
if parsed_line.is_requirement:
27+
install_req_and_lines.append(
28+
(
29+
constructors.install_req_from_line(parsed_line.requirement),
30+
line
31+
)
32+
)
33+
34+
else:
35+
extra_pip_args.extend(shlex.split(line))
36+
return install_req_and_lines
1837

1938

20-
def repo_names_and_requirements(install_reqs: List[InstallRequirement], repo_prefix: str) -> List[Tuple[str, str]]:
39+
def repo_names_and_requirements(install_reqs: List[Tuple[InstallRequirement, str]], repo_prefix: str) -> List[Tuple[str, str]]:
2140
return [
2241
(
2342
bazel.sanitise_name(ir.name, prefix=repo_prefix),
24-
str(ir.req)
43+
line,
2544
)
26-
for ir in install_reqs
45+
for ir, line in install_reqs
2746
]
2847

2948
def deserialize_structured_args(args):
@@ -35,6 +54,8 @@ def deserialize_structured_args(args):
3554
for arg_name in structured_args:
3655
if args.get(arg_name) is not None:
3756
args[arg_name] = json.loads(args[arg_name])["args"]
F438 57+
else:
58+
args[arg_name] = []
3859
return args
3960

4061

@@ -54,13 +75,13 @@ def generate_parsed_requirements_contents(all_args: argparse.Namespace) -> str:
5475
requirements_lock = args.pop("requirements_lock")
5576
repo_prefix = bazel.whl_library_repo_prefix(args["repo"])
5677

57-
install_reqs = parse_install_requirements(requirements_lock)
58-
repo_names_and_reqs = repo_names_and_requirements(install_reqs, repo_prefix)
78+
install_req_and_lines = parse_install_requirements(requirements_lock, args["extra_pip_args"])
79+
repo_names_and_reqs = repo_names_and_requirements(install_req_and_lines, repo_prefix)
5980
all_requirements = ", ".join(
60-
[bazel.sanitised_repo_library_label(ir.name, repo_prefix=repo_prefix) for ir in install_reqs]
81+
[bazel.sanitised_repo_library_label(ir.name, repo_prefix=repo_prefix) for ir, _ in install_req_and_lines]
6182
)
6283
all_whl_requirements = ", ".join(
63-
[bazel.sanitised_repo_file_label(ir.name, repo_prefix=repo_prefix) for ir in install_reqs]
84+
[bazel.sanitised_repo_file_label(ir.name, repo_prefix=repo_prefix) for ir, _ in install_req_and_lines]
6485
)
6586
return textwrap.dedent("""\
6687
load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library")

‎python/pip_install/parse_requirements_to_bzl/extract_single_wheel/__init__.py

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
import subprocess
55
import json
66

7+
from tempfile import NamedTemporaryFile
8+
79
from python.pip_install.extract_wheels.lib import bazel, requirements, arguments
810
from python.pip_install.extract_wheels import configure_reproducible_wheels
911

@@ -27,10 +29,15 @@ def main() -> None:
2729
if args.extra_pip_args:
2830
pip_args += json.loads(args.extra_pip_args)["args"]
2931

30-
pip_args.append(args.requirement)
32+
with NamedTemporaryFile(mode='wb') as requirement_file:
33+
requirement_file.write(args.requirement.encode("utf-8"))
34+
requirement_file.flush()
35+
# Requirement specific args like --hash can only be passed in a requirements file,
36+
# so write our single requirement into a temp file in case it has any of those flags.
37+
pip_args.extend(["-r", requirement_file.name])
3138

32-
# Assumes any errors are logged by pip so do nothing. This command will fail if pip fails
33-
subprocess.run(pip_args, check=True)
39+
# Assumes any errors are logged by pip so do nothing. This command will fail if pip fails
40+
subprocess.run(pip_args, check=True)
3441

3542
name, extras_for_pkg = requirements._parse_requirement_for_extra(args.requirement)
3643
extras = {name: extras_for_pkg} if extras_for_pkg and name else dict()

‎python/pip_install/parse_requirements_to_bzl/parse_requirements_to_bzl_test.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,9 @@ class TestParseRequirementsToBzl(unittest.TestCase):
1515

1616
def test_generated_requirements_bzl(self) -> None:
1717
with NamedTemporaryFile() as requirements_lock:
18-
requirement_string = "foo==0.0.0"
19-
requirements_lock.write(bytes(requirement_string, encoding="utf-8"))
18+
comments_and_flags = "#comment\n--require-hashes True\n"
19+
requirement_string = "foo==0.0.0 --hash=sha256:hashofFoowhl"
20+
requirements_lock.write(bytes(comments_and_flags + requirement_string, encoding="utf-8"))
2021
requirements_lock.flush()
2122
args = argparse.Namespace()
2223
args.requirements_lock = requirements_lock.name
@@ -32,7 +33,8 @@ def test_generated_requirements_bzl(self) -> None:
3233
self.assertIn(all_whl_requirements, contents, contents)
3334
self.assertIn(requirement_string, contents, contents)
3435
self.assertIn(requirement_string, contents, contents)
35-
self.assertIn("'extra_pip_args': {}".format(repr(extra_pip_args)), contents, contents)
36+
all_flags = extra_pip_args + ["--require-hashes", "True"]
37+
self.assertIn("'extra_pip_args': {}".format(repr(all_flags)), contents, contents)
3638

3739

3840
if __name__ == "__main__":

0 commit comments

Comments
 (0)
0