8000 Migrate pip.sh into piptool.py · bazel-contrib/rules_python@ee457a0 · GitHub
[go: up one dir, main page]
Skip to content

Commit ee457a0

Browse files
mattmoormattmoor
committed
Migrate pip.sh into piptool.py
This migrates the logic from pip.sh into piptool.py, which should improve portability by removing the bash dependency. This also has the beginnings of wrapping piptool as a closed redistributable that doesn't rely on a system-installed copy of PIP, but instead uses these rules to pull pip into a PAR bundle. Besides needing to work out the details of releasing and redistributing the PAR, we have two unresolved issues: * When bundled as a PAR (vs. py_binary), piptool seems to pick up the system-installed version of pip. * When bundled as a PAR, piptool sometimes sees cert issues resolving requirements (similar to what we see with httplib2).
1 parent 56be8e8 commit ee457a0

File tree

7 files changed

+203
-99
lines changed

7 files changed

+203
-99
lines changed

‎WORKSPACE

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,27 @@ load("@io_bazel_skydoc//skylark:skylark.bzl", "skydoc_repositories")
3434

3535
skydoc_repositories()
3636

37+
# Requirements for building our piptool.
38+
load("//python:pip.bzl", "pip_import")
39+
40+
pip_import(
41+
name = "piptool_deps",
42+
requirements = "//python:requirements.txt",
43+
)
44+
45+
load(
46+
"@piptool_deps//:requirements.bzl",
47+
_piptool_install = "pip_install",
48+
)
49+
50+
_piptool_install()
51+
52+
git_repository(
53+
name = "subpar",
54+
remote = "https://github.com/google/subpar",
55+
tag = "1.0.0",
56+
)
57+
3758
# Test data for WHL tool testing.
3859
http_file(
3960
name = "grpc_whl",
@@ -63,8 +84,6 @@ http_file(
6384
)
6485

6586
# Imports for examples
66-
load("//python:pip.bzl", "pip_import")
67-
6887
pip_import(
6988
name = "examples_helloworld",
7089
requirements = "//examples/helloworld:requirements.txt",

‎python/BUILD

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ exports_files([
2323
"whl.sh",
2424
])
2525

26-
load(":python.bzl", "py_library", "py_test")
26+
load(":python.bzl", "py_binary", "py_library", "py_test")
2727

2828
py_library(
2929
name = "whl",
@@ -40,3 +40,16 @@ py_test(
4040
],
4141
deps = [":whl"],
4242
)
43+
44+
load("@subpar//:subpar.bzl", "par_binary")
45+
load("@piptool_deps//:requirements.bzl", "all_packages")
46+
47+
# TODO(mattmoor): Bundle this tool as a PAR without any
48+
# system-installed pre-requisites. See TODOs in piptool.py.
49+
par_binary(
50+
name = "piptool",
51+
srcs = ["piptool.py"],
52+
deps = [
53+
":whl",
54+
] + all_packages,
55+
)

‎python/pip.bzl

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,15 @@ def _import_impl(repository_ctx):
1919
# Add an empty top-level BUILD file.
2020
repository_ctx.file("BUILD", "")
2121

22+
# To see the output, pass: quiet=False
2223
result = repository_ctx.execute([
23-
repository_ctx.path(repository_ctx.attr._script),
24-
repository_ctx.attr.name,
25-
repository_ctx.path(repository_ctx.attr.requirements),
26-
repository_ctx.path("requirements.bzl"),
27-
repository_ctx.path(""),
24+
"python", repository_ctx.path(repository_ctx.attr._script),
25+
"--name", repository_ctx.attr.name,
26+
"--input", repository_ctx.path(repository_ctx.attr.requirements),
27+
"--output", repository_ctx.path("requirements.bzl"),
28+
"--directory", repository_ctx.path(""),
2829
])
30+
2931
if result.return_code:
3032
fail("pip_import failed: %s (%s)" % (result.stdout, result.stderr))
3133

@@ -38,7 +40,7 @@ pip_import = repository_rule(
3840
),
3941
"_script": attr.label(
4042
executable = True,
41-
default = Label("//python:pip.sh"),
43+
default = Label("//python:piptool.py"),
4244
cfg = "host",
4345
),
4446
},

‎python/pip.sh

Lines changed: 0 additions & 84 deletions
This file was deleted.

‎python/piptool.py

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
# Copyright 2017 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
"""The piptool module imports pip requirements into Bazel rules."""
15+
16+
import argparse
17+
import json
18+
import os
19+
import pkgutil
20+
import sys
21+
import tempfile
22+
import zipfile
23+
24+
# TODO(mattmoor): When this tool is invoked bundled as a PAR file,
25+
# but not as a py_binary, we get a warning that indicates the system
26+
# installed version of PIP is being picked up instead of our bundled
27+
# version, which should be 9.0.1, e.g.
28+
# You are using pip version 1.5.4, however version 9.0.1 is available.
29+
# You should consider upgrading via the 'pip install --upgrade pip' command.
30+
try:
31+
from pip import main as _pip_main
32+
from pip._vendor import requests
33+
34+
def pip_main(argv):
35+
# Extract the certificates from the PAR following the example of get-pip.py
36+
# https://github.com/pypa/get-pip/blob/430ba37776ae2ad89/template.py#L164-L168
37+
cert_path = os.path.join(tempfile.mkdtemp(), "cacert.pem")
38+
with open(cert_path, "wb") as cert:
39+
cert.write(pkgutil.get_data("pip._vendor.requests", "cacert.pem"))
40+
return _pip_main(argv + ["--cert", os.path.basename(cert_path)])
41+
42+
except:
43+
import subprocess
44+
45+
def pip_main(argv):
46+
return subprocess.call(['pip'] + argv)
47+
48+
# TODO(mattmoor): We can't easily depend on other libraries when
49+
# being invoked as a raw .py file. Once bundled, we should be able
50+
# to remove this fallback on a stub implementation of Wheel.
51+
try:
52+
from python.whl import Wheel
53+
except:
54+
class Wheel(object):
55+
56+
def __init__(self, path):
57+
self._path = path
58+
59+
def basename(self):
60+
return os.path.basename(self._path)
61+
62+
def distribution(self):
63+
# See https://www.python.org/dev/peps/pep-0427/#file-name-convention
64+
parts = self.basename().split('-')
65+
return parts[0]
66+
67+
68+
parser = argparse.ArgumentParser(
69+
description='Import Python dependencies into Bazel.')
70+
71+
parser.add_argument('--name', action='store',
72+
help=('The namespace of the import.'))
73+
74+
parser.add_argument('--input', action='store',
75+
help=('The requirements.txt file to import.'))
76+
77+
parser.add_argument('--output', action='store',
78+
help=('The requirements.bzl file to export.'))
79+
80+
parser.add_argument('--directory', action='store',
81+
help=('The directory into which to put .whl files.'))
82+
83+
84+
def main():
85+
args = parser.parse_args()
86+
87+
# https://github.com/pypa/pip/blob/9.0.1/pip/__init__.py#L209
88+
if pip_main(["wheel", "-w", args.directory, "-r", args.input]):
89+
sys.exit(1)
90+
91+
# Enumerate the .whl files we downloaded.
92+
def list_whls():
93+
dir = args.directory + '/'
94+
for root, unused_dirnames, filenames in os.walk(dir):
95+
for fname in filenames:
96+
if fname.endswith('.whl'):
97+
yield os.path.join(root, fname)
98+
99+
def repo_name(wheel):
100+
return '{repo}_{pkg}'.format(
101+
repo=args.name, pkg=wheel.distribution())
102+
103+
def whl_library(wheel):
104+
# Indentation here matters. whl_library must be within the scope
105+
# of the function below.
106+
return """
107+
whl_library(
108+
name = "{repo_name}",
109+
whl = "@{name}//:{path}",
110+
requirements = "@{name}//:requirements.bzl",
111+
)""".format(name=args.name, repo_name=repo_name(wheel),
112+
path=wheel.basename())
113+
114+
whls = [Wheel(path) for path in list_whls()]
115+
116+
with open(args.output, 'w') as f:
117+
f.write("""\
118+
# Install pip requirements.
119+
#
120+
# Generated from {input}
121+
122+
load("@io_bazel_rules_python//python:whl.bzl", "whl_library")
123+
124+
def pip_install():
125+
{whl_libraries}
126+
127+
_packages = {{
128+
{mappings}
129+
}}
130+
131+
all_packages = _packages.values()
132+
133+
def packages(name):
134+
name = name.replace("-", "_")
135+
return _packages[name]
136+
""".format(input=args.input,
137+
whl_libraries='\n'.join(map(whl_library, whls)),
138+
mappings=','.join([
139+
'"%s": "@%s//:pkg"' % (wheel.distribution(), repo_name(wheel))
140+
for wheel in whls
141+
])))
142+
143+
if __name__ == '__main__':
144+
main()

‎python/requirements.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
pip==9.0.1
2+
wheel==0.30.0a0

‎python/whl.py

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,20 @@ class Wheel(object):
2424
def __init__(self, path):
2525
self._path = path
2626

27-
def _basename(self):
28-
return os.path.basename(self._path)
27+
def path(self):
28+
return self._path
29+
30+
def basename(self):
31+
return os.path.basename(self.path())
2932

3033
def distribution(self):
3134
# See https://www.python.org/dev/peps/pep-0427/#file-name-convention
32-
parts = self._basename().split('-')
35+
parts = self.basename().split('-')
3336
return parts[0]
3437

3538
def version(self):
3639
# See https://www.python.org/dev/peps/pep-0427/#file-name-convention
37-
parts = self._basename().split('-')
40+
parts = self.basename().split('-')
3841
return parts[1]
3942

4043
def _dist_info(self):
@@ -44,7 +47, D7CF 7 @@ def _dist_info(self):
4447
def metadata(self):
4548
# Extract the structured data from metadata.json in the WHL's dist-info
4649
# directory.
47-
with zipfile.ZipFile(self._path, 'r') as whl:
50+
with zipfile.ZipFile(self.path(), 'r') as whl:
4851
with whl.open(os.path.join(self._dist_info(), 'metadata.json')) as f:
4952
return json.loads(f.read())
5053

@@ -59,14 +62,19 @@ def dependencies(self):
5962
# TODO(mattmoor): What's the best way to support "extras"?
6063
# https://packaging.python.org/tutorials/installing-packages/#installing-setuptools-extras
6164
continue
65+
if 'environment' in requirement:
66+
# TODO(mattmoor): What's the best way to support "environment"?
67+
# This typically communicates things like python version (look at
68+
# "wheel" for a good example)
69+
continue
6270
requires = requirement.get('requires', [])
6371
for entry in requires:
6472
# Strip off any trailing versioning data.
6573
parts = entry.split(' ', 1)
6674
yield parts[0]
6775

6876
def expand(self, directory):
69-
with zipfile.ZipFile(self._path, 'r') as whl:
77+
with zipfile.ZipFile(self.path(), 'r') as whl:
7078
whl.extractall(directory)
7179

7280

0 commit comments

Comments
 (0)
0