8000 This adds support for "extras". · psprecher/rules_python@2b6c185 · GitHub 8000
[go: up one dir, main page]
Skip to content

Commit 2b6c185

Browse files
mattmoormattmoor
committed
This adds support for "extras".
"Extras" are additional dependencies of a given library, which are consumed by passing the "extra" name in brackets after the distribution name, for example: ``` mock[docs]==1.0.1 ``` We see this in the dependencies of several Google Cloud libraries, which depend on: `googleapis_common_protos[grpc]` I've added a simple test that the dependency structure we synthesize for this kind of thing is correct via an "extras" test that has a `requirements.txt` of: ``` google-cloud-language==0.27.0 ``` Fixes: bazel-contrib#12
1 parent 72456c9 commit 2b6c185

File tree

12 files changed

+260
-57
lines changed

12 files changed

+260
-57
lines changed

‎README.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,10 @@ for dependencies, however, it is recommended that folks stick with the
106106
`requirement` pattern in case the need arises for us to make changes to this
107107
format in the future.
108108

109+
["Extras"](
110+
https://packaging.python.org/tutorials/installing-packages/#installing-setuptools-extras)
111+
will have a target of the extra name (in place of `pkg` above).
112+
109113
## Updating `docs/`
110114

111115
All of the content (except `BUILD`) under `docs/` is generated. To update the

‎WORKSPACE

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ sass_repositories()
2626

2727
git_repository(
2828
name = "io_bazel_skydoc",
29-
remote = "https://github.com/bazelbuild/skydoc.git",
3029
commit = "e9be81cf5be41e4200749f5d8aa2db7955f8aacc",
30+
remote = "https://github.com/bazelbuild/skydoc.git",
3131
)
3232

3333
load("@io_bazel_skydoc//skylark:skylark.bzl", "skydoc_repositories")
@@ -92,6 +92,15 @@ http_file(
9292
"mock-2.0.0-py2.py3-none-any.whl"),
9393
)
9494

95+
http_file(
96+
name = "google_cloud_language_whl",
97+
sha256 = "a2dd34f0a0ebf5705dcbe34bd41199b1d0a55c4597d38ed045bd183361a561e9",
98+
# From https://pypi.python.org/pypi/google-cloud-language
99+
url = ("https://pypi.python.org/packages/6e/86/" +
100+
"cae57e4802e72d9e626ee5828ed5a646cf4016b473a4a022f1038dba3460/" +
101+
"google_cloud_language-0.29.0-py2.py3-none-any.whl"),
102+
)
103+
95104
# Imports for examples
96105
pip_import(
97106
name = "examples_helloworld",
@@ -128,3 +137,15 @@ load(
128137
)
129138

130139
_boto_install()
140+
141+
pip_import(
142+
name = "examples_extras",
143+
requirements = "//examples/extras:requirements.txt",
144+
)
145+
146+
load(
147+
"@examples_extras//:requirements.bzl",
148+
_extras_install = "pip_install",
149+
)
150+
151+
_extras_install()

‎examples/extras/BUILD

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Copyright 2017 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
package(default_visibility = ["//visibility:public"])
15+
16+
licenses(["notice"]) # Apache 2.0
17+
18+
load("@examples_extras//:requirements.bzl", "requirement")
19+
load("//python:python.bzl", "py_test")
20+
21+
py_test(
22+
name = "extras_test",
23+
srcs = ["extras_test.py"],
24+
deps = [
25+
requirement("google-cloud-language"),
26+
# Make sure that we can resolve the "extra" dependency
27+
requirement("googleapis-common-protos[grpc]"),
28+
],
29+
)

‎examples/extras/extras_test.py

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Copyright 2017 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import unittest
16+
17+
18+
# The test is the build itself, which should not work if extras are missing.
19+
class ExtrasTest(unittest.TestCase):
20+
def test_nothing(self):
21+
pass
22+
23+
24+
if __name__ == '__main__':
25+
unittest.main()

‎examples/extras/requirements.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
google-cloud-language==0.27.0

‎examples/extras/version_test.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Copyright 2017 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import pip
16+
import unittest
17+
18+
19+
class VersionTest(unittest.TestCase):
20+
21+
def test_version(self):
22+
self.assertEqual(pip.__version__, '9.0.1')
23+
24+
25+
if __name__ == '__main__':
26+
unittest.main()

‎python/whl.bzl

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,17 @@
1616
def _whl_impl(repository_ctx):
1717
"""Core implementation of whl_library."""
1818

19-
result = repository_ctx.execute([
19+
args = [
2020
"python",
2121
repository_ctx.path(repository_ctx.attr._script),
2222
"--whl", repository_ctx.path(repository_ctx.attr.whl),
2323
"--requirements", repository_ctx.attr.requirements,
24-
])
24+
]
25+
26+
if repository_ctx.attr.extras:
27+
args += ["--extras", ",".join(repository_ctx.attr.extras)]
28+
29+
result = repository_ctx.execute(args)
2530
if result.return_code:
2631
fail("whl_library failed: %s (%s)" % (result.stdout, result.stderr))
2732

@@ -33,6 +38,7 @@ whl_library = repository_rule(
3338
single_file = True,
3439
),
3540
"requirements": attr.string(),
41+
"extras": attr.string_list(),
3642
"_script": attr.label(
3743
executable = True,
3844
default = Label("//rules_python:whl.py"),
@@ -64,4 +70,7 @@ Args:
6470
6571
requirements: The name of the pip_import repository rule from which to
6672
load this .whl's dependencies.
73+
74+
extras: A subset of the "extras" available from this <code>.whl</code> for which
75+
<code>requirements</code> has the dependencies.
6776
"""

‎rules_python/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ py_test(
2828
data = [
2929
"@futures_3_1_1_whl//file",
3030
"@futures_2_2_0_whl//file",
31+
"@google_cloud_language_whl//file",
3132
"@grpc_whl//file",
3233
"@mock_whl//file",
3334
],

‎rules_python/piptool.py

Lines changed: 77 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import json
1919
import os
2020
import pkgutil
21+
import pkg_resources
2122
import re
2223
import shutil
2324
import sys
@@ -78,36 +79,7 @@ def pip_main(argv):
7879
argv = ["--disable-pip-version-check", "--cert", cert_path] + argv
7980
return pip.main(argv)
8081

81-
82-
# TODO(mattmoor): We can't easily depend on other libraries when
83-
# being invoked as a raw .py file. Once bundled, we should be able
84-
# to remove this fallback on a stub implementation of Wheel.
85-
try:
86-
from rules_python.whl import Wheel
87-
except:
88-
class Wheel(object):
89-
90-
def __init__(self, path):
91-
self._path = path
92-
93-
def basename(self):
94-
return os.path.basename(self._path)
95-
96-
def distribution(self):
97-
# See https://www.python.org/dev/peps/pep-0427/#file-name-convention
98-
parts = self.basename().split('-')
99-
return parts[0]
100-
101-
def version(self):
102-
# See https://www.python.org/dev/peps/pep-0427/#file-name-convention
103-
parts = self.basename().split('-')
104-
return parts[1]
105-
106-
def repository_name(self):
107-
# Returns the canonical name of the Bazel repository for this package.
108-
canonical = 'pypi__{}_{}'.format(self.distribution(), self.version())
109-
# Escape any illegal characters with underscore.
110-
return re.sub('[-.]', '_', canonical)
82+
from rules_python.whl import Wheel
11183

11284
parser = argparse.ArgumentParser(
11385
description='Import Python dependencies into Bazel.')
@@ -124,6 +96,59 @@ def repository_name(self):
12496
parser.add_argument('--directory', action='store',
12597
help=('The directory into which to put .whl files.'))
12698

99+
def determine_possible_extras(whls):
100+
"""Determines the list of possible "extras" for each .whl
101+
102+
The possibility of an extra is determined by looking at its
103+
additional requirements, and determinine whether they are
104+
satisfied by the complete list of available wheels.
105+
106+
Args:
107+
whls: a list of Wheel objects
108+
109+
Returns:
110+
a dict that is keyed by the Wheel objects in whls, and whose
111+
values are lists of possible extras.
112+
"""
113+
whl_map = {
114+
whl.distribution(): whl
115+
for whl in whls
116+
}
117+
118+
# TODO(mattmoor): Consider memoizing if this recursion ever becomes
119+
# expensive enough to warrant it.
120+
def is_possible(distro, extra):
121+
distro = distro.replace("-", "_")
122+
# If we don't have the .whl at all, then this isn't possible.
123+
if distro not in whl_map:
124+
return False
125+
whl = whl_map[distro]
126+
# If we have the .whl, and we don't need anything extra then
127+
# we can satisfy this dependency.
128+
if not extra:
129+
return True
130+
# If we do need something extra, then check the extra's
131+
# dependencies to make sure they are fully satisfied.
132+
for extra_dep in whl.dependencies(extra=extra):
133+
req = pkg_resources.Requirement.parse(extra_dep)
134+
# Check that the dep and any extras are all possible.
135+
if not is_possible(req.project_name, None):
136+
return False
137+
for e in req.extras:
138+
if not is_possible(req.project_name, e):
139+
return False
140+
# If all of the dependencies of the extra are satisfiable then
141+
# it is possible to construct this dependency.
142+
return True
143+
144+
return {
145+
whl: [
146+
extra
147+
for extra in whl.extras()
148+
if is_possible(whl.distribution(), extra)
149+
]
150+
for whl in whls
151+
}
127152

128153
def main():
129154
args = parser.parse_args()
@@ -140,6 +165,9 @@ def list_whls():
140165
if fname.endswith('.whl'):
141166
yield os.path.join(root, fname)
142167

168+
whls = [Wheel(path) for path in list_whls()]
169+
possible_extras = determine_possible_extras(whls)
170+
143171
def whl_library(wheel):
144172
# Indentation here matters. whl_library must be within the scope
145173
# of the function below. We also avoid reimporting an existing WHL.
@@ -149,10 +177,25 @@ def whl_library(wheel):
149177
name = "{repo_name}",
150178
whl = "@{name}//:{path}",
151179
requirements = "@{name}//:requirements.bzl",
180+
extras = [{extras}]
152181
)""".format(name=args.name, repo_name=wheel.repository_name(),
153-
path=wheel.basename())
154-
155-
whls = [Wheel(path) for path in list_whls()]
182+
path=wheel.basename(),
183+
extras=','.join([
184+
'"%s"' % extra
185+
for extra in possible_extras.get(wheel, [])
186+
]))
187+
188+
whl_targets = ','.join([
189+
','.join([
190+
'"%s": "@%s//:pkg"' % (whl.distribution().lower(), whl.repository_name())
191+
] + [
192+
# For every extra that is possible from this requirements.txt
193+
'"%s[%s]": "@%s//:%s"' % (whl.distribution().lower(), extra.lower(),
194+
whl.repository_name(), extra)
195+
for extra in possible_extras.get(whl, [])
196+
])
197+
for whl in whls
198+
])
156199

157200
with open(args.output, 'w') as f:
158201
f.write("""\
@@ -178,10 +221,7 @@ def requirement(name):
178221
return _requirements[name_key]
179222
""".format(input=args.input,
180223
whl_libraries='\n'.join(map(whl_library, whls)) if whls else "pass",
181-
mappings=','.join([
182-
'"%s": "@%s//:pkg"' % (wheel.distribution().lower(), wheel.repository_name())
183-
for wheel in whls
184-
])))
224+
mappings=whl_targets))
185225

186226
if __name__ == '__main__':
187227
main()

0 commit comments

Comments
 (0)
0