From 4835fcc7d11d832938f638b8af391dbab045531e Mon Sep 17 00:00:00 2001 From: Ryan Beasley Date: Wed, 27 Mar 2019 11:21:49 -0400 Subject: [PATCH 1/2] Add py_import rule This change adds adds a `py_import` rule to import Python eggs like `java_import` imports Java jars. py_import.egg generated using `zipper`[1]: ```console $ third_party/ijar/zipper Cc examples/py_import/py_import.egg examples/py_import/helloworld.py=examples/helloworld/helloworld.py examples/__init__.py= examples/py_import/__init__.py= ``` Partially addresses bazelbuild/bazel#7312. Addresses #222. [1]: https://github.com/bazelbuild/bazel/tree/master/third_party/ijar Testing Done: ```console $ bazelisk test --override_repository=rules_python=$PWD/../.. ... //:py_import_test PASSED in 0.6s ``` --- .bazelrc | 4 +-- README.md | 6 ++-- examples/py_import/BUILD | 33 +++++++++++++++++++++ examples/py_import/WORKSPACE | 23 +++++++++++++++ examples/py_import/py_import.egg | Bin 0 -> 949 bytes examples/py_import/py_import_test.py | 41 +++++++++++++++++++++++++++ examples/py_import/requirements.txt | 1 + python/defs.bzl | 41 +++++++++++++++++++++++++++ 8 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 examples/py_import/BUILD create mode 100644 examples/py_import/WORKSPACE create mode 100644 examples/py_import/py_import.egg create mode 100644 examples/py_import/py_import_test.py create mode 100644 examples/py_import/requirements.txt diff --git a/.bazelrc b/.bazelrc index ddba1f3b1a..c428dd68df 100644 --- a/.bazelrc +++ b/.bazelrc @@ -3,7 +3,7 @@ # This lets us glob() up all the files inside the examples to make them inputs to tests # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh -build --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse -query --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse +build --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import,examples/wheel,examples/wheel/lib +query --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import,examples/wheel,examples/wheel/lib test --test_output=errors diff --git a/README.md b/README.md index f0f74dbb1d..c166574b6c 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ ## Overview This repository is the home of the core Python rules -- `py_library`, -`py_binary`, `py_test`, and related symbols that provide the basis for Python -support in Bazel. It also contains packaging rules for integrating with PyPI -(`pip`). Documentation lives in the +`py_binary`, `py_test`, `py_import`, and related symbols that provide the +basis for Python support in Bazel. It also contains packaging rules for +integrating with PyPI (`pip`). Documentation lives in the [`docs/`](https://github.com/bazelbuild/rules_python/tree/master/docs) directory and in the [Bazel Build Encyclopedia](https://docs.bazel.build/versions/master/be/python.html). diff --git a/examples/py_import/BUILD b/examples/py_import/BUILD new file mode 100644 index 0000000000..9f128b0161 --- /dev/null +++ b/examples/py_import/BUILD @@ -0,0 +1,33 @@ +# Copyright 2020 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@examples_py_import//:requirements.bzl", "requirement") +load("@rules_python//python:defs.bzl", "py_import", "py_test") + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +py_import( + name = "py_import", + srcs = ["py_import.egg"], + deps = [requirement("futures")], +) + +py_test( + name = "py_import_test", + srcs = ["py_import_test.py"], + python_version = "PY2", + deps = [":py_import"], +) diff --git a/examples/py_import/WORKSPACE b/examples/py_import/WORKSPACE new file mode 100644 index 0000000000..22cf85284a --- /dev/null +++ b/examples/py_import/WORKSPACE @@ -0,0 +1,23 @@ +workspace(name = "py_import") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +# XXX 0.0.3 doesn't include py_import. Need to bump to a later release. +http_archive( + name = "rules_python", + sha256 = "e46612e9bb0dae8745de6a0643be69e8665a03f63163ac6610c210e80d14c3e4", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.3/rules_python-0.0.3.tar.gz", +) + +load("@rules_python//python:pip.bzl", "pip_import", "pip_repositories") + +pip_repositories() + +pip_import( + name = "examples_py_import", + requirements = "//:requirements.txt", +) + +load("@examples_py_import//:requirements.bzl", "pip_install") + +pip_install() diff --git a/examples/py_import/py_import.egg b/examples/py_import/py_import.egg new file mode 100644 index 0000000000000000000000000000000000000000..f41f396fe90e3cdc86d2849bc84886ca4c7361ea GIT binary patch literal 949 zcmWIWW@h1HU|`??Vnv&8tM~=QnHU%rF*7hI07X(O5_1c3Qj7HqD&sSA3-XIf^fOX( za`Mabi*i!*3My+SUG%?gAaLyd=kPhFytPX;ooA^TeiT?Bd@;a!qQQ~ir&iOC%+9X< zR+;(z`un6VHYX+h&hz(c|J6R9b6{#I`}$q;9=!STIDJKRi}y!yQ;~^Pubuf?uT2nP zv|;;Fzv1NO$!}Qlrn(4kXv~oM;&3O*Lh)zmotcOEzc@xRr~lOSaX`L1x3dfM6)w49 zv_y`y#D8ty2s zdy?*9>SuUYCZuCML)32;5gD(mS|@k8Rjdt)^3P158nN^b5h!KNrz_L9J5mz zw{n&-md;+q@Hl zSmGT2%W6H3ulh>9&a_W%@h#00-yeLUm{+OhImy2K?{>x4_uju#HZ3!_?5Oc5VQas| z>3@s?&_o4HLBK=>B4I!T$bcuR`1s7c%#!$cU_!#JLk?L7A{nCSWn^Mt#+`^^b}wlJ zvEa5N%tA@h0p2jvK&mla4-y4|ZH%G7i~=_et9wA|P)z{&1_rha1V-EtZX3l P%s_YtNFN63U|;|M5)fT0 literal 0 HcmV?d00001 diff --git a/examples/py_import/py_import_test.py b/examples/py_import/py_import_test.py new file mode 100644 index 0000000000..1d7212dc3e --- /dev/null +++ b/examples/py_import/py_import_test.py @@ -0,0 +1,41 @@ +# Copyright 2017-2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from examples.py_import import helloworld + + +class HelloWorldTest(unittest.TestCase): + + def test_helloworld(self): + hw = helloworld.HelloWorld() + hw.SayHello() + + def test_helloworld_async(self): + hw = helloworld.HelloWorld() + hw.SayHelloAsync() + hw.Stop() + + def test_helloworld_multiple(self): + hw = helloworld.HelloWorld() + hw.SayHelloAsync() + hw.SayHelloAsync() + hw.SayHelloAsync() + hw.SayHelloAsync() + hw.Stop() + + +if __name__ == '__main__': + unittest.main() diff --git a/examples/py_import/requirements.txt b/examples/py_import/requirements.txt new file mode 100644 index 0000000000..372420dc2a --- /dev/null +++ b/examples/py_import/requirements.txt @@ -0,0 +1 @@ +futures>=3.1 diff --git a/python/defs.bzl b/python/defs.bzl index 890c221e72..335ee650f6 100644 --- a/python/defs.bzl +++ b/python/defs.bzl @@ -71,6 +71,47 @@ def py_test(**attrs): # buildifier: disable=native-python native.py_test(**_add_tags(attrs)) +def _py_import_impl(ctx): + # See https://github.com/bazelbuild/bazel/blob/0.24.0/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java#L104 . + import_paths = [ + "/".join([ctx.workspace_name, x.short_path]) + for x in ctx.files.srcs + ] + + return [ + DefaultInfo( + default_runfiles = ctx.runfiles(ctx.files.srcs, collect_default = True), + ), + PyInfo( + transitive_sources = depset(transitive = [ + dep[PyInfo].transitive_sources + for dep in ctx.attr.deps + ]), + imports = depset(direct = import_paths, transitive = [ + dep[PyInfo].imports + for dep in ctx.attr.deps + ]), + ), + ] + +py_import = rule( + doc = "This rule allows the use of Python eggs as libraries for " + + "`py_library` and `py_binary` rules.", + implementation = _py_import_impl, + attrs = { + "deps": attr.label_list( + doc = "The list of other libraries to be linked in to the " + + "binary target.", + providers = [PyInfo], + ), + "srcs": attr.label_list( + doc = "The list of Python eggs provided to Python targets " + + "that depend on this target.", + allow_files = [".egg"], + ), + }, +) + def py_runtime(**attrs): """See the Bazel core [py_runtime](https://docs.bazel.build/versions/master/be/python.html#py_runtime) documentation. From a4ba2726d460e021630bf77cbd6232c873843be0 Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Wed, 21 Apr 2021 13:17:52 -0700 Subject: [PATCH 2/2] code review cleanups --- .bazelrc | 4 ++-- README.md | 6 +++--- examples/BUILD | 5 +++++ examples/py_import/BUILD | 13 +++++++++--- examples/py_import/WORKSPACE | 20 ++++++------------ .../{py_import.egg => helloworld.egg} | Bin examples/py_import/some_library.egg | Bin 0 -> 949 bytes python/defs.bzl | 17 +++++++++++---- .../bazel_integration_test.bzl | 4 ++-- 9 files changed, 41 insertions(+), 28 deletions(-) rename examples/py_import/{py_import.egg => helloworld.egg} (100%) create mode 100644 examples/py_import/some_library.egg diff --git a/.bazelrc b/.bazelrc index c428dd68df..6aee0a663d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -3,7 +3,7 @@ # This lets us glob() up all the files inside the examples to make them inputs to tests # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh -build --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import,examples/wheel,examples/wheel/lib -query --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import,examples/wheel,examples/wheel/lib +build --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import +query --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import test --test_output=errors diff --git a/README.md b/README.md index c166574b6c..f0f74dbb1d 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ ## Overview This repository is the home of the core Python rules -- `py_library`, -`py_binary`, `py_test`, `py_import`, and related symbols that provide the -basis for Python support in Bazel. It also contains packaging rules for -integrating with PyPI (`pip`). Documentation lives in the +`py_binary`, `py_test`, and related symbols that provide the basis for Python +support in Bazel. It also contains packaging rules for integrating with PyPI +(`pip`). Documentation lives in the [`docs/`](https://github.com/bazelbuild/rules_python/tree/master/docs) directory and in the [Bazel Build Encyclopedia](https://docs.bazel.build/versions/master/be/python.html). diff --git a/examples/BUILD b/examples/BUILD index 5b798d53a7..e263c07368 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -31,3 +31,8 @@ bazel_integration_test( name = "pip_parse_example", timeout = "long", ) + +bazel_integration_test( + name = "py_import_example", + timeout = "long", +) diff --git a/examples/py_import/BUILD b/examples/py_import/BUILD index 9f128b0161..30ab5845a8 100644 --- a/examples/py_import/BUILD +++ b/examples/py_import/BUILD @@ -12,22 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@examples_py_import//:requirements.bzl", "requirement") +load("@pip//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_import", "py_test") package(default_visibility = ["//visibility:public"]) licenses(["notice"]) # Apache 2.0 +# Adapt helloworld.egg so it can be depended on as if it were a +# py_library target py_import( name = "py_import", - srcs = ["py_import.egg"], + # Note: this .egg file can be regenerated using zipper: + # $ third_party/ijar/zipper Cc \ + # examples/py_import/helloworld.egg \ + # examples/py_import/helloworld.py=examples/legacy_pip_import/helloworld/helloworld.py \ + # examples/__init__.py= \ + # examples/py_import/__init__.py= + srcs = ["helloworld.egg"], deps = [requirement("futures")], ) py_test( name = "py_import_test", srcs = ["py_import_test.py"], - python_version = "PY2", deps = [":py_import"], ) diff --git a/examples/py_import/WORKSPACE b/examples/py_import/WORKSPACE index 22cf85284a..78aba2e04d 100644 --- a/examples/py_import/WORKSPACE +++ b/examples/py_import/WORKSPACE @@ -2,22 +2,14 @@ workspace(name = "py_import") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -# XXX 0.0.3 doesn't include py_import. Need to bump to a later release. +# Note, this relies on a pre-release of 0.3.0, and doesn't actually work with 0.2.0 +# Use --override_repository=rules_python=$HOME/Projects/rules_python to test for now http_archive( name = "rules_python", - sha256 = "e46612e9bb0dae8745de6a0643be69e8665a03f63163ac6610c210e80d14c3e4", - url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.3/rules_python-0.0.3.tar.gz", + sha256 = "778197e26c5fbeb07ac2a2c5ae405b30f6cb7ad1f5510ea6fdac03bded96cc6f", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.2.0/rules_python-0.2.0.tar.gz", ) -load("@rules_python//python:pip.bzl", "pip_import", "pip_repositories") +load("@rules_python//python:pip.bzl", "pip_install") -pip_repositories() - -pip_import( - name = "examples_py_import", - requirements = "//:requirements.txt", -) - -load("@examples_py_import//:requirements.bzl", "pip_install") - -pip_install() +pip_install(requirements = "//:requirements.txt") diff --git a/examples/py_import/py_import.egg b/examples/py_import/helloworld.egg similarity index 100% rename from examples/py_import/py_import.egg rename to examples/py_import/helloworld.egg diff --git a/examples/py_import/some_library.egg b/examples/py_import/some_library.egg new file mode 100644 index 0000000000000000000000000000000000000000..f41f396fe90e3cdc86d2849bc84886ca4c7361ea GIT binary patch literal 949 zcmWIWW@h1HU|`??Vnv&8tM~=QnHU%rF*7hI07X(O5_1c3Qj7HqD&sSA3-XIf^fOX( za`Mabi*i!*3My+SUG%?gAaLyd=kPhFytPX;ooA^TeiT?Bd@;a!qQQ~ir&iOC%+9X< zR+;(z`un6VHYX+h&hz(c|J6R9b6{#I`}$q;9=!STIDJKRi}y!yQ;~^Pubuf?uT2nP zv|;;Fzv1NO$!}Qlrn(4kXv~oM;&3O*Lh)zmotcOEzc@xRr~lOSaX`L1x3dfM6)w49 zv_y`y#D8ty2s zdy?*9>SuUYCZuCML)32;5gD(mS|@k8Rjdt)^3P158nN^b5h!KNrz_L9J5mz zw{n&-md;+q@Hl zSmGT2%W6H3ulh>9&a_W%@h#00-yeLUm{+OhImy2K?{>x4_uju#HZ3!_?5Oc5VQas| z>3@s?&_o4HLBK=>B4I!T$bcuR`1s7c%#!$cU_!#JLk?L7A{nCSWn^Mt#+`^^b}wlJ zvEa5N%tA@h0p2jvK&mla4-y4|ZH%G7i~=_et9wA|P)z{&1_rha1V-EtZX3l P%s_YtNFN63U|;|M5)fT0 literal 0 HcmV?d00001 diff --git a/python/defs.bzl b/python/defs.bzl index 335ee650f6..28073d4716 100644 --- a/python/defs.bzl +++ b/python/defs.bzl @@ -95,8 +95,15 @@ def _py_import_impl(ctx): ] py_import = rule( - doc = "This rule allows the use of Python eggs as libraries for " + - "`py_library` and `py_binary` rules.", + doc = """This rule allows the use of Python packages as dependencies. + + It imports the given `.egg` file(s), which might be checked in source files, + fetched externally as with `http_file`, or produced as outputs of other rules. + + It may be used like a `py_library`, in the `deps` of other Python rules. + + This is similar to [java_import](https://docs.bazel.build/versions/master/be/java.html#java_import). + """, implementation = _py_import_impl, attrs = { "deps": attr.label_list( @@ -105,8 +112,10 @@ py_import = rule( providers = [PyInfo], ), "srcs": attr.label_list( - doc = "The list of Python eggs provided to Python targets " + - "that depend on this target.", + doc = "The list of Python package files provided to Python targets " + + "that depend on this target. Note that currently only the .egg " + + "format is accepted. For .whl files, try the whl_library rule. " + + "We accept contributions to extend py_import to handle .whl.", allow_files = [".egg"], ), }, diff --git a/tools/bazel_integration_test/bazel_integration_test.bzl b/tools/bazel_integration_test/bazel_integration_test.bzl index caac0d9343..7fcace650b 100644 --- a/tools/bazel_integration_test/bazel_integration_test.bzl +++ b/tools/bazel_integration_test/bazel_integration_test.bzl @@ -13,8 +13,8 @@ _ATTRS = { It is assumed by the test runner that the bazel binary is found at label_workspace/bazel (wksp/bazel.exe on Windows)""", ), "bazel_commands": attr.string_list( - default = ["info", "test ..."], - doc = """The list of bazel commands to run. Defaults to `["info", "test ..."]`. + default = ["info", "test --test_output=errors ..."], + doc = """The list of bazel commands to run. Note that if a command contains a bare `--` argument, the --test_arg passed to Bazel will appear before it. """,