diff --git a/MODULE.bazel b/MODULE.bazel index e1be99374c..f755d2a3dc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,6 +6,10 @@ module( bazel_dep(name = "platforms", version = "0.0.4") +# Those are loaded only when using py_proto_library +bazel_dep(name = "rules_proto", version = "5.3.0-21.7") +bazel_dep(name = "protobuf", repo_name = "com_google_protobuf", version = "21.7") + internal_deps = use_extension("@rules_python//python:extensions.bzl", "internal_deps") internal_deps.install() diff --git a/README.md b/README.md index 53f195672e..a509e28d7e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ ## Overview This repository is the home of the core Python rules -- `py_library`, -`py_binary`, `py_test`, and related symbols that provide the basis for Python +`py_binary`, `py_test`, `py_proto_library`, and related symbols that provide the basis for Python support in Bazel. It also contains package installation rules for integrating with PyPI and other package indices. Documentation lives in the [`docs/`](https://github.com/bazelbuild/rules_python/tree/main/docs) directory and in the diff --git a/internal_deps.bzl b/internal_deps.bzl index a41d5fb34b..942a8720e2 100644 --- a/internal_deps.bzl +++ b/internal_deps.bzl @@ -20,10 +20,10 @@ def rules_python_internal_deps(): http_archive, name = "rules_pkg", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.2.4/rules_pkg-0.2.4.tar.gz", - "https://github.com/bazelbuild/rules_pkg/releases/download/0.2.4/rules_pkg-0.2.4.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz", ], - sha256 = "4ba8f4ab0ff85f2484287ab06c0d871dcb31cc54d439457d28fd4ae14b18450a", + sha256 = "8a298e832762eda1830597d64fe7db58178aa84cd5926d76d5b744d6558941c2", ) maybe( @@ -124,3 +124,24 @@ def rules_python_internal_deps(): strip_prefix = "bazel-integration-testing-165440b2dbda885f8d1ccb8d0f417e6cf8c54f17", sha256 = "2401b1369ef44cc42f91dc94443ef491208dbd06da1e1e10b702d8c189f098e3", ) + + maybe( + http_archive, + name = "rules_proto", + sha256 = "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd", + strip_prefix = "rules_proto-5.3.0-21.7", + urls = [ + "https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.7.tar.gz", + ], + ) + + maybe( + http_archive, + name = "com_google_protobuf", + sha256 = "75be42bd736f4df6d702a0e4e4d30de9ee40eac024c4b845d17ae4cc831fe4ae", + strip_prefix = "protobuf-21.7", + urls = [ + "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz", + "https://github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz", + ], + ) diff --git a/internal_setup.bzl b/internal_setup.bzl index beb26e8a81..2bb181cc19 100644 --- a/internal_setup.bzl +++ b/internal_setup.bzl @@ -17,7 +17,9 @@ load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") load("@build_bazel_integration_testing//tools:repositories.bzl", "bazel_binaries") +load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS") load("//gazelle:deps.bzl", _go_repositories = "gazelle_deps") load("//python/pip_install:repositories.bzl", "pip_install_dependencies") @@ -48,3 +50,8 @@ def rules_python_internal_setup(): go_register_toolchains(version = "1.19.2") gazelle_dependencies() + + rules_proto_dependencies() + rules_proto_toolchains() + + protobuf_deps() diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index c99b040103..75e48bc70a 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -19,7 +19,7 @@ licenses(["notice"]) # Apache 2.0 filegroup( name = "distribution", - srcs = glob(["**"]), + srcs = glob(["**"]) + ["//python/private/proto:distribution"], visibility = ["//python:__pkg__"], ) diff --git a/python/private/proto/BUILD b/python/private/proto/BUILD new file mode 100644 index 0000000000..8483d19c2f --- /dev/null +++ b/python/private/proto/BUILD @@ -0,0 +1,32 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +filegroup( + name = "distribution", + srcs = glob(["**"]), + visibility = ["//python/private:__pkg__"], +) + +proto_lang_toolchain( + name = "python_toolchain", + command_line = "--python_out=%s", + progress_message = "Generating Python proto_library %{label}", + runtime = "@com_google_protobuf//:protobuf_python", +) diff --git a/python/private/proto/py_proto_library.bzl b/python/private/proto/py_proto_library.bzl new file mode 100644 index 0000000000..ef5f2cae70 --- /dev/null +++ b/python/private/proto/py_proto_library.bzl @@ -0,0 +1,191 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The implementation of the `py_proto_library` rule and its aspect.""" + +load("@rules_proto//proto:defs.bzl", "ProtoInfo", "proto_common") +load("//python:defs.bzl", "PyInfo") + +ProtoLangToolchainInfo = proto_common.ProtoLangToolchainInfo + +_PyProtoInfo = provider( + doc = "Encapsulates information needed by the Python proto rules.", + fields = { + "runfiles_from_proto_deps": """ + (depset[File]) Files from the transitive closure implicit proto + dependencies""", + "transitive_sources": """(depset[File]) The Python sources.""", + }, +) + +def _filter_provider(provider, *attrs): + return [dep[provider] for attr in attrs for dep in attr if provider in dep] + +def _py_proto_aspect_impl(target, ctx): + """Generates and compiles Python code for a proto_library. + + The function runs protobuf compiler on the `proto_library` target generating + a .py file for each .proto file. + + Args: + target: (Target) A target providing `ProtoInfo`. Usually this means a + `proto_library` target, but not always; you must expect to visit + non-`proto_library` targets, too. + ctx: (RuleContext) The rule context. + + Returns: + ([_PyProtoInfo]) Providers collecting transitive information about + generated files. + """ + + _proto_library = ctx.rule.attr + + # Check Proto file names + for proto in target[ProtoInfo].direct_sources: + if proto.is_source and "-" in proto.dirname: + fail("Cannot generate Python code for a .proto whose path contains '-' ({}).".format( + proto.path, + )) + + proto_lang_toolchain_info = ctx.attr._aspect_proto_toolchain[ProtoLangToolchainInfo] + api_deps = [proto_lang_toolchain_info.runtime] + + generated_sources = [] + proto_info = target[ProtoInfo] + if proto_info.direct_sources: + # Generate py files + generated_sources = proto_common.declare_generated_files( + actions = ctx.actions, + proto_info = proto_info, + extension = "_pb2.py", + name_mapper = lambda name: name.replace("-", "_").replace(".", "/"), + ) + + proto_common.compile( + actions = ctx.actions, + proto_info = proto_info, + proto_lang_toolchain_info = proto_lang_toolchain_info, + generated_files = generated_sources, + plugin_output = ctx.bin_dir.path, + ) + + # Generated sources == Python sources + python_sources = generated_sources + + deps = _filter_provider(_PyProtoInfo, getattr(_proto_library, "deps", [])) + runfiles_from_proto_deps = depset( + transitive = [dep[DefaultInfo].default_runfiles.files for dep in api_deps] + + [dep.runfiles_from_proto_deps for dep in deps], + ) + transitive_sources = depset( + direct = python_sources, + transitive = [dep.transitive_sources for dep in deps], + ) + + return [ + _PyProtoInfo( + runfiles_from_proto_deps = runfiles_from_proto_deps, + transitive_sources = transitive_sources, + ), + ] + +_py_proto_aspect = aspect( + implementation = _py_proto_aspect_impl, + attrs = { + "_aspect_proto_toolchain": attr.label( + default = ":python_toolchain", + ), + }, + attr_aspects = ["deps"], + required_providers = [ProtoInfo], + provides = [_PyProtoInfo], +) + +def _py_proto_library_rule(ctx): + """Merges results of `py_proto_aspect` in `deps`. + + Args: + ctx: (RuleContext) The rule context. + Returns: + ([PyInfo, DefaultInfo, OutputGroupInfo]) + """ + if not ctx.attr.deps: + fail("'deps' attribute mustn't be empty.") + + pyproto_infos = _filter_provider(_PyProtoInfo, ctx.attr.deps) + default_outputs = depset( + transitive = [info.transitive_sources for info in pyproto_infos], + ) + + return [ + DefaultInfo( + files = default_outputs, + default_runfiles = ctx.runfiles(transitive_files = depset( + transitive = + [default_outputs] + + [info.runfiles_from_proto_deps for info in pyproto_infos], + )), + ), + OutputGroupInfo( + default = depset(), + ), + PyInfo( + transitive_sources = default_outputs, + # Proto always produces 2- and 3- compatible source files + has_py2_only_sources = False, + has_py3_only_sources = False, + ), + ] + +py_proto_library = rule( + implementation = _py_proto_library_rule, + doc = """ + Use `py_proto_library` to generate Python libraries from `.proto` files. + + The convention is to name the `py_proto_library` rule `foo_py_pb2`, + when it is wrapping `proto_library` rule `foo_proto`. + + `deps` must point to a `proto_library` rule. + + Example: + +```starlark +py_library( + name = "lib", + deps = [":foo_py_pb2"], +) + +py_proto_library( + name = "foo_py_pb2", + deps = [":foo_proto"], +) + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], +) +```""", + attrs = { + "deps": attr.label_list( + doc = """ + The list of `proto_library` rules to generate Python libraries for. + + Usually this is just the one target: the proto library of interest. + It can be any target providing `ProtoInfo`.""", + providers = [ProtoInfo], + aspects = [_py_proto_aspect], + ), + }, + provides = [PyInfo], +) diff --git a/python/proto.bzl b/python/proto.bzl new file mode 100644 index 0000000000..3f455aee58 --- /dev/null +++ b/python/proto.bzl @@ -0,0 +1,21 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Python proto library. +""" + +load("//python/private/proto:py_proto_library.bzl", _py_proto_library = "py_proto_library") + +py_proto_library = _py_proto_library