From efca34a21ca9259e7466ba1e85f5b95fdcda0261 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 2 Jan 2023 15:55:48 +0900 Subject: [PATCH 1/8] Start with a subset of the build_file_generation example --- .../bzlmod_build_file_generation/.bazelrc | 7 ++ .../.bazelversion | 1 + .../bzlmod_build_file_generation/.gitignore | 1 + .../bzlmod_build_file_generation/BUILD.bazel | 59 +++++++++ .../bzlmod_build_file_generation/MODULE.bazel | 35 ++++++ .../bzlmod_build_file_generation/README.md | 20 +++ .../bzlmod_build_file_generation/WORKSPACE | 2 + .../bzlmod_build_file_generation/__init__.py | 30 +++++ .../bzlmod_build_file_generation/__main__.py | 18 +++ .../bzlmod_build_file_generation/__test__.py | 31 +++++ .../gazelle_python.yaml | 117 ++++++++++++++++++ .../random_number_generator/BUILD.bazel | 19 +++ .../random_number_generator/__init__.py | 0 .../random_number_generator/__test__.py | 28 +++++ .../generate_random_number.py | 21 ++++ .../requirements.in | 1 + .../requirements_lock.txt | 78 ++++++++++++ .../requirements_windows.txt | 82 ++++++++++++ 18 files changed, 550 insertions(+) create mode 100644 examples/bzlmod_build_file_generation/.bazelrc create mode 100644 examples/bzlmod_build_file_generation/.bazelversion create mode 100644 examples/bzlmod_build_file_generation/.gitignore create mode 100644 examples/bzlmod_build_file_generation/BUILD.bazel create mode 100644 examples/bzlmod_build_file_generation/MODULE.bazel create mode 100644 examples/bzlmod_build_file_generation/README.md create mode 100644 examples/bzlmod_build_file_generation/WORKSPACE create mode 100644 examples/bzlmod_build_file_generation/__init__.py create mode 100644 examples/bzlmod_build_file_generation/__main__.py create mode 100644 examples/bzlmod_build_file_generation/__test__.py create mode 100644 examples/bzlmod_build_file_generation/gazelle_python.yaml create mode 100644 examples/bzlmod_build_file_generation/random_number_generator/BUILD.bazel create mode 100644 examples/bzlmod_build_file_generation/random_number_generator/__init__.py create mode 100644 examples/bzlmod_build_file_generation/random_number_generator/__test__.py create mode 100644 examples/bzlmod_build_file_generation/random_number_generator/generate_random_number.py create mode 100644 examples/bzlmod_build_file_generation/requirements.in create mode 100644 examples/bzlmod_build_file_generation/requirements_lock.txt create mode 100644 examples/bzlmod_build_file_generation/requirements_windows.txt diff --git a/examples/bzlmod_build_file_generation/.bazelrc b/examples/bzlmod_build_file_generation/.bazelrc new file mode 100644 index 0000000000..ef799026d6 --- /dev/null +++ b/examples/bzlmod_build_file_generation/.bazelrc @@ -0,0 +1,7 @@ +common --experimental_enable_bzlmod + +test --test_output=errors + +# Windows requires these for multi-python support: +build --enable_runfiles +startup --windows_enable_symlinks diff --git a/examples/bzlmod_build_file_generation/.bazelversion b/examples/bzlmod_build_file_generation/.bazelversion new file mode 100644 index 0000000000..09b254e90c --- /dev/null +++ b/examples/bzlmod_build_file_generation/.bazelversion @@ -0,0 +1 @@ +6.0.0 diff --git a/examples/bzlmod_build_file_generation/.gitignore b/examples/bzlmod_build_file_generation/.gitignore new file mode 100644 index 0000000000..ac51a054d2 --- /dev/null +++ b/examples/bzlmod_build_file_generation/.gitignore @@ -0,0 +1 @@ +bazel-* diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel new file mode 100644 index 0000000000..8e170d1344 --- /dev/null +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -0,0 +1,59 @@ +# Load various rules so that we can have bazel download +# various rulesets and dependencies. +# The `load` statement imports the symbol for the rule, in the defined +# ruleset. When the symbol is loaded you can use the rule. +load("@pip//:requirements.bzl", "requirement") +load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") +load("@rules_python//python:pip.bzl", "compile_pip_requirements") + +compile_pip_requirements( + name = "requirements", + extra_args = ["--allow-unsafe"], + requirements_in = "requirements.in", + requirements_txt = "requirements_lock.txt", + requirements_windows = "requirements_windows.txt", +) + +# This rule is auto-generated and managed by Gazelle, +# because it found the __init__.py file in this folder. +# See: https://bazel.build/reference/be/python#py_library +py_library( + name = "bzlmod_build_file_generation", + srcs = ["__init__.py"], + visibility = ["//:__subpackages__"], + deps = [ + "//random_number_generator", + requirement("flask"), + ], +) + +# A py_binary is an executable Python program consisting of a collection of .py source files. +# See: https://bazel.build/reference/be/python#py_binary +# +# This rule is auto-generated and managed by Gazelle, +# because it found the __main__.py file in this folder. +# This rule creates a target named //:bzlmod_build_file_generation_bin and you can use +# bazel to run the target: +# `bazel run //:bzlmod_build_file_generation_bin` +py_binary( + name = "bzlmod_build_file_generation_bin", + srcs = ["__main__.py"], + main = "__main__.py", + visibility = ["//:__subpackages__"], + deps = [":bzlmod_build_file_generation"], +) + +# A py_test is a Python unit test. +# See: https://bazel.build/reference/be/python#py_test +# +# This rule is auto-generated and managed by Gazelle, +# because it found the __test__.py file in this folder. +# This rule creates a target named //:bzlmod_build_file_generation_test and you can use +# bazel to run the target: +# `bazel test //:bzlmod_build_file_generation_test` +py_test( + name = "bzlmod_build_file_generation_test", + srcs = ["__test__.py"], + main = "__test__.py", + deps = [":bzlmod_build_file_generation"], +) diff --git a/examples/bzlmod_build_file_generation/MODULE.bazel b/examples/bzlmod_build_file_generation/MODULE.bazel new file mode 100644 index 0000000000..14a338cc52 --- /dev/null +++ b/examples/bzlmod_build_file_generation/MODULE.bazel @@ -0,0 +1,35 @@ +module( + name = "example_bzlmod", + compatibility_level = 1, + version = "0.0.0", +) + +bazel_dep(name = "rules_python", version = "0.0.0") + +local_path_override( + module_name = "rules_python", + path = "../..", +) + +python = use_extension("@rules_python//python:extensions.bzl", "python") + +python.toolchain( + name = "python3_9", + python_version = "3.9", +) + +use_repo(python, "python3_9_toolchains") + +register_toolchains( + "@python3_9_toolchains//:all", +) + +pip = use_extension("@rules_python//python:extensions.bzl", "pip") + +pip.parse( + name = "pip", + requirements_lock = "//:requirements_lock.txt", + requirements_windows = "//:requirements_windows.txt", +) + +use_repo(pip, "pip") diff --git a/examples/bzlmod_build_file_generation/README.md b/examples/bzlmod_build_file_generation/README.md new file mode 100644 index 0000000000..9b2fe1a7be --- /dev/null +++ b/examples/bzlmod_build_file_generation/README.md @@ -0,0 +1,20 @@ +# Build file generation with Gazelle + +This example shows a project that has Gazelle setup with the rules_python +extension, so that targets like `py_library` and `py_binary` can be +automatically created just by running + +```sh +$ bazel run //:gazelle +``` + +As a demo, try creating a `__main__.py` file in this directory, then +re-run that gazelle command. You'll see that a `py_binary` target +is created in the `BUILD` file. + +Or, try importing the `requests` library in `__init__.py`. +You'll see that `deps = ["@pip//pypi__requests"]` is automatically +added to the `py_library` target in the `BUILD` file. + +For more information on the behavior of the rules_python gazelle extension, +see the README.md file in the /gazelle folder. diff --git a/examples/bzlmod_build_file_generation/WORKSPACE b/examples/bzlmod_build_file_generation/WORKSPACE new file mode 100644 index 0000000000..78cc252e57 --- /dev/null +++ b/examples/bzlmod_build_file_generation/WORKSPACE @@ -0,0 +1,2 @@ +# Empty file indicating the root of a Bazel workspace. +# Dependencies and setup are in MODULE.bazel. diff --git a/examples/bzlmod_build_file_generation/__init__.py b/examples/bzlmod_build_file_generation/__init__.py new file mode 100644 index 0000000000..7b873cfa9f --- /dev/null +++ b/examples/bzlmod_build_file_generation/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from flask import Flask, jsonify +from random_number_generator import generate_random_number + +app = Flask(__name__) + + +@app.route("/random-number", methods=["GET"]) +def get_random_number(): + return jsonify({"number": generate_random_number.generate_random_number()}) + + +"""Start the python web server""" + + +def main(): + app.run() diff --git a/examples/bzlmod_build_file_generation/__main__.py b/examples/bzlmod_build_file_generation/__main__.py new file mode 100644 index 0000000000..a77055f2d5 --- /dev/null +++ b/examples/bzlmod_build_file_generation/__main__.py @@ -0,0 +1,18 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __init__ import main + +if __name__ == "__main__": + main() diff --git a/examples/bzlmod_build_file_generation/__test__.py b/examples/bzlmod_build_file_generation/__test__.py new file mode 100644 index 0000000000..45e127bab8 --- /dev/null +++ b/examples/bzlmod_build_file_generation/__test__.py @@ -0,0 +1,31 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from __init__ import app + + +class TestServer(unittest.TestCase): + def setUp(self): + self.app = app.test_client() + + def test_get_random_number(self): + response = self.app.get("/random-number") + self.assertEqual(response.status_code, 200) + self.assertIn("number", response.json) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/bzlmod_build_file_generation/gazelle_python.yaml b/examples/bzlmod_build_file_generation/gazelle_python.yaml new file mode 100644 index 0000000000..0be959a67e --- /dev/null +++ b/examples/bzlmod_build_file_generation/gazelle_python.yaml @@ -0,0 +1,117 @@ +# GENERATED FILE - DO NOT EDIT! +# +# To update this file, run: +# bazel run //:gazelle_python_manifest.update + +manifest: + modules_mapping: + click: click + click.core: click + click.decorators: click + click.exceptions: click + click.formatting: click + click.globals: click + click.parser: click + click.shell_completion: click + click.termui: click + click.testing: click + click.types: click + click.utils: click + flask: Flask + flask.app: Flask + flask.blueprints: Flask + flask.cli: Flask + flask.config: Flask + flask.ctx: Flask + flask.debughelpers: Flask + flask.globals: Flask + flask.helpers: Flask + flask.json: Flask + flask.json.provider: Flask + flask.json.tag: Flask + flask.logging: Flask + flask.scaffold: Flask + flask.sessions: Flask + flask.signals: Flask + flask.templating: Flask + flask.testing: Flask + flask.typing: Flask + flask.views: Flask + flask.wrappers: Flask + importlib_metadata: importlib_metadata + itsdangerous: itsdangerous + itsdangerous.encoding: itsdangerous + itsdangerous.exc: itsdangerous + itsdangerous.serializer: itsdangerous + itsdangerous.signer: itsdangerous + itsdangerous.timed: itsdangerous + itsdangerous.url_safe: itsdangerous + jinja2: Jinja2 + jinja2.async_utils: Jinja2 + jinja2.bccache: Jinja2 + jinja2.compiler: Jinja2 + jinja2.constants: Jinja2 + jinja2.debug: Jinja2 + jinja2.defaults: Jinja2 + jinja2.environment: Jinja2 + jinja2.exceptions: Jinja2 + jinja2.ext: Jinja2 + jinja2.filters: Jinja2 + jinja2.idtracking: Jinja2 + jinja2.lexer: Jinja2 + jinja2.loaders: Jinja2 + jinja2.meta: Jinja2 + jinja2.nativetypes: Jinja2 + jinja2.nodes: Jinja2 + jinja2.optimizer: Jinja2 + jinja2.parser: Jinja2 + jinja2.runtime: Jinja2 + jinja2.sandbox: Jinja2 + jinja2.utils: Jinja2 + jinja2.visitor: Jinja2 + markupsafe: MarkupSafe + werkzeug: Werkzeug + werkzeug.datastructures: Werkzeug + werkzeug.debug: Werkzeug + werkzeug.debug.console: Werkzeug + werkzeug.debug.repr: Werkzeug + werkzeug.debug.tbtools: Werkzeug + werkzeug.exceptions: Werkzeug + werkzeug.formparser: Werkzeug + werkzeug.http: Werkzeug + werkzeug.local: Werkzeug + werkzeug.middleware: Werkzeug + werkzeug.middleware.dispatcher: Werkzeug + werkzeug.middleware.http_proxy: Werkzeug + werkzeug.middleware.lint: Werkzeug + werkzeug.middleware.profiler: Werkzeug + werkzeug.middleware.proxy_fix: Werkzeug + werkzeug.middleware.shared_data: Werkzeug + werkzeug.routing: Werkzeug + werkzeug.routing.converters: Werkzeug + werkzeug.routing.exceptions: Werkzeug + werkzeug.routing.map: Werkzeug + werkzeug.routing.matcher: Werkzeug + werkzeug.routing.rules: Werkzeug + werkzeug.sansio: Werkzeug + werkzeug.sansio.http: Werkzeug + werkzeug.sansio.multipart: Werkzeug + werkzeug.sansio.request: Werkzeug + werkzeug.sansio.response: Werkzeug + werkzeug.sansio.utils: Werkzeug + werkzeug.security: Werkzeug + werkzeug.serving: Werkzeug + werkzeug.test: Werkzeug + werkzeug.testapp: Werkzeug + werkzeug.urls: Werkzeug + werkzeug.user_agent: Werkzeug + werkzeug.utils: Werkzeug + werkzeug.wrappers: Werkzeug + werkzeug.wrappers.request: Werkzeug + werkzeug.wrappers.response: Werkzeug + werkzeug.wsgi: Werkzeug + zipp: zipp + zipp.py310compat: zipp + pip_repository: + name: pip +integrity: 4153df7683d64d7d6ad56c14ea1c7f7bec84a2ddf9ef8f075d1bb9313b8d11aa diff --git a/examples/bzlmod_build_file_generation/random_number_generator/BUILD.bazel b/examples/bzlmod_build_file_generation/random_number_generator/BUILD.bazel new file mode 100644 index 0000000000..95e16fd301 --- /dev/null +++ b/examples/bzlmod_build_file_generation/random_number_generator/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + +py_library( + name = "random_number_generator", + srcs = [ + "__init__.py", + "generate_random_number.py", + ], + imports = [".."], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "random_number_generator_test", + srcs = ["__test__.py"], + imports = [".."], + main = "__test__.py", + deps = [":random_number_generator"], +) diff --git a/examples/bzlmod_build_file_generation/random_number_generator/__init__.py b/examples/bzlmod_build_file_generation/random_number_generator/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/bzlmod_build_file_generation/random_number_generator/__test__.py b/examples/bzlmod_build_file_generation/random_number_generator/__test__.py new file mode 100644 index 0000000000..5facfeee9e --- /dev/null +++ b/examples/bzlmod_build_file_generation/random_number_generator/__test__.py @@ -0,0 +1,28 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import random_number_generator.generate_random_number as generate_random_number + + +class TestRandomNumberGenerator(unittest.TestCase): + def test_generate_random_number(self): + number = generate_random_number.generate_random_number() + self.assertGreaterEqual(number, 1) + self.assertLessEqual(number, 10) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/bzlmod_build_file_generation/random_number_generator/generate_random_number.py b/examples/bzlmod_build_file_generation/random_number_generator/generate_random_number.py new file mode 100644 index 0000000000..cb37c3891e --- /dev/null +++ b/examples/bzlmod_build_file_generation/random_number_generator/generate_random_number.py @@ -0,0 +1,21 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random + +"""Generate a random number""" + + +def generate_random_number(): + return random.randint(1, 10) diff --git a/examples/bzlmod_build_file_generation/requirements.in b/examples/bzlmod_build_file_generation/requirements.in new file mode 100644 index 0000000000..7e1060246f --- /dev/null +++ b/examples/bzlmod_build_file_generation/requirements.in @@ -0,0 +1 @@ +flask diff --git a/examples/bzlmod_build_file_generation/requirements_lock.txt b/examples/bzlmod_build_file_generation/requirements_lock.txt new file mode 100644 index 0000000000..f73827a36e --- /dev/null +++ b/examples/bzlmod_build_file_generation/requirements_lock.txt @@ -0,0 +1,78 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# bazel run //:requirements.update +# +click==8.1.3 \ + --hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \ + --hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48 + # via flask +flask==2.2.2 \ + --hash=sha256:642c450d19c4ad482f96729bd2a8f6d32554aa1e231f4f6b4e7e5264b16cca2b \ + --hash=sha256:b9c46cc36662a7949f34b52d8ec7bb59c0d74ba08ba6cb9ce9adc1d8676d9526 + # via -r ./requirements.in +importlib-metadata==5.2.0 \ + --hash=sha256:0eafa39ba42bf225fc00e67f701d71f85aead9f878569caf13c3724f704b970f \ + --hash=sha256:404d48d62bba0b7a77ff9d405efd91501bef2e67ff4ace0bed40a0cf28c3c7cd + # via flask +itsdangerous==2.1.2 \ + --hash=sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44 \ + --hash=sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a + # via flask +jinja2==3.1.2 \ + --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ + --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 + # via flask +markupsafe==2.1.1 \ + --hash=sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003 \ + --hash=sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88 \ + --hash=sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5 \ + --hash=sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7 \ + --hash=sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a \ + --hash=sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603 \ + --hash=sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1 \ + --hash=sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135 \ + --hash=sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247 \ + --hash=sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6 \ + --hash=sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601 \ + --hash=sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77 \ + --hash=sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02 \ + --hash=sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e \ + --hash=sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63 \ + --hash=sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f \ + --hash=sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980 \ + --hash=sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b \ + --hash=sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812 \ + --hash=sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff \ + --hash=sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96 \ + --hash=sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1 \ + --hash=sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925 \ + --hash=sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a \ + --hash=sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6 \ + --hash=sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e \ + --hash=sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f \ + --hash=sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4 \ + --hash=sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f \ + --hash=sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3 \ + --hash=sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c \ + --hash=sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a \ + --hash=sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417 \ + --hash=sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a \ + --hash=sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a \ + --hash=sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37 \ + --hash=sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452 \ + --hash=sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933 \ + --hash=sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a \ + --hash=sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7 + # via + # jinja2 + # werkzeug +werkzeug==2.2.2 \ + --hash=sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f \ + --hash=sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5 + # via flask +zipp==3.11.0 \ + --hash=sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa \ + --hash=sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766 + # via importlib-metadata diff --git a/examples/bzlmod_build_file_generation/requirements_windows.txt b/examples/bzlmod_build_file_generation/requirements_windows.txt new file mode 100644 index 0000000000..fc097141c5 --- /dev/null +++ b/examples/bzlmod_build_file_generation/requirements_windows.txt @@ -0,0 +1,82 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# bazel run //:requirements.update +# +click==8.1.3 \ + --hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \ + --hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48 + # via flask +colorama==0.4.6 \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 + # via click +flask==2.2.2 \ + --hash=sha256:642c450d19c4ad482f96729bd2a8f6d32554aa1e231f4f6b4e7e5264b16cca2b \ + --hash=sha256:b9c46cc36662a7949f34b52d8ec7bb59c0d74ba08ba6cb9ce9adc1d8676d9526 + # via -r ./requirements.in +importlib-metadata==5.2.0 \ + --hash=sha256:0eafa39ba42bf225fc00e67f701d71f85aead9f878569caf13c3724f704b970f \ + --hash=sha256:404d48d62bba0b7a77ff9d405efd91501bef2e67ff4ace0bed40a0cf28c3c7cd + # via flask +itsdangerous==2.1.2 \ + --hash=sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44 \ + --hash=sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a + # via flask +jinja2==3.1.2 \ + --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ + --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 + # via flask +markupsafe==2.1.1 \ + --hash=sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003 \ + --hash=sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88 \ + --hash=sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5 \ + --hash=sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7 \ + --hash=sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a \ + --hash=sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603 \ + --hash=sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1 \ + --hash=sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135 \ + --hash=sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247 \ + --hash=sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6 \ + --hash=sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601 \ + --hash=sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77 \ + --hash=sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02 \ + --hash=sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e \ + --hash=sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63 \ + --hash=sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f \ + --hash=sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980 \ + --hash=sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b \ + --hash=sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812 \ + --hash=sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff \ + --hash=sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96 \ + --hash=sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1 \ + --hash=sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925 \ + --hash=sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a \ + --hash=sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6 \ + --hash=sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e \ + --hash=sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f \ + --hash=sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4 \ + --hash=sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f \ + --hash=sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3 \ + --hash=sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c \ + --hash=sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a \ + --hash=sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417 \ + --hash=sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a \ + --hash=sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a \ + --hash=sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37 \ + --hash=sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452 \ + --hash=sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933 \ + --hash=sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a \ + --hash=sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7 + # via + # jinja2 + # werkzeug +werkzeug==2.2.2 \ + --hash=sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f \ + --hash=sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5 + # via flask +zipp==3.11.0 \ + --hash=sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa \ + --hash=sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766 + # via importlib-metadata From d0c1549280ba7e90bcd4f1e3e52ba81073e5597c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 2 Jan 2023 15:56:51 +0900 Subject: [PATCH 2/8] Add the new example test to the integration test list --- examples/BUILD.bazel | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/BUILD.bazel b/examples/BUILD.bazel index e0a7e5a72d..739f9d4a33 100644 --- a/examples/BUILD.bazel +++ b/examples/BUILD.bazel @@ -53,3 +53,9 @@ bazel_integration_test( bzlmod = True, override_bazel_version = "6.0.0", ) + +bazel_integration_test( + name = "bzlmod_build_file_generation_example", + bzlmod = True, + override_bazel_version = "6.0.0", +) From 7f4e1253d0e7f60d65735fbf3cfcf4af226134b8 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 2 Jan 2023 16:04:43 +0900 Subject: [PATCH 3/8] Use the plain string instead of requirement macro Gazelle operates on plain strings and before we do any refactoring we should start with the plain string of the requirement first. This shows, that bzlmod and non-bzlmod usage of gazelle introduces differences in BUILD.bazel files, but maybe this can be resolved later. --- examples/bzlmod_build_file_generation/BUILD.bazel | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index 8e170d1344..a4e6779272 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -2,7 +2,6 @@ # various rulesets and dependencies. # The `load` statement imports the symbol for the rule, in the defined # ruleset. When the symbol is loaded you can use the rule. -load("@pip//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") @@ -23,7 +22,7 @@ py_library( visibility = ["//:__subpackages__"], deps = [ "//random_number_generator", - requirement("flask"), + "@pip//:flask_pkg", ], ) From aa900a503e6cd010e5588aa044ff0afbda9e74ec Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 2 Jan 2023 16:08:20 +0900 Subject: [PATCH 4/8] Add gazelle target definition and fix bzlmod dependencies --- MODULE.bazel | 19 +++++++++++++ .../bzlmod_build_file_generation/BUILD.bazel | 27 +++++++++++++++++++ .../bzlmod_build_file_generation/MODULE.bazel | 2 ++ 3 files changed, 48 insertions(+) diff --git a/MODULE.bazel b/MODULE.bazel index e1be99374c..edb553d8ac 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -27,3 +27,22 @@ use_repo( "pypi__wheel", "pypi__zipp", ) + +bazel_dep(name = "rules_go", repo_name = "io_bazel_rules_go", version = "0.35.0") +bazel_dep(name = "gazelle", repo_name = "bazel_gazelle", version = "0.28.0") + +go_deps = use_extension("@bazel_gazelle//:extensions.bzl", "go_deps") + +go_deps.from_file(go_mod = "//:go.mod") + +use_repo( + go_deps, + "com_github_bazelbuild_bazel_gazelle", + "com_github_bazelbuild_buildtools", + "com_github_bazelbuild_rules_go", + "com_github_bmatcuk_doublestar", + "com_github_emirpasic_gods", + "com_github_ghodss_yaml", + "com_github_google_uuid", + "in_gopkg_yaml_v2", +) diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index a4e6779272..8c298b98ff 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -2,6 +2,8 @@ # various rulesets and dependencies. # The `load` statement imports the symbol for the rule, in the defined # ruleset. When the symbol is loaded you can use the rule. +load("@gazelle//:def.bzl", "gazelle") +load("@rules_python//gazelle:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") @@ -13,6 +15,17 @@ compile_pip_requirements( requirements_windows = "requirements_windows.txt", ) +# Our gazelle target points to the python gazelle binary. +# This is the simple case where we only need one language supported. +# If you also had proto, go, or other gazelle-supported languages, +# you would also need a gazelle_binary rule. +# See https://github.com/bazelbuild/bazel-gazelle/blob/master/extend.rst#example +gazelle( + name = "gazelle", + data = GAZELLE_PYTHON_RUNTIME_DEPS, + gazelle = "@rules_python//gazelle:gazelle_python_binary", +) + # This rule is auto-generated and managed by Gazelle, # because it found the __init__.py file in this folder. # See: https://bazel.build/reference/be/python#py_library @@ -56,3 +69,17 @@ py_test( main = "__test__.py", deps = [":bzlmod_build_file_generation"], ) + +# This repository rule fetches the metadata for python packages we +# depend on. That data is required for the gazelle_python_manifest +# rule to update our manifest file. +# To see what this rule does, try `bazel run @modules_map//:print` +modules_mapping( + name = "modules_map", + exclude_patterns = [ + "^_|(\\._)+", # This is the default. + "(\\.tests)+", # Add a custom one to get rid of the psutil tests. + ], + # TODO @aignas 2023-01-02: Think of a better way to do this + wheels = [r.replace("@pip_", "@pip//:").replace("//:whl", "_whl") for r in all_whl_requirements], +) diff --git a/examples/bzlmod_build_file_generation/MODULE.bazel b/examples/bzlmod_build_file_generation/MODULE.bazel index 14a338cc52..534ee36cd0 100644 --- a/examples/bzlmod_build_file_generation/MODULE.bazel +++ b/examples/bzlmod_build_file_generation/MODULE.bazel @@ -33,3 +33,5 @@ pip.parse( ) use_repo(pip, "pip") + +bazel_dep(name = "gazelle", version = "0.28.0") From c2a07ad6c371fa1046b7967d80018907a66727cb Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 2 Jan 2023 16:18:11 +0900 Subject: [PATCH 5/8] Add modules_map generation based on the whl list Fix the `all_whl_requirements` and `all_requirements` macros to be usable when using bzlmod. --- .../bzlmod_build_file_generation/BUILD.bazel | 29 ++++++++++--------- python/pip_install/extract_wheels/bazel.py | 27 ++++++++++------- .../parse_requirements_to_bzl.py | 5 ++-- 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index 8c298b98ff..eb1e295517 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -3,7 +3,9 @@ # The `load` statement imports the symbol for the rule, in the defined # ruleset. When the symbol is loaded you can use the rule. load("@gazelle//:def.bzl", "gazelle") +load("@pip//:requirements.bzl", "all_whl_requirements") load("@rules_python//gazelle:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS") +load("@rules_python//gazelle/modules_mapping:def.bzl", "modules_mapping") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") @@ -15,6 +17,19 @@ compile_pip_requirements( requirements_windows = "requirements_windows.txt", ) +# This repository rule fetches the metadata for python packages we +# depend on. That data is required for the gazelle_python_manifest +# rule to update our manifest file. +# To see what this rule does, try `bazel run @modules_map//:print` +modules_mapping( + name = "modules_map", + exclude_patterns = [ + "^_|(\\._)+", # This is the default. + "(\\.tests)+", # Add a custom one to get rid of the psutil tests. + ], + wheels = all_whl_requirements, +) + # Our gazelle target points to the python gazelle binary. # This is the simple case where we only need one language supported. # If you also had proto, go, or other gazelle-supported languages, @@ -69,17 +84,3 @@ py_test( main = "__test__.py", deps = [":bzlmod_build_file_generation"], ) - -# This repository rule fetches the metadata for python packages we -# depend on. That data is required for the gazelle_python_manifest -# rule to update our manifest file. -# To see what this rule does, try `bazel run @modules_map//:print` -modules_mapping( - name = "modules_map", - exclude_patterns = [ - "^_|(\\._)+", # This is the default. - "(\\.tests)+", # Add a custom one to get rid of the psutil tests. - ], - # TODO @aignas 2023-01-02: Think of a better way to do this - wheels = [r.replace("@pip_", "@pip//:").replace("//:whl", "_whl") for r in all_whl_requirements], -) diff --git a/python/pip_install/extract_wheels/bazel.py b/python/pip_install/extract_wheels/bazel.py index c0a4aec422..273a673cc6 100644 --- a/python/pip_install/extract_wheels/bazel.py +++ b/python/pip_install/extract_wheels/bazel.py @@ -25,17 +25,22 @@ def sanitise_name(name: str, prefix: str) -> str: return prefix + name.replace("-", "_").replace(".", "_").lower() -def _whl_name_to_repo_root(whl_name: str, repo_prefix: str) -> str: - return "@{}//".format(sanitise_name(whl_name, prefix=repo_prefix)) - - -def sanitised_repo_library_label(whl_name: str, repo_prefix: str) -> str: - return '"{}:{}"'.format( - _whl_name_to_repo_root(whl_name, repo_prefix), PY_LIBRARY_LABEL +def _sanitized_label(whl_name: str, repo_prefix: str, label: str, bzlmod: bool = False) -> str: + if bzlmod: + return '"@{}//:{}_{}"'.format( + repo[:-1], + sanitise_name(whl_name, prefix=""), + label, + ) + + return '"@{}//:{}"'.format( + sanitise_name(whl_name, prefix=repo_prefix), + label, ) +def sanitised_repo_library_label(whl_name: str, repo_prefix: str, bzlmod: bool = False) -> str: + return _sanitized_label(whl_name, repo_prefix, PY_LIBRARY_LABEL) -def sanitised_repo_file_label(whl_name: str, repo_prefix: str) -> str: - return '"{}:{}"'.format( - _whl_name_to_repo_root(whl_name, repo_prefix), WHEEL_FILE_LABEL - ) + +def sanitised_repo_file_label(whl_name: str, repo_prefix: str, bzlmod: bool = False) -> str: + return _sanitized_label(whl_name, repo_prefix, WHEEL_FILE_LABEL) diff --git a/python/pip_install/extract_wheels/parse_requirements_to_bzl.py b/python/pip_install/extract_wheels/parse_requirements_to_bzl.py index 686a57d8b2..bd616e38d9 100644 --- a/python/pip_install/extract_wheels/parse_requirements_to_bzl.py +++ b/python/pip_install/extract_wheels/parse_requirements_to_bzl.py @@ -117,13 +117,14 @@ def generate_parsed_requirements_contents( all_requirements = ", ".join( [ - bazel.sanitised_repo_library_label(ir.name, repo_prefix=repo_prefix) + bazel.sanitised_repo_library_label(ir.name, repo_prefix=repo_prefix, bzlmod=True) for ir, _ in install_req_and_lines ] ) + all_whl_requirements = ", ".join( [ - bazel.sanitised_repo_file_label(ir.name, repo_prefix=repo_prefix) + bazel.sanitised_repo_file_label(ir.name, repo_prefix=repo_prefix, bzlmod=True) for ir, _ in install_req_and_lines ] ) From 0a5f79fcb0dd3e4ff627dd5135ac3c7f7cf39e48 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 2 Jan 2023 16:47:51 +0900 Subject: [PATCH 6/8] Add gazelle_python_manifest generation to the example --- examples/bzlmod_build_file_generation/BUILD.bazel | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index eb1e295517..e9b0f2d0e0 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -5,6 +5,7 @@ load("@gazelle//:def.bzl", "gazelle") load("@pip//:requirements.bzl", "all_whl_requirements") load("@rules_python//gazelle:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS") +load("@rules_python//gazelle/manifest:defs.bzl", "gazelle_python_manifest") load("@rules_python//gazelle/modules_mapping:def.bzl", "modules_mapping") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") @@ -30,6 +31,20 @@ modules_mapping( wheels = all_whl_requirements, ) +# Gazelle python extension needs a manifest file mapping from +# an import to the installed package that provides it. +# This macro produces two targets: +# - //:gazelle_python_manifest.update can be used with `bazel run` +# to recalculate the manifest +# - //:gazelle_python_manifest.test is a test target ensuring that +# the manifest doesn't need to be updated +gazelle_python_manifest( + name = "gazelle_python_manifest", + modules_mapping = ":modules_map", + pip_repository_name = "pip", + requirements = "//:requirements_lock.txt", +) + # Our gazelle target points to the python gazelle binary. # This is the simple case where we only need one language supported. # If you also had proto, go, or other gazelle-supported languages, From 4fd35ade9747bd9834fbc864a3b3f11d4f3f15c1 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 2 Jan 2023 17:28:13 +0900 Subject: [PATCH 7/8] Allow configuring the pip dependency naming convention --- .../bzlmod_build_file_generation/BUILD.bazel | 5 + gazelle/configure.go | 9 ++ gazelle/pythonconfig/pythonconfig.go | 135 +++++++++++++----- gazelle/pythonconfig/types.go | 2 +- 4 files changed, 113 insertions(+), 38 deletions(-) diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index e9b0f2d0e0..18b4a2c396 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -56,6 +56,11 @@ gazelle( gazelle = "@rules_python//gazelle:gazelle_python_binary", ) +# Using bzlmod we also need to set a different pip repository naming convention: +# +# gazelle:python_pip_repo_naming_convention $repo_name$ +# gazelle:python_pip_target_naming_convention $distribution_name$_pkg + # This rule is auto-generated and managed by Gazelle, # because it found the __init__.py file in this folder. # See: https://bazel.build/reference/be/python#py_library diff --git a/gazelle/configure.go b/gazelle/configure.go index 8e71110d0a..21d84f8a84 100644 --- a/gazelle/configure.go +++ b/gazelle/configure.go @@ -48,6 +48,9 @@ func (py *Configurer) KnownDirectives() []string { pythonconfig.LibraryNamingConvention, pythonconfig.BinaryNamingConvention, pythonconfig.TestNamingConvention, + pythonconfig.PipRepoNamingConvention, + pythonconfig.PipPackageNamingConvention, + pythonconfig.PipTargetNamingConvention, } } @@ -136,6 +139,12 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) { config.SetBinaryNamingConvention(strings.TrimSpace(d.Value)) case pythonconfig.TestNamingConvention: config.SetTestNamingConvention(strings.TrimSpace(d.Value)) + case pythonconfig.PipRepoNamingConvention: + config.SetPipRepoNamingConvention(strings.TrimSpace(d.Value)) + case pythonconfig.PipPackageNamingConvention: + config.SetPipPackageNamingConvention(strings.TrimSpace(d.Value)) + case pythonconfig.PipTargetNamingConvention: + config.SetPipTargetNamingConvention(strings.TrimSpace(d.Value)) } } diff --git a/gazelle/pythonconfig/pythonconfig.go b/gazelle/pythonconfig/pythonconfig.go index 64f6264323..8488140ee8 100644 --- a/gazelle/pythonconfig/pythonconfig.go +++ b/gazelle/pythonconfig/pythonconfig.go @@ -49,6 +49,17 @@ const ( // naming convention. See python_library_naming_convention for more info on // the package name interpolation. TestNamingConvention = "python_test_naming_convention" + // PipRepoNamingConvention represents the directive that controls the + // mapping between Python modules and the repos. E.g. if the Python package name is `foo` and the + // pip repository is named `my_pip`, setting this + // to `$repo_name$_$distribution_name$` would render to `@my_pip_foo`. + PipRepoNamingConvention = "python_pip_repo_naming_convention" + // PipPackageNamingConvention represents the directive that controls the + // mapping between Python modules and the repos. By default it is `` + PipPackageNamingConvention = "python_pip_package_naming_convention" + // PipTargetNamingConvention represents the directive that controls the + // mapping between Python modules and the repos. By default it is `pkg` + PipTargetNamingConvention = "python_pip_target_naming_convention" ) // GenerationModeType represents one of the generation modes for the Python @@ -67,7 +78,9 @@ const ( ) const ( - packageNameNamingConventionSubstitution = "$package_name$" + repoNamePipRepoNamingConventionSubstitution = "$repo_name$" + distributionNamePipRepoNamingConventionSubstitution = "$distribution_name$" + packageNameNamingConventionSubstitution = "$package_name$" ) // defaultIgnoreFiles is the list of default values used in the @@ -99,14 +112,17 @@ type Config struct { pythonProjectRoot string gazelleManifest *manifest.Manifest - excludedPatterns *singlylinkedlist.List - ignoreFiles map[string]struct{} - ignoreDependencies map[string]struct{} - validateImportStatements bool - coarseGrainedGeneration bool - libraryNamingConvention string - binaryNamingConvention string - testNamingConvention string + excludedPatterns *singlylinkedlist.List + ignoreFiles map[string]struct{} + ignoreDependencies map[string]struct{} + validateImportStatements bool + coarseGrainedGeneration bool + libraryNamingConvention string + binaryNamingConvention string + testNamingConvention string + pipRepoNamingConvention string + pipPackageNamingConvention string + pipTargetNamingConvention string } // New creates a new Config. @@ -115,17 +131,20 @@ func New( pythonProjectRoot string, ) *Config { return &Config{ - extensionEnabled: true, - repoRoot: repoRoot, - pythonProjectRoot: pythonProjectRoot, - excludedPatterns: singlylinkedlist.New(), - ignoreFiles: make(map[string]struct{}), - ignoreDependencies: make(map[string]struct{}), - validateImportStatements: true, - coarseGrainedGeneration: false, - libraryNamingConvention: packageNameNamingConventionSubstitution, - binaryNamingConvention: fmt.Sprintf("%s_bin", packageNameNamingConventionSubstitution), - testNamingConvention: fmt.Sprintf("%s_test", packageNameNamingConventionSubstitution), + extensionEnabled: true, + repoRoot: repoRoot, + pythonProjectRoot: pythonProjectRoot, + excludedPatterns: singlylinkedlist.New(), + ignoreFiles: make(map[string]struct{}), + ignoreDependencies: make(map[string]struct{}), + validateImportStatements: true, + coarseGrainedGeneration: false, + libraryNamingConvention: packageNameNamingConventionSubstitution, + binaryNamingConvention: fmt.Sprintf("%s_bin", packageNameNamingConventionSubstitution), + testNamingConvention: fmt.Sprintf("%s_test", packageNameNamingConventionSubstitution), + pipRepoNamingConvention: fmt.Sprintf("%s_%s", repoNamePipRepoNamingConventionSubstitution, distributionNamePipRepoNamingConventionSubstitution), + pipPackageNamingConvention: "", + pipTargetNamingConvention: "pkg", } } @@ -150,6 +169,9 @@ func (c *Config) NewChild() *Config { libraryNamingConvention: c.libraryNamingConvention, binaryNamingConvention: c.binaryNamingConvention, testNamingConvention: c.testNamingConvention, + pipRepoNamingConvention: c.pipRepoNamingConvention, + pipPackageNamingConvention: c.pipPackageNamingConvention, + pipTargetNamingConvention: c.pipTargetNamingConvention, } } @@ -195,24 +217,32 @@ func (c *Config) SetGazelleManifest(gazelleManifest *manifest.Manifest) { // name. func (c *Config) FindThirdPartyDependency(modName string) (string, bool) { for currentCfg := c; currentCfg != nil; currentCfg = currentCfg.parent { - if currentCfg.gazelleManifest != nil { - gazelleManifest := currentCfg.gazelleManifest - if distributionName, ok := gazelleManifest.ModulesMapping[modName]; ok { - var distributionRepositoryName string - if gazelleManifest.PipDepsRepositoryName != "" { - distributionRepositoryName = gazelleManifest.PipDepsRepositoryName - } else if gazelleManifest.PipRepository != nil { - distributionRepositoryName = gazelleManifest.PipRepository.Name - } - sanitizedDistribution := strings.ToLower(distributionName) - sanitizedDistribution = strings.ReplaceAll(sanitizedDistribution, "-", "_") - var lbl label.Label - // @_//:pkg - distributionRepositoryName = distributionRepositoryName + "_" + sanitizedDistribution - lbl = label.New(distributionRepositoryName, "", "pkg") - return lbl.String(), true - } + + if currentCfg.gazelleManifest == nil { + continue + } + + gazelleManifest := currentCfg.gazelleManifest + distributionName, ok := gazelleManifest.ModulesMapping[modName] + if !ok { + continue } + + var distributionRepositoryName string + if gazelleManifest.PipDepsRepositoryName != "" { + distributionRepositoryName = gazelleManifest.PipDepsRepositoryName + } else if gazelleManifest.PipRepository != nil { + distributionRepositoryName = gazelleManifest.PipRepository.Name + } + sanitizedDistribution := strings.ToLower(distributionName) + sanitizedDistribution = strings.ReplaceAll(sanitizedDistribution, "-", "_") + + lbl := label.New( + currentCfg.renderPipRepository(distributionRepositoryName, sanitizedDistribution), + currentCfg.renderPipPackage(sanitizedDistribution), + currentCfg.renderPipTarget(sanitizedDistribution), + ) + return lbl.String(), true } return "", false } @@ -327,6 +357,37 @@ func (c *Config) SetTestNamingConvention(testNamingConvention string) { c.testNamingConvention = testNamingConvention } +// SetPipRepoNamingConvention sets the dependency naming convention. +func (c *Config) SetPipRepoNamingConvention(pipRepoNamingConvention string) { + c.pipRepoNamingConvention = pipRepoNamingConvention +} + +// Accepts sanitized input. +func (c *Config) renderPipRepository(distributionRepoName, distributionName string) string { + rendered := strings.ReplaceAll(c.pipRepoNamingConvention, repoNamePipRepoNamingConventionSubstitution, distributionRepoName) + return strings.ReplaceAll(rendered, distributionNamePipRepoNamingConventionSubstitution, distributionName) +} + +// SetPipPackageNamingConvention sets the dependency naming convention. +func (c *Config) SetPipPackageNamingConvention(pipPackageNamingConvention string) { + c.pipPackageNamingConvention = pipPackageNamingConvention +} + +// Accepts sanitized input. +func (c *Config) renderPipPackage(distributionName string) string { + return strings.ReplaceAll(c.pipPackageNamingConvention, distributionNamePipRepoNamingConventionSubstitution, distributionName) +} + +// SetPipTargetNamingConvention sets the dependency naming convention. +func (c *Config) SetPipTargetNamingConvention(pipTargetNamingConvention string) { + c.pipTargetNamingConvention = pipTargetNamingConvention +} + +// Accepts sanitized input. +func (c *Config) renderPipTarget(distributionName string) string { + return strings.ReplaceAll(c.pipTargetNamingConvention, distributionNamePipRepoNamingConventionSubstitution, distributionName) +} + // RenderTestName returns the py_test target name by performing all // substitutions. func (c *Config) RenderTestName(packageName string) string { diff --git a/gazelle/pythonconfig/types.go b/gazelle/pythonconfig/types.go index bdb535bf6e..1c151b20a2 100644 --- a/gazelle/pythonconfig/types.go +++ b/gazelle/pythonconfig/types.go @@ -100,4 +100,4 @@ func (sml *StringMapList) Set(s string) error { func (sml *StringMapList) Get(key string) (string, bool) { val, exists := sml.mapping[key] return val, exists -} \ No newline at end of file +} From 07322ad68d9870fb2b86d1ebceaa222b71ad687d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 2 Jan 2023 19:26:51 +0900 Subject: [PATCH 8/8] Ignore the new example in .bazelrc --- .bazelrc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bazelrc b/.bazelrc index 2ad0284c3b..66c6c0f89d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -3,8 +3,8 @@ # This lets us glob() up all the files inside the examples to make them inputs to tests # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/get_url,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/compile_pip_requirements,tests/pip_repository_entry_points,tests/pip_deps -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/get_url,examples/bzlmod,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/compile_pip_requirements,tests/pip_repository_entry_points,tests/pip_deps +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/get_url,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/get_url,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/compile_pip_requirements,tests/pip_repository_entry_points,tests/pip_deps +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/get_url,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/get_url,examples/multi_python_versions,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/compile_pip_requirements,tests/pip_repository_entry_points,tests/pip_deps test --test_output=errors