From 0e5b5f8c437e7ca93b0143f6af72df2b5961d22e Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Wed, 18 Jan 2023 17:00:45 -0800 Subject: [PATCH 1/3] chore: publish a runfiles library as a wheel Wire it up to GH actions so it is published for each release. Tested locally with: bazel build python/runfiles:wheel --embed_label=1.0.2 --stamp PYTHONPATH=bazel-bin/python/runfiles/bazel_runfiles-_BUILD_EMBED_LABEL_-py3-none-any.whl python >>> import runfiles >>> runfiles.Create() Note, I would have liked to call the package bazel-runfiles, but this isn't possible without either refactoring the paths in this repo, or doing some fancy starlark to copy files around to create a folder that we turn into the wheel. There is no project https://pypi.org/project/runfiles though there is a https://pypi.org/project/runfile We could try harder to get the name we prefer. --- .github/workflows/release.yml | 12 ++++++++- python/runfiles/BUILD.bazel | 23 +++++++++++++++- python/runfiles/README.md | 51 +++++++++++++++++++++++++++++++++++ python/runfiles/__init__.py | 1 + python/runfiles/runfiles.py | 44 ------------------------------ 5 files changed, 85 insertions(+), 46 deletions(-) create mode 100644 python/runfiles/README.md create mode 100644 python/runfiles/__init__.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a675fe1562..5906289e66 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,17 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Prepare workspace snippet - run: .github/workflows/workspace_snippet.sh ${{ env.GITHUB_REF_NAME }} > release_notes.txt + run: .github/workflows/workspace_snippet.sh > release_notes.txt + - name: Build wheel dist + run: bazel build --stamp --embed_label=${{ env.GITHUB_REF_NAME }} //python/runfiles:wheel + - name: Publish runfiles package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + # Note, the PYPI_API_TOKEN was added on + # https://github.com/bazelbuild/rules_python/settings/secrets/actions + # and currently uses a token which authenticates as https://pypi.org/user/alexeagle/ + password: ${{ secrets.PYPI_API_TOKEN }} + packages_dir: bazel-bin/python/runfiles - name: Release uses: softprops/action-gh-release@v1 with: diff --git a/python/runfiles/BUILD.bazel b/python/runfiles/BUILD.bazel index 2089c418d8..ba612fa33e 100644 --- a/python/runfiles/BUILD.bazel +++ b/python/runfiles/BUILD.bazel @@ -13,6 +13,7 @@ # limitations under the License. load("//python:defs.bzl", "py_library") +load("//python:packaging.bzl", "py_wheel") filegroup( name = "distribution", @@ -22,6 +23,26 @@ filegroup( py_library( name = "runfiles", - srcs = ["runfiles.py"], + srcs = [ + "__init__.py", + "runfiles.py", + ], visibility = ["//visibility:public"], ) + +# TODO(alexeagle): add some example which depends on this wheel to verify it can be used +py_wheel( + name = "wheel", + # From https://pypi.org/classifiers/ + classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Apache Software License", + ], + description_file = "README.md", + distribution = "bazel_runfiles", + homepage = "https://github.com/bazelbuild/rules_python", + strip_path_prefixes = ["python"], + version = "{BUILD_EMBED_LABEL}", + visibility = ["//visibility:public"], + deps = [":runfiles"], +) diff --git a/python/runfiles/README.md b/python/runfiles/README.md new file mode 100644 index 0000000000..997808463f --- /dev/null +++ b/python/runfiles/README.md @@ -0,0 +1,51 @@ +# bazel-runfiles library + +This is a Bazel Runfiles lookup library for Bazel-built Python binaries and tests. + +Typical Usage +------------- + +1. Add the 'runfiles' dependency along with other third-party dependencies, for example in your + `requirements.txt` file. + +2. Depend on this runfiles library from your build rule, like you would other third-party libraries. + + py_binary( + name = "my_binary", + ... + deps = [requirement("runfiles")], + ) + +3. Import the runfiles library. + + import runfiles + +4. Create a Runfiles object and use rlocation to look up runfile paths: + + r = runfiles.Create() + ... + with open(r.Rlocation("my_workspace/path/to/my/data.txt"), "r") as f: + contents = f.readlines() + ... + + The code above creates a manifest- or directory-based implementations based + on the environment variables in os.environ. See `Create()` for more info. + + If you want to explicitly create a manifest- or directory-based + implementations, you can do so as follows: + + r1 = runfiles.CreateManifestBased("path/to/foo.runfiles_manifest") + + r2 = runfiles.CreateDirectoryBased("path/to/foo.runfiles/") + + If you want to start subprocesses that also need runfiles, you need to set + the right environment variables for them: + + import subprocess + import runfiles + + r = runfiles.Create() + env = {} + ... + env.update(r.EnvVars()) + p = subprocess.Popen([r.Rlocation("path/to/binary")], env, ...) \ No newline at end of file diff --git a/python/runfiles/__init__.py b/python/runfiles/__init__.py new file mode 100644 index 0000000000..eb42f79c8d --- /dev/null +++ b/python/runfiles/__init__.py @@ -0,0 +1 @@ +from .runfiles import * diff --git a/python/runfiles/runfiles.py b/python/runfiles/runfiles.py index 49a1e7b910..ab08a9aa88 100644 --- a/python/runfiles/runfiles.py +++ b/python/runfiles/runfiles.py @@ -13,50 +13,6 @@ # limitations under the License. """Runfiles lookup library for Bazel-built Python binaries and tests. - -USAGE: - -1. Depend on this runfiles library from your build rule: - - py_binary( - name = "my_binary", - ... - deps = ["@rules_python//python/runfiles"], - ) - -2. Import the runfiles library. - - from python.runfiles import runfiles - -3. Create a Runfiles object and use rlocation to look up runfile paths: - - r = runfiles.Create() - ... - with open(r.Rlocation("my_workspace/path/to/my/data.txt"), "r") as f: - contents = f.readlines() - ... - - The code above creates a manifest- or directory-based implementations based - on the environment variables in os.environ. See `Create()` for more info. - - If you want to explicitly create a manifest- or directory-based - implementations, you can do so as follows: - - r1 = runfiles.CreateManifestBased("path/to/foo.runfiles_manifest") - - r2 = runfiles.CreateDirectoryBased("path/to/foo.runfiles/") - - If you want to start subprocesses that also need runfiles, you need to set - the right environment variables for them: - - import subprocess - from bazel_tools.tools.python.runfiles import runfiles - - r = runfiles.Create() - env = {} - ... - env.update(r.EnvVars()) - p = subprocess.Popen([r.Rlocation("path/to/binary")], env, ...) """ import os From 1dc028c7975405d6d7386fc97aca7b3ab91d85be Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Fri, 20 Jan 2023 17:24:03 -0800 Subject: [PATCH 2/3] Apply suggestions from code review Co-authored-by: Richard Levasseur --- python/runfiles/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/runfiles/README.md b/python/runfiles/README.md index 997808463f..79ba82c1de 100644 --- a/python/runfiles/README.md +++ b/python/runfiles/README.md @@ -18,7 +18,7 @@ Typical Usage 3. Import the runfiles library. - import runfiles + import runfiles # not "from runfiles import runfiles" 4. Create a Runfiles object and use rlocation to look up runfile paths: @@ -38,8 +38,9 @@ Typical Usage r2 = runfiles.CreateDirectoryBased("path/to/foo.runfiles/") - If you want to start subprocesses that also need runfiles, you need to set - the right environment variables for them: + If you wnat to start subprocesses, and the subprocess can't automatically + find the correct runfiles directory, you can explicitly set the right + environment variables for them: import subprocess import runfiles From 17c6353506dc88d7f7855c4b0c48849a82393adf Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Fri, 20 Jan 2023 18:12:01 -0800 Subject: [PATCH 3/3] more code review cleanup --- python/runfiles/BUILD.bazel | 4 +++- python/runfiles/runfiles.py | 2 ++ tests/runfiles/runfiles_wheel_integration_test.sh | 10 ++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100755 tests/runfiles/runfiles_wheel_integration_test.sh diff --git a/python/runfiles/BUILD.bazel b/python/runfiles/BUILD.bazel index ba612fa33e..ea171ccd8e 100644 --- a/python/runfiles/BUILD.bazel +++ b/python/runfiles/BUILD.bazel @@ -30,7 +30,9 @@ py_library( visibility = ["//visibility:public"], ) -# TODO(alexeagle): add some example which depends on this wheel to verify it can be used +# This can be manually tested by running tests/runfiles/runfiles_wheel_integration_test.sh +# We ought to have an automated integration test for it, too. +# see https://github.com/bazelbuild/rules_python/issues/1002 py_wheel( name = "wheel", # From https://pypi.org/classifiers/ diff --git a/python/runfiles/runfiles.py b/python/runfiles/runfiles.py index ab08a9aa88..096040351a 100644 --- a/python/runfiles/runfiles.py +++ b/python/runfiles/runfiles.py @@ -13,6 +13,8 @@ # limitations under the License. """Runfiles lookup library for Bazel-built Python binaries and tests. + +See README.md for usage instructions. """ import os diff --git a/tests/runfiles/runfiles_wheel_integration_test.sh b/tests/runfiles/runfiles_wheel_integration_test.sh new file mode 100755 index 0000000000..7faa027909 --- /dev/null +++ b/tests/runfiles/runfiles_wheel_integration_test.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Manual test, run outside of Bazel, to check that our runfiles wheel should be functional +# for users who install it from pypi. +set -o errexit + +SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +bazel 2>/dev/null build --stamp --embed_label=1.2.3 //python/runfiles:wheel +wheelpath=$SCRIPTPATH/../../$(bazel 2>/dev/null cquery --output=files //python/runfiles:wheel) +PYTHONPATH=$wheelpath python3 -c 'import importlib;print(importlib.import_module("runfiles"))'