From f64817503d1bb2fdee024d24806d75af5d88c6f4 Mon Sep 17 00:00:00 2001 From: Ian O'Connell Date: Mon, 7 Mar 2022 17:30:06 -0700 Subject: [PATCH 1/6] Update parse.py Test --- gazelle/parse.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gazelle/parse.py b/gazelle/parse.py index dec3a16cc9..ff3fc7ef2f 100644 --- a/gazelle/parse.py +++ b/gazelle/parse.py @@ -13,7 +13,10 @@ def parse_import_statements(content, filepath): modules = list() - tree = ast.parse(content) + try: + tree = ast.parse(content) + except Exception as inst: + raise('cannot parse', filepath, inst) for node in ast.walk(tree): if isinstance(node, ast.Import): for subnode in node.names: From 70e93a8f6dde3319e4443ba18d9e292dde9d983a Mon Sep 17 00:00:00 2001 From: Ian O'Connell Date: Mon, 7 Mar 2022 17:33:36 -0700 Subject: [PATCH 2/6] Update parse.py --- gazelle/parse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gazelle/parse.py b/gazelle/parse.py index ff3fc7ef2f..cf59f4198d 100644 --- a/gazelle/parse.py +++ b/gazelle/parse.py @@ -16,7 +16,8 @@ def parse_import_statements(content, filepath): try: tree = ast.parse(content) except Exception as inst: - raise('cannot parse', filepath, inst) + print("cannot parse", filepath, inst) + raise inst for node in ast.walk(tree): if isinstance(node, ast.Import): for subnode in node.names: From 3c4c15aed59adfcf0edf2500eba0cbf6c358d957 Mon Sep 17 00:00:00 2001 From: Ian O'Connell Date: Mon, 7 Mar 2022 17:38:05 -0700 Subject: [PATCH 3/6] Update parse.py --- gazelle/parse.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gazelle/parse.py b/gazelle/parse.py index cf59f4198d..ca06ead9a1 100644 --- a/gazelle/parse.py +++ b/gazelle/parse.py @@ -16,8 +16,7 @@ def parse_import_statements(content, filepath): try: tree = ast.parse(content) except Exception as inst: - print("cannot parse", filepath, inst) - raise inst + raise Exception("cannot parse", filepath, inst) for node in ast.walk(tree): if isinstance(node, ast.Import): for subnode in node.names: From 55e5625756dcdf0ad37a88c2d30dbdd61f812f0c Mon Sep 17 00:00:00 2001 From: Ian O Connell Date: Wed, 1 Jun 2022 12:59:00 -0600 Subject: [PATCH 4/6] Revert "Init: pypa/installer (#700)" This reverts commit 9b8ab1ea442064ee5ae5cafc1b66137281813c81. --- python/pip_install/extract_wheels/lib/BUILD | 2 +- .../pip_install/extract_wheels/lib/bazel.py | 19 +++--- .../pip_install/extract_wheels/lib/wheel.py | 63 +++++++++++-------- python/pip_install/repositories.bzl | 10 +-- 4 files changed, 53 insertions(+), 41 deletions(-) diff --git a/python/pip_install/extract_wheels/lib/BUILD b/python/pip_install/extract_wheels/lib/BUILD index 48214126e4..3e2f307a73 100644 --- a/python/pip_install/extract_wheels/lib/BUILD +++ b/python/pip_install/extract_wheels/lib/BUILD @@ -18,7 +18,7 @@ py_library( "//python/pip_install/parse_requirements_to_bzl:__subpackages__", ], deps = [ - requirement("installer"), + requirement("pkginfo"), requirement("setuptools"), ], ) diff --git a/python/pip_install/extract_wheels/lib/bazel.py b/python/pip_install/extract_wheels/lib/bazel.py index 4b6a4ff0dd..9876e6cdf0 100644 --- a/python/pip_install/extract_wheels/lib/bazel.py +++ b/python/pip_install/extract_wheels/lib/bazel.py @@ -21,28 +21,29 @@ def generate_entry_point_contents( - module: str, attribute: str, shebang: str = "#!/usr/bin/env python3" + entry_point: str, shebang: str = "#!/usr/bin/env python3" ) -> str: """Generate the contents of an entry point script. Args: - module (str): The name of the module to use. - attribute (str): The name of the attribute to call. + entry_point (str): The name of the entry point as show in the + `console_scripts` section of `entry_point.txt`. shebang (str, optional): The shebang to use for the entry point python file. Returns: str: A string of python code. """ + module, method = entry_point.split(":", 1) return textwrap.dedent( """\ {shebang} import sys - from {module} import {attribute} + from {module} import {method} if __name__ == "__main__": - sys.exit({attribute}()) + sys.exit({method}()) """.format( - shebang=shebang, module=module, attribute=attribute + shebang=shebang, module=module, method=method ) ) @@ -407,14 +408,14 @@ def extract_wheel( directory_path = Path(directory) entry_points = [] - for name, (module, attribute) in sorted(whl.entry_points().items()): + for name, entry_point in sorted(whl.entry_points().items()): # There is an extreme edge-case with entry_points that end with `.py` # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174 entry_point_without_py = name[:-3] if name.endswith(".py") else name entry_point_target_name = f"{WHEEL_ENTRY_POINT_PREFIX}_{entry_point_without_py}" entry_point_script_name = f"{entry_point_target_name}.py" (directory_path / entry_point_script_name).write_text( - generate_entry_point_contents(module, attribute) + generate_entry_point_contents(entry_point) ) entry_points.append( generate_entry_point_rule( @@ -453,7 +454,7 @@ def extract_wheel( data_exclude=data_exclude, data=data, srcs_exclude=srcs_exclude, - tags=["pypi_name=" + whl.name, "pypi_version=" + whl.version], + tags=["pypi_name=" + whl.name, "pypi_version=" + whl.metadata.version], additional_content=additional_content, ) build_file.write(contents) diff --git a/python/pip_install/extract_wheels/lib/wheel.py b/python/pip_install/extract_wheels/lib/wheel.py index 73d5eb53da..85bc95830d 100644 --- a/python/pip_install/extract_wheels/lib/wheel.py +++ b/python/pip_install/extract_wheels/lib/wheel.py @@ -1,13 +1,13 @@ """Utility class to inspect an extracted wheel directory""" -import email +import configparser import glob import os import stat import zipfile from typing import Dict, Optional, Set -import installer import pkg_resources +import pkginfo def current_umask() -> int: @@ -37,46 +37,57 @@ def path(self) -> str: @property def name(self) -> str: - # TODO Also available as installer.sources.WheelSource.distribution - return str(self.metadata['Name']) + return str(self.metadata.name) @property - def metadata(self) -> email.message.Message: - with installer.sources.WheelFile.open(self.path) as wheel_source: - metadata_contents = wheel_source.read_dist_info("METADATA") - metadata = installer.utils.parse_metadata_file(metadata_contents) - return metadata + def metadata(self) -> pkginfo.Wheel: + return pkginfo.get_metadata(self.path) - @property - def version(self) -> str: - # TODO Also available as installer.sources.WheelSource.version - return str(self.metadata["Version"]) - - def entry_points(self) -> Dict[str, tuple[str, str]]: + def entry_points(self) -> Dict[str, str]: """Returns the entrypoints defined in the current wheel See https://packaging.python.org/specifications/entry-points/ for more info Returns: - Dict[str, Tuple[str, str]]: A mapping of the entry point's name to it's module and attribute + Dict[str, str]: A mappying of the entry point's name to it's method """ - with installer.sources.WheelFile.open(self.path) as wheel_source: - if "entry_points.txt" not in wheel_source.dist_info_filenames: + with zipfile.ZipFile(self.path, "r") as whl: + # Calculate the location of the entry_points.txt file + metadata = self.metadata + name = "{}-{}".format(metadata.name.replace("-", "_"), metadata.version) + + # Note that the zipfile module always uses the forward slash as + # directory separator, even on Windows, so don't use os.path.join + # here. Reference for Python 3.10: + # https://github.com/python/cpython/blob/3.10/Lib/zipfile.py#L355. + # TODO: use zipfile.Path once 3.8 is our minimum supported version + entry_points_path = "{}.dist-info/entry_points.txt".format(name) + + # If this file does not exist in the wheel, there are no entry points + if entry_points_path not in whl.namelist(): return dict() - entry_points_mapping = dict() - entry_points_contents = wheel_source.read_dist_info("entry_points.txt") - entry_points = installer.utils.parse_entrypoints(entry_points_contents) - for script, module, attribute, script_section in entry_points: - if script_section == "console": - entry_points_mapping[script] = (module, attribute) + # Parse the avaialble entry points + config = configparser.ConfigParser() + try: + config.read_string(whl.read(entry_points_path).decode("utf-8")) + if "console_scripts" in config.sections(): + return dict(config["console_scripts"]) + + # TODO: It's unclear what to do in a situation with duplicate sections or options. + # For now, we treat the config file as though it contains no scripts. For more + # details on the config parser, see: + # https://docs.python.org/3.7/library/configparser.html#configparser.ConfigParser + # https://docs.python.org/3.7/library/configparser.html#configparser.Error + except configparser.Error: + pass - return entry_points_mapping + return dict() def dependencies(self, extras_requested: Optional[Set[str]] = None) -> Set[str]: dependency_set = set() - for wheel_req in self.metadata.get_all('Requires-Dist', []): + for wheel_req in self.metadata.requires_dist: req = pkg_resources.Requirement(wheel_req) # type: ignore if req.marker is None or any( diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index a57ff9deaa..352f66341c 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -17,11 +17,6 @@ _RULE_DEPS = [ "https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl", "9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2", ), - ( - "pypi__installer", - "https://files.pythonhosted.org/packages/1b/21/3e6ebd12d8dccc55bcb7338db462c75ac86dbd0ac7439ac114616b21667b/installer-0.5.1-py3-none-any.whl", - "1d6c8d916ed82771945b9c813699e6f57424ded970c9d8bf16bbc23e1e826ed3", - ), ( "pypi__pip", "https://files.pythonhosted.org/packages/4d/16/0a14ca596f30316efd412a60bdfac02a7259bf8673d4d917dc60b9a21812/pip-22.0.4-py3-none-any.whl", @@ -32,6 +27,11 @@ _RULE_DEPS = [ "https://files.pythonhosted.org/packages/6d/16/75d65bdccd48bb59a08e2bf167b01d8532f65604270d0a292f0f16b7b022/pip_tools-5.5.0-py2.py3-none-any.whl", "10841c1e56c234d610d0466447685b9ea4ee4a2c274f858c0ef3c33d9bd0d985", ), + ( + "pypi__pkginfo", + "https://files.pythonhosted.org/packages/cd/00/49f59cdd2c6a52e6665fda4de671dac5614366dc827e050c55428241b929/pkginfo-1.8.2-py2.py3-none-any.whl", + "c24c487c6a7f72c66e816ab1796b96ac6c3d14d49338293d2141664330b55ffc", + ), ( "pypi__setuptools", "https://files.pythonhosted.org/packages/7c/5b/3d92b9f0f7ca1645cba48c080b54fe7d8b1033a4e5720091d1631c4266db/setuptools-60.10.0-py3-none-any.whl", From c58ba0e74698f21b6d9a4306c927ba0c12e1168f Mon Sep 17 00:00:00 2001 From: Ian O Connell Date: Wed, 1 Jun 2022 13:10:47 -0600 Subject: [PATCH 5/6] Revert "Updated `pip`, `pkginfo`, `setuptools`, and `wheel`. (#661)" This reverts commit fe30f1561e3afef59e3f8be6339c88444100de25. --- examples/pip_repository_annotations/WORKSPACE | 10 ---------- examples/relative_requirements/WORKSPACE | 9 --------- python/pip_install/repositories.bzl | 16 ++++++++-------- tests/pip_repository_entry_points/WORKSPACE | 12 ------------ 4 files changed, 8 insertions(+), 39 deletions(-) diff --git a/examples/pip_repository_annotations/WORKSPACE b/examples/pip_repository_annotations/WORKSPACE index eb712cfc6b..8c0146139e 100644 --- a/examples/pip_repository_annotations/WORKSPACE +++ b/examples/pip_repository_annotations/WORKSPACE @@ -16,14 +16,6 @@ http_archive( ], ) -load("@rules_python//python:repositories.bzl", "python_register_toolchains") - -python_register_toolchains( - name = "python39", - python_version = "3.9", -) - -load("@python39//:defs.bzl", "interpreter") load("@rules_python//python:pip.bzl", "package_annotation", "pip_install", "pip_parse") # Here we can see an example of annotations being applied to an arbitrary @@ -50,7 +42,6 @@ write_file( pip_parse( name = "pip_parsed", annotations = ANNOTATIONS, - python_interpreter_target = interpreter, requirements_lock = "//:requirements.txt", ) @@ -62,6 +53,5 @@ install_deps() pip_install( name = "pip_installed", annotations = ANNOTATIONS, - python_interpreter_target = interpreter, requirements = "//:requirements.txt", ) diff --git a/examples/relative_requirements/WORKSPACE b/examples/relative_requirements/WORKSPACE index 4ae91c39d8..0f2da3131c 100644 --- a/examples/relative_requirements/WORKSPACE +++ b/examples/relative_requirements/WORKSPACE @@ -5,17 +5,8 @@ local_repository( path = "../..", ) -load("@rules_python//python:repositories.bzl", "python_register_toolchains") - -python_register_toolchains( - name = "python39", - python_version = "3.9", -) - -load("@python39//:defs.bzl", "interpreter") load("@rules_python//python:pip.bzl", "pip_install") pip_install( - python_interpreter_target = interpreter, requirements = "//:requirements.txt", ) diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index 352f66341c..942711d0c2 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -19,8 +19,8 @@ _RULE_DEPS = [ ), ( "pypi__pip", - "https://files.pythonhosted.org/packages/4d/16/0a14ca596f30316efd412a60bdfac02a7259bf8673d4d917dc60b9a21812/pip-22.0.4-py3-none-any.whl", - "c6aca0f2f081363f689f041d90dab2a07a9a07fb840284db2218117a52da800b", + "https://files.pythonhosted.org/packages/47/ca/f0d790b6e18b3a6f3bd5e80c2ee4edbb5807286c21cdd0862ca933f751dd/pip-21.1.3-py3-none-any.whl", + "78cb760711fedc073246543801c84dc5377affead832e103ad0211f99303a204", ), ( "pypi__pip_tools", @@ -29,18 +29,18 @@ _RULE_DEPS = [ ), ( "pypi__pkginfo", - "https://files.pythonhosted.org/packages/cd/00/49f59cdd2c6a52e6665fda4de671dac5614366dc827e050c55428241b929/pkginfo-1.8.2-py2.py3-none-any.whl", - "c24c487c6a7f72c66e816ab1796b96ac6c3d14d49338293d2141664330b55ffc", + "https://files.pythonhosted.org/packages/77/83/1ef010f7c4563e218854809c0dff9548de65ebec930921dedf6ee5981f27/pkginfo-1.7.1-py2.py3-none-any.whl", + "37ecd857b47e5f55949c41ed061eb51a0bee97a87c969219d144c0e023982779", ), ( "pypi__setuptools", - "https://files.pythonhosted.org/packages/7c/5b/3d92b9f0f7ca1645cba48c080b54fe7d8b1033a4e5720091d1631c4266db/setuptools-60.10.0-py3-none-any.whl", - "782ef48d58982ddb49920c11a0c5c9c0b02e7d7d1c2ad0aa44e1a1e133051c96", + "https://files.pythonhosted.org/packages/a2/e1/902fbc2f61ad6243cd3d57ffa195a9eb123021ec912ec5d811acf54a39f8/setuptools-57.1.0-py3-none-any.whl", + "ddae4c1b9220daf1e32ba9d4e3714df6019c5b583755559be84ff8199f7e1fe3", ), ( "pypi__wheel", - "https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl", - "4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a", + "https://files.pythonhosted.org/packages/65/63/39d04c74222770ed1589c0eaba06c05891801219272420b40311cd60c880/wheel-0.36.2-py2.py3-none-any.whl", + "78b5b185f0e5763c26ca1e324373aadd49182ca90e825f7853f4b2509215dc0e", ), ] diff --git a/tests/pip_repository_entry_points/WORKSPACE b/tests/pip_repository_entry_points/WORKSPACE index 07a5d3aad0..5b893f5319 100644 --- a/tests/pip_repository_entry_points/WORKSPACE +++ b/tests/pip_repository_entry_points/WORKSPACE @@ -5,22 +5,11 @@ local_repository( path = "../..", ) -load("@rules_python//python:repositories.bzl", "python_register_toolchains") - -# This toolchain is explicitly 3.10 while `rules_python` is 3.9 to act as -# a regression test, ensuring 3.10 is functional -python_register_toolchains( - name = "python310", - python_version = "3.10", -) - -load("@python310//:defs.bzl", "interpreter") load("@rules_python//python:pip.bzl", "pip_install", "pip_parse") # For a more thorough example of `pip_parse`. See `@rules_python//examples/pip_parse` pip_parse( name = "pip_parsed", - python_interpreter_target = interpreter, requirements_lock = "//:requirements.txt", ) @@ -31,6 +20,5 @@ install_deps() # For a more thorough example of `pip_install`. See `@rules_python//examples/pip_install` pip_install( name = "pip_installed", - python_interpreter_target = interpreter, requirements = "//:requirements.txt", ) From cc35d2a20b7b704806417d4c0e44d895657d03d7 Mon Sep 17 00:00:00 2001 From: Ian O Connell Date: Wed, 1 Jun 2022 13:29:47 -0600 Subject: [PATCH 6/6] Try comment this to unblock things --- python/pip_install/extract_wheels/lib/bazel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pip_install/extract_wheels/lib/bazel.py b/python/pip_install/extract_wheels/lib/bazel.py index 9876e6cdf0..93453e8c44 100644 --- a/python/pip_install/extract_wheels/lib/bazel.py +++ b/python/pip_install/extract_wheels/lib/bazel.py @@ -142,7 +142,7 @@ def generate_build_file_contents( # RECORD is known to contain sha256 checksums of files which might include the checksums # of generated files produced when wheels are installed. The file is ignored to avoid # Bazel caching issues. - "**/*.dist-info/RECORD", + # "**/*.dist-info/RECORD", ] data_exclude = list(