From d7106ec720db37bda4990fd74209267d454bbda6 Mon Sep 17 00:00:00 2001 From: Shai Nagar Date: Wed, 3 Jul 2019 12:22:00 +0300 Subject: [PATCH 1/5] added the ability to specify a custom wheel package root --- experimental/examples/wheel/BUILD | 14 ++++++++++++++ experimental/examples/wheel/wheel_test.py | 17 +++++++++++++++++ experimental/python/wheel.bzl | 5 +++++ experimental/rules_python/wheelmaker.py | 22 +++++++++++++++++++--- 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/experimental/examples/wheel/BUILD b/experimental/examples/wheel/BUILD index d7ed5da49c..742ca6d402 100644 --- a/experimental/examples/wheel/BUILD +++ b/experimental/examples/wheel/BUILD @@ -85,6 +85,19 @@ py_wheel( deps = [":example_pkg"], ) +# An example of how to change the wheel package root directory. +py_wheel( + name = "custom_package_root", + # Package data. We're building "custom_package_root-0.0.1-py3-none-any.whl" + distribution = "example_custom_package_root", + python_tag = "py3", + version = "0.0.1", + deps = [ + ":example_pkg" + ], + package_root_path = "experimental" +) + py_test( name = "wheel_test", srcs = ["wheel_test.py"], @@ -92,5 +105,6 @@ py_test( ":customized", ":minimal_with_py_library", ":minimal_with_py_package", + ":custom_package_root", ], ) diff --git a/experimental/examples/wheel/wheel_test.py b/experimental/examples/wheel/wheel_test.py index 278f00726a..51d2f83150 100644 --- a/experimental/examples/wheel/wheel_test.py +++ b/experimental/examples/wheel/wheel_test.py @@ -102,6 +102,23 @@ def test_customized_wheel(self): This is a sample description of a wheel. """) + def test_custom_package_root_path_wheel(self): + filename = os.path.join(os.environ['TEST_SRCDIR'], + 'io_bazel_rules_python', 'experimental', + 'examples', 'wheel', + 'example_custom_package_root-0.0.1-py3-none-any.whl') + + with zipfile.ZipFile(filename) as zf: + self.assertEquals( + zf.namelist(), + ['examples/wheel/lib/data.txt', + 'examples/wheel/lib/module_with_data.py', + 'examples/wheel/lib/simple_module.py', + 'examples/wheel/main.py', + 'example_custom_package_root-0.0.1.dist-info/WHEEL', + 'example_custom_package_root-0.0.1.dist-info/METADATA', + 'example_custom_package_root-0.0.1.dist-info/RECORD']) + if __name__ == '__main__': unittest.main() diff --git a/experimental/python/wheel.bzl b/experimental/python/wheel.bzl index 75c4711d6d..c5b933ef01 100644 --- a/experimental/python/wheel.bzl +++ b/experimental/python/wheel.bzl @@ -102,6 +102,7 @@ def _py_wheel_impl(ctx): args.add("--abi", ctx.attr.abi) args.add("--platform", ctx.attr.platform) args.add("--out", outfile.path) + args.add("--package_root_path", ctx.attr.package_root_path) args.add_all(inputs_to_package, format_each = "--input_file=%s", map_each = _input_file_to_arg) @@ -242,6 +243,10 @@ to refer to the package in other packages' dependencies. "license": attr.string(default = ""), "classifiers": attr.string_list(), "description_file": attr.label(allow_single_file = True), + "package_root_path": attr.string( + default = "none", + doc = "path to the root of the generated package (default is workspace path)", + ), # Requirements "requires": attr.string_list( doc = "List of requirements for this package", diff --git a/experimental/rules_python/wheelmaker.py b/experimental/rules_python/wheelmaker.py index 7a50ae4a20..c0c8ab8300 100644 --- a/experimental/rules_python/wheelmaker.py +++ b/experimental/rules_python/wheelmaker.py @@ -35,7 +35,7 @@ def commonpath(path1, path2): class WheelMaker(object): def __init__(self, name, version, build_tag, python_tag, abi, platform, - outfile=None): + outfile=None, package_root_path=None): self._name = name self._version = version self._build_tag = build_tag @@ -43,6 +43,7 @@ def __init__(self, name, version, build_tag, python_tag, abi, platform, self._abi = abi self._platform = platform self._outfile = outfile + self._package_root_path = package_root_path if package_root_path != "none" else None self._zipfile = None self._record = [] @@ -93,8 +94,18 @@ def add_string(self, filename, contents): def add_file(self, package_filename, real_filename): """Add given file to the distribution.""" + def strip_prefix(path, prefix): + if path.startswith(prefix): + return path[len(prefix):] + else: + return path + + resolved_package_filename = package_filename \ + if self._package_root_path is None \ + else strip_prefix(package_filename, self._package_root_path) + # Always use unix path separators. - arcname = package_filename.replace(os.path.sep, '/') + arcname = resolved_package_filename.replace(os.path.sep, '/') self._zipfile.write(real_filename, arcname=arcname) # Find the hash and length hash = hashlib.sha256() @@ -208,6 +219,9 @@ def main(): output_group.add_argument('--out', type=str, default=None, help="Override name of ouptut file") + output_group.add_argument('--package_root_path', type=str, default=None, + help="File path prefix to strip from input package files") + wheel_group = parser.add_argument_group("Wheel metadata") wheel_group.add_argument( '--header', action='append', @@ -254,7 +268,9 @@ def main(): python_tag=arguments.python_tag, abi=arguments.abi, platform=arguments.platform, - outfile=arguments.out) as maker: + outfile=arguments.out, + package_root_path=arguments.package_root_path, + ) as maker: for package_filename, real_filename in all_files: maker.add_file(package_filename, real_filename) maker.add_wheelfile() From a3e6439229125152ac7ab95c42725c327261a56f Mon Sep 17 00:00:00 2001 From: Shai Nagar Date: Tue, 9 Jul 2019 07:14:19 +0300 Subject: [PATCH 2/5] renamed 'package_root_path' to 'strip_path_prefix' --- experimental/examples/wheel/BUILD | 4 ++-- experimental/examples/wheel/wheel_test.py | 2 +- experimental/python/wheel.bzl | 4 ++-- experimental/rules_python/wheelmaker.py | 18 ++++++++---------- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/experimental/examples/wheel/BUILD b/experimental/examples/wheel/BUILD index 742ca6d402..83d65639e8 100644 --- a/experimental/examples/wheel/BUILD +++ b/experimental/examples/wheel/BUILD @@ -85,7 +85,7 @@ py_wheel( deps = [":example_pkg"], ) -# An example of how to change the wheel package root directory. +# An example of how to change the wheel package root directory using 'strip_path_prefix'. py_wheel( name = "custom_package_root", # Package data. We're building "custom_package_root-0.0.1-py3-none-any.whl" @@ -95,7 +95,7 @@ py_wheel( deps = [ ":example_pkg" ], - package_root_path = "experimental" + strip_path_prefix = "experimental" ) py_test( diff --git a/experimental/examples/wheel/wheel_test.py b/experimental/examples/wheel/wheel_test.py index 51d2f83150..4b0874c5b2 100644 --- a/experimental/examples/wheel/wheel_test.py +++ b/experimental/examples/wheel/wheel_test.py @@ -102,7 +102,7 @@ def test_customized_wheel(self): This is a sample description of a wheel. """) - def test_custom_package_root_path_wheel(self): + def test_custom_package_root_wheel(self): filename = os.path.join(os.environ['TEST_SRCDIR'], 'io_bazel_rules_python', 'experimental', 'examples', 'wheel', diff --git a/experimental/python/wheel.bzl b/experimental/python/wheel.bzl index c5b933ef01..b157daa8f7 100644 --- a/experimental/python/wheel.bzl +++ b/experimental/python/wheel.bzl @@ -102,7 +102,7 @@ def _py_wheel_impl(ctx): args.add("--abi", ctx.attr.abi) args.add("--platform", ctx.attr.platform) args.add("--out", outfile.path) - args.add("--package_root_path", ctx.attr.package_root_path) + args.add("--strip_path_prefix", ctx.attr.strip_path_prefix) args.add_all(inputs_to_package, format_each = "--input_file=%s", map_each = _input_file_to_arg) @@ -243,7 +243,7 @@ to refer to the package in other packages' dependencies. "license": attr.string(default = ""), "classifiers": attr.string_list(), "description_file": attr.label(allow_single_file = True), - "package_root_path": attr.string( + "strip_path_prefix": attr.string( default = "none", doc = "path to the root of the generated package (default is workspace path)", ), diff --git a/experimental/rules_python/wheelmaker.py b/experimental/rules_python/wheelmaker.py index c0c8ab8300..64a2156cbf 100644 --- a/experimental/rules_python/wheelmaker.py +++ b/experimental/rules_python/wheelmaker.py @@ -35,7 +35,7 @@ def commonpath(path1, path2): class WheelMaker(object): def __init__(self, name, version, build_tag, python_tag, abi, platform, - outfile=None, package_root_path=None): + outfile=None, strip_path_prefix=None): self._name = name self._version = version self._build_tag = build_tag @@ -43,7 +43,7 @@ def __init__(self, name, version, build_tag, python_tag, abi, platform, self._abi = abi self._platform = platform self._outfile = outfile - self._package_root_path = package_root_path if package_root_path != "none" else None + self._strip_path_prefix = strip_path_prefix if strip_path_prefix != "none" else None self._zipfile = None self._record = [] @@ -95,14 +95,12 @@ def add_string(self, filename, contents): def add_file(self, package_filename, real_filename): """Add given file to the distribution.""" def strip_prefix(path, prefix): - if path.startswith(prefix): - return path[len(prefix):] - else: + if prefix is None or not path.startswith(prefix): return path + else: + return path[len(prefix):] - resolved_package_filename = package_filename \ - if self._package_root_path is None \ - else strip_prefix(package_filename, self._package_root_path) + resolved_package_filename = strip_prefix(package_filename, self._strip_path_prefix) # Always use unix path separators. arcname = resolved_package_filename.replace(os.path.sep, '/') @@ -219,7 +217,7 @@ def main(): output_group.add_argument('--out', type=str, default=None, help="Override name of ouptut file") - output_group.add_argument('--package_root_path', type=str, default=None, + output_group.add_argument('--strip_path_prefix', type=str, default=None, help="File path prefix to strip from input package files") wheel_group = parser.add_argument_group("Wheel metadata") @@ -269,7 +267,7 @@ def main(): abi=arguments.abi, platform=arguments.platform, outfile=arguments.out, - package_root_path=arguments.package_root_path, + strip_path_prefix=arguments.strip_path_prefix, ) as maker: for package_filename, real_filename in all_files: maker.add_file(package_filename, real_filename) From f00ef526dd44120b6b86a52da14e5a01a8240b62 Mon Sep 17 00:00:00 2001 From: Shai Nagar Date: Tue, 9 Jul 2019 08:05:07 +0300 Subject: [PATCH 3/5] changed 'strip_path_prefix' to 'strip_path_prefixes' which is now an ordered list of prefixes to strip from package file paths --- experimental/examples/wheel/BUILD | 38 +++++++++++++++++++++-- experimental/examples/wheel/wheel_test.py | 34 ++++++++++++++++++++ experimental/python/wheel.bzl | 8 ++--- experimental/rules_python/wheelmaker.py | 31 +++++++++++------- 4 files changed, 94 insertions(+), 17 deletions(-) diff --git a/experimental/examples/wheel/BUILD b/experimental/examples/wheel/BUILD index 83d65639e8..f0f6984621 100644 --- a/experimental/examples/wheel/BUILD +++ b/experimental/examples/wheel/BUILD @@ -85,7 +85,7 @@ py_wheel( deps = [":example_pkg"], ) -# An example of how to change the wheel package root directory using 'strip_path_prefix'. +# An example of how to change the wheel package root directory using 'strip_path_prefixes'. py_wheel( name = "custom_package_root", # Package data. We're building "custom_package_root-0.0.1-py3-none-any.whl" @@ -95,7 +95,39 @@ py_wheel( deps = [ ":example_pkg" ], - strip_path_prefix = "experimental" + strip_path_prefixes = [ + "experimental" + ] +) + +py_wheel( + name = "custom_package_root_multi_prefix", + # Package data. We're building "custom_custom_package_root_multi_prefix-0.0.1-py3-none-any.whl" + distribution = "example_custom_package_root_multi_prefix", + python_tag = "py3", + version = "0.0.1", + deps = [ + ":example_pkg" + ], + strip_path_prefixes = [ + "experimental/examples/wheel/lib", + "experimental/examples/wheel" , + ] +) + +py_wheel( + name = "custom_package_root_multi_prefix_reverse_order", + # Package data. We're building "custom_custom_package_root_multi_prefix_reverse_order-0.0.1-py3-none-any.whl" + distribution = "example_custom_package_root_multi_prefix_reverse_order", + python_tag = "py3", + version = "0.0.1", + deps = [ + ":example_pkg" + ], + strip_path_prefixes = [ + "experimental/examples/wheel" , + "experimental/examples/wheel/lib", # this is not effective, because the first prefix takes priority + ] ) py_test( @@ -106,5 +138,7 @@ py_test( ":minimal_with_py_library", ":minimal_with_py_package", ":custom_package_root", + ":custom_package_root_multi_prefix", + ":custom_package_root_multi_prefix_reverse_order", ], ) diff --git a/experimental/examples/wheel/wheel_test.py b/experimental/examples/wheel/wheel_test.py index 4b0874c5b2..d7bb8b6b4e 100644 --- a/experimental/examples/wheel/wheel_test.py +++ b/experimental/examples/wheel/wheel_test.py @@ -119,6 +119,40 @@ def test_custom_package_root_wheel(self): 'example_custom_package_root-0.0.1.dist-info/METADATA', 'example_custom_package_root-0.0.1.dist-info/RECORD']) + def test_custom_package_root_multi_prefix_wheel(self): + filename = os.path.join(os.environ['TEST_SRCDIR'], + 'io_bazel_rules_python', 'experimental', + 'examples', 'wheel', + 'example_custom_package_root_multi_prefix-0.0.1-py3-none-any.whl') + + with zipfile.ZipFile(filename) as zf: + self.assertEquals( + zf.namelist(), + ['data.txt', + 'module_with_data.py', + 'simple_module.py', + 'main.py', + 'example_custom_package_root_multi_prefix-0.0.1.dist-info/WHEEL', + 'example_custom_package_root_multi_prefix-0.0.1.dist-info/METADATA', + 'example_custom_package_root_multi_prefix-0.0.1.dist-info/RECORD']) + + def test_custom_package_root_multi_prefix_reverse_order_wheel(self): + filename = os.path.join(os.environ['TEST_SRCDIR'], + 'io_bazel_rules_python', 'experimental', + 'examples', 'wheel', + 'example_custom_package_root_multi_prefix_reverse_order-0.0.1-py3-none-any.whl') + + with zipfile.ZipFile(filename) as zf: + self.assertEquals( + zf.namelist(), + ['lib/data.txt', + 'lib/module_with_data.py', + 'lib/simple_module.py', + 'main.py', + 'example_custom_package_root_multi_prefix_reverse_order-0.0.1.dist-info/WHEEL', + 'example_custom_package_root_multi_prefix_reverse_order-0.0.1.dist-info/METADATA', + 'example_custom_package_root_multi_prefix_reverse_order-0.0.1.dist-info/RECORD']) + if __name__ == '__main__': unittest.main() diff --git a/experimental/python/wheel.bzl b/experimental/python/wheel.bzl index b157daa8f7..1c43409cd5 100644 --- a/experimental/python/wheel.bzl +++ b/experimental/python/wheel.bzl @@ -102,7 +102,7 @@ def _py_wheel_impl(ctx): args.add("--abi", ctx.attr.abi) args.add("--platform", ctx.attr.platform) args.add("--out", outfile.path) - args.add("--strip_path_prefix", ctx.attr.strip_path_prefix) + args.add_all(ctx.attr.strip_path_prefixes, format_each = "--strip_path_prefix=%s") args.add_all(inputs_to_package, format_each = "--input_file=%s", map_each = _input_file_to_arg) @@ -243,9 +243,9 @@ to refer to the package in other packages' dependencies. "license": attr.string(default = ""), "classifiers": attr.string_list(), "description_file": attr.label(allow_single_file = True), - "strip_path_prefix": attr.string( - default = "none", - doc = "path to the root of the generated package (default is workspace path)", + "strip_path_prefixes": attr.string_list( + default = [], + doc = "path prefixes to strip from files added to the generated package", ), # Requirements "requires": attr.string_list( diff --git a/experimental/rules_python/wheelmaker.py b/experimental/rules_python/wheelmaker.py index 64a2156cbf..95c07342aa 100644 --- a/experimental/rules_python/wheelmaker.py +++ b/experimental/rules_python/wheelmaker.py @@ -35,7 +35,7 @@ def commonpath(path1, path2): class WheelMaker(object): def __init__(self, name, version, build_tag, python_tag, abi, platform, - outfile=None, strip_path_prefix=None): + outfile=None, strip_path_prefixes=None): self._name = name self._version = version self._build_tag = build_tag @@ -43,7 +43,7 @@ def __init__(self, name, version, build_tag, python_tag, abi, platform, self._abi = abi self._platform = platform self._outfile = outfile - self._strip_path_prefix = strip_path_prefix if strip_path_prefix != "none" else None + self._strip_path_prefixes = strip_path_prefixes if strip_path_prefixes is not None else [] self._zipfile = None self._record = [] @@ -94,13 +94,14 @@ def add_string(self, filename, contents): def add_file(self, package_filename, real_filename): """Add given file to the distribution.""" - def strip_prefix(path, prefix): - if prefix is None or not path.startswith(prefix): - return path - else: - return path[len(prefix):] + def strip_prefix(path, prefixes): + for prefix in prefixes: + if path.startswith(prefix): + return path[len(prefix):] + + return path - resolved_package_filename = strip_prefix(package_filename, self._strip_path_prefix) + resolved_package_filename = strip_prefix(package_filename, self._strip_path_prefixes) # Always use unix path separators. arcname = resolved_package_filename.replace(os.path.sep, '/') @@ -217,8 +218,14 @@ def main(): output_group.add_argument('--out', type=str, default=None, help="Override name of ouptut file") - output_group.add_argument('--strip_path_prefix', type=str, default=None, - help="File path prefix to strip from input package files") + output_group.add_argument('--strip_path_prefix', + type=str, + action="append", + default=[], + help="Path prefix to be stripped from input package files' path. " + "Can be supplied multiple times. " + "Evaluated in order." + ) wheel_group = parser.add_argument_group("Wheel metadata") wheel_group.add_argument( @@ -260,6 +267,8 @@ def main(): # Sort the files for reproducible order in the archive. all_files = sorted(all_files.items()) + strip_prefixes = [p for p in arguments.strip_path_prefix] + with WheelMaker(name=arguments.name, version=arguments.version, build_tag=arguments.build_tag, @@ -267,7 +276,7 @@ def main(): abi=arguments.abi, platform=arguments.platform, outfile=arguments.out, - strip_path_prefix=arguments.strip_path_prefix, + strip_path_prefixes=strip_prefixes, ) as maker: for package_filename, real_filename in all_files: maker.add_file(package_filename, real_filename) From 117a6f8cb8b97e7292fb84e9cd50f19560d751c8 Mon Sep 17 00:00:00 2001 From: Shai Nagar Date: Tue, 9 Jul 2019 08:25:00 +0300 Subject: [PATCH 4/5] removed trailing commas --- experimental/examples/wheel/BUILD | 8 ++++---- experimental/rules_python/wheelmaker.py | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/experimental/examples/wheel/BUILD b/experimental/examples/wheel/BUILD index f0f6984621..5c3bff6690 100644 --- a/experimental/examples/wheel/BUILD +++ b/experimental/examples/wheel/BUILD @@ -111,7 +111,7 @@ py_wheel( ], strip_path_prefixes = [ "experimental/examples/wheel/lib", - "experimental/examples/wheel" , + "experimental/examples/wheel" ] ) @@ -126,7 +126,7 @@ py_wheel( ], strip_path_prefixes = [ "experimental/examples/wheel" , - "experimental/examples/wheel/lib", # this is not effective, because the first prefix takes priority + "experimental/examples/wheel/lib" # this is not effective, because the first prefix takes priority ] ) @@ -139,6 +139,6 @@ py_test( ":minimal_with_py_package", ":custom_package_root", ":custom_package_root_multi_prefix", - ":custom_package_root_multi_prefix_reverse_order", - ], + ":custom_package_root_multi_prefix_reverse_order" + ] ) diff --git a/experimental/rules_python/wheelmaker.py b/experimental/rules_python/wheelmaker.py index 95c07342aa..482c1508dd 100644 --- a/experimental/rules_python/wheelmaker.py +++ b/experimental/rules_python/wheelmaker.py @@ -15,9 +15,7 @@ import argparse import base64 import collections -import csv import hashlib -import io import os import os.path import sys @@ -276,7 +274,7 @@ def main(): abi=arguments.abi, platform=arguments.platform, outfile=arguments.out, - strip_path_prefixes=strip_prefixes, + strip_path_prefixes=strip_prefixes ) as maker: for package_filename, real_filename in all_files: maker.add_file(package_filename, real_filename) From 49ccf407b46794fb13322960b1c32091373d6186 Mon Sep 17 00:00:00 2001 From: Shai Nagar Date: Tue, 9 Jul 2019 09:02:42 +0300 Subject: [PATCH 5/5] normalizing archive path separator before stripping prefix --- experimental/rules_python/wheelmaker.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/experimental/rules_python/wheelmaker.py b/experimental/rules_python/wheelmaker.py index 482c1508dd..ca3279f3ff 100644 --- a/experimental/rules_python/wheelmaker.py +++ b/experimental/rules_python/wheelmaker.py @@ -92,17 +92,17 @@ def add_string(self, filename, contents): def add_file(self, package_filename, real_filename): """Add given file to the distribution.""" - def strip_prefix(path, prefixes): - for prefix in prefixes: - if path.startswith(prefix): - return path[len(prefix):] + def arcname_from(name): + # Always use unix path separators. + normalized_arcname = name.replace(os.path.sep, '/') + for prefix in self._strip_path_prefixes: + if normalized_arcname.startswith(prefix): + return normalized_arcname[len(prefix):] - return path + return normalized_arcname - resolved_package_filename = strip_prefix(package_filename, self._strip_path_prefixes) + arcname = arcname_from(package_filename) - # Always use unix path separators. - arcname = resolved_package_filename.replace(os.path.sep, '/') self._zipfile.write(real_filename, arcname=arcname) # Find the hash and length hash = hashlib.sha256()