8000 Fix unnecessary namespace directory creation (#54) · satishhiremath/rules_python@cdc0305 · GitHub
[go: up one dir, main page]
Skip to content

Commit cdc0305

Browse files
dillon-giacoppodillon-giacoppo
authored
Fix unnecessary namespace directory creation (bazel-contrib#54)
1 parent bd1f4be commit cdc0305

File tree

9 files changed

+117
-187
lines changed

9 files changed

+117
-187
lines changed

‎example/BUILD

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,29 @@
11
load("@pip//:requirements.bzl", "requirement")
2-
load("@rules_python//python:defs.bzl", "py_runtime_pair")
32

43
# Toolchain setup, this is optional.
54
# Demonstrate that we can use the same python interpreter for the toolchain and executing pip in pip install (see WORKSPACE).
6-
py_runtime(
7-
name = "python3_runtime",
8-
files = ["@python_interpreter//:files"],
9-
interpreter = "@python_interpreter//:python_bin",
10-
python_version = "PY3",
11-
visibility = ["//visibility:public"],
12-
)
13-
14-
py_runtime_pair(
15-
name = "my_py_runtime_pair",
16-
py2_runtime = None,
17-
py3_runtime = ":python3_runtime",
18-
)
19-
20-
toolchain(
21-
name = "my_py_toolchain",
22-
toolchain = ":my_py_runtime_pair",
23-
toolchain_type = "@bazel_tools//tools/python:toolchain_type",
24-
)
5+
#
6+
#load("@rules_python//python:defs.bzl", "py_runtime_pair")
7+
#
8+
#py_runtime(
9+
# name = "python3_runtime",
10+
# files = ["@python_interpreter//:files"],
11+
# interpreter = "@python_interpreter//:python_bin",
12+
# python_version = "PY3",
13+
# visibility = ["//visibility:public"],
14+
#)
15+
#
16+
#py_runtime_pair(
17+
# name = "my_py_runtime_pair",
18+
# py2_runtime = None,
19+
# py3_runtime = ":python3_runtime",
20+
#)
21+
#
22+
#toolchain(
23+
# name = "my_py_toolchain",
24+
# toolchain = ":my_py_runtime_pair",
25+
# toolchain_type = "@bazel_tools//tools/python:toolchain_type",
26+
#)
2527
# End of toolchain setup.
2628

2729
py_binary(

‎example/WORKSPACE

Lines changed: 43 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -4,52 +4,15 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
44

55
http_archive(
66
name = "rules_python",
7-
sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161",
8-
url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz",
7+
url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.2/rules_python-0.0.2.tar.gz",
8+
strip_prefix = "rules_python-0.0.2",
9+
sha256 = "b5668cde8bb6e3515057ef465a35ad712214962f0b3a314e551204266c7be90c",
910
)
1011

1112
load("@rules_python//python:repositories.bzl", "py_repositories")
1213

1314
py_repositories()
1415

15-
# You could optionally use an in-build, compiled python interpreter as a toolchain,
16-
# and also use it to execute pip.
17-
18-
# Special logic for building python interpreter with OpenSSL from homebrew.
19-
# See https://devguide.python.org/setup/#macos-and-os-x
20-
_py_configure = """
21-
if [[ "$OSTYPE" == "darwin"* ]]; then
22-
./configure --prefix=$(pwd)/bazel_install --with-openssl=$(brew --prefix openssl)
23-
else
24-
./configure --prefix=$(pwd)/bazel_install
25-
fi
26-
"""
27-
28-
# NOTE: you need to have the SSL headers installed to build with openssl support (and use HTTPS).
29-
# E.g. on Ubuntu: `sudo apt install libssl-dev`
30-
http_archive(
31-
name = "python_interpreter",
32-
build_file_content = """
33-
exports_files(["python_bin"])
34-
filegroup(
35-
name = "files",
36-
srcs = glob(["bazel_install/**"], exclude = ["**/* *"]),
37-
visibility = ["//visibility:public"],
38-
)
39-
""",
40-
patch_cmds = [
41-
"mkdir $(pwd)/bazel_install",
42-
_py_configure,
43-
"make",
44-
"make install",
45-
"ln -s bazel_install/bin/python3 python_bin",
46-
],
47-
sha256 = "dfab5ec723c218082fe3d5d7ae17ecbdebffa9a1aea4d64aa3a2ecdd2e795864",
48-
strip_prefix = "Python-3.8.3",
49-
urls = ["https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tar.xz"],
50-
)
51-
# End of in-build Python interpreter setup.
52-
5316
local_repository(
5417
name = "rules_python_external",
5518
path = "../",
@@ -75,7 +38,7 @@ pip_install(
7538
# 1. Python interpreter that you compile in the build file (as above in @python_interpreter).
7639
# 2. Pre-compiled python interpreter included with http_archive
7740
# 3. Wrapper script, like in the autodetecting python toolchain.
78-
python_interpreter_target = "@python_interpreter//:python_bin",
41+
#python_interpreter_target = "@python_interpreter//:python_bin",
7942

8043
# (Optional) You can set quiet to False if you want to see pip output.
8144
#quiet = False,
@@ -84,6 +47,44 @@ pip_install(
8447
requirements = "//:requirements.txt",
8548
)
8649

50+
# You could optionally use an in-build, compiled python interpreter as a toolchain,
51+
# and also use it to execute pip.
52+
#
53+
# Special logic for building python interpreter with OpenSSL from homebrew.
54+
# See https://devguide.python.org/setup/#macos-and-os-x
55+
#_py_configure = """
56+
#if [[ "$OSTYPE" == "darwin"* ]]; then
57+
# ./configure --prefix=$(pwd)/bazel_install --with-openssl=$(brew --prefix openssl)
58+
#else
59+
# ./configure --prefix=$(pwd)/bazel_install
60+
#fi
61+
#"""
62+
#
63+
# NOTE: you need to have the SSL headers installed to build with openssl support (and use HTTPS).
64+
# E.g. on Ubuntu: `sudo apt install libssl-dev`
65+
#http_archive(
66+
# name = "python_interpreter",
67+
# build_file_content = """
68+
#exports_files(["python_bin"])
69+
#filegroup(
70+
# name = "files",
71+
# srcs = glob(["bazel_install/**"], exclude = ["**/* *"]),
72+
# visibility = ["//visibility:public"],
73+
#)
74+
#""",
75+
# patch_cmds = [
76+
# "mkdir $(pwd)/bazel_install",
77+
# _py_configure,
78+
# "make",
10000 79+
# "make install",
80+
# "ln -s bazel_install/bin/python3 python_bin",
81+
# ],
82+
# sha256 = "dfab5ec723c218082fe3d5d7ae17ecbdebffa9a1aea4d64aa3a2ecdd2e795864",
83+
# strip_prefix = "Python-3.8.3",
84+
# urls = ["https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tar.xz"],
85+
#)
86+
8787
# Optional:
8888
# Register the toolchain with the same python interpreter we used for pip in pip_install().
89-
register_toolchains("//:my_py_toolchain")
89+
#register_toolchains("//:my_py_toolchain")
90+
# End of in-build Python interpreter setup.

‎example/main.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
import boto3
22

3-
print("Hello World")
3+
if __name__ == "__main__":
4+
pass

‎extract_wheels/__init__.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,18 +64,22 @@ def main() -> None:
6464
required=True,
6565
help="The external repo name to install dependencies. In the format '@{REPO_NAME}'",
6666
)
67-
parser.add_argument('--extra_pip_args', action='store',
68-
help=('Extra arguments to pass down to pip.'))
67+
parser.add_argument(
68+
"--extra_pip_args",
69+
action="store",
70+
help=("Extra arguments to pass down to pip."),
71+
)
6972
parser.add_argument(
7073
"--pip_data_exclude",
71-
action='store',
72-
help='Additional data exclusion parameters to add to the pip packages BUILD file.'
74+
action="store",
75+
help="Additional data exclusion parameters to add to the pip packages BUILD file.",
7376
)
7477
args = parser.parse_args()
7578

7679
pip_args = [sys.executable, "-m", "pip", "wheel", "-r", args.requirements]
7780
if args.extra_pip_args:
7881
pip_args += json.loads(args.extra_pip_args)["args"]
82+
7983
# Assumes any errors are logged by pip so do nothing. This command will fail if pip fails
8084
subprocess.run(pip_args, check=True)
8185

‎extract_wheels/lib/bazel.py

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,9 @@
77
from extract_wheels.lib import namespace_pkgs, wheel, purelib
88

99

10-
def generate_build_file_contents(name: str, dependencies: List[str], pip_data_exclude: List[str]) -> str:
10+
def generate_build_file_contents(
11+
name: str, dependencies: List[str], pip_data_exclude: List[str]
12+
) -> str:
1113
"""Generate a BUILD file for an unzipped Wheel
1214
1315
Args:
@@ -39,7 +41,9 @@ def generate_build_file_contents(name: str, dependencies: List[str], pip_data_ex
3941
deps = [{dependencies}],
4042
)
4143
""".format(
42-
name=name, dependencies=",".join(dependencies), data_exclude=json.dumps(data_exclude)
44+
name=name,
45+
dependencies=",".join(dependencies),
46+
data_exclude=json.dumps(data_exclude),
4347
)
4448
)
4549

@@ -93,33 +97,31 @@ def sanitise_name(name: str) -> str:
9397

9498

9599
def setup_namespace_pkg_compatibility(wheel_dir: str) -> None:
96-
"""Converts native namespace packages and pkg_resource-style packages to pkgutil-style packages
100+
"""Converts native namespace packages to pkgutil-style packages
97101
98102
Namespace packages can be created in one of three ways. They are detailed here:
99103
https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package
100104
101-
'pkgutil-style namespace packages' (2) works in Bazel, but 'native namespace packages' (1) and
102-
'pkg_resources-style namespace packages' (3) do not.
105+
'pkgutil-style namespace packages' (2) and 'pkg_resources-style namespace packages' (3) works in Bazel, but
106+
'native namespace packages' (1) do not.
103107
104-
We ensure compatibility with Bazel of methods 1 and 3 by converting them into method 2.
108+
We ensure compatibility with Bazel of method 1 by converting them into method 2.
105109
106110
Args:
107111
wheel_dir: the directory of the wheel to convert
108112
"""
109113

110-
namespace_pkg_dirs = namespace_pkgs.pkg_resources_style_namespace_packages(
111-
wheel_dir
114+
namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages(
115+
wheel_dir, ignored_dirnames=["%s/bin" % wheel_dir,],
112116
)
113-
if not namespace_pkg_dirs and namespace_pkgs.native_namespace_packages_supported():
114-
namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages(
115-
wheel_dir, ignored_dirnames=["%s/bin" % wheel_dir,],
116-
)
117117

118118
for ns_pkg_dir in namespace_pkg_dirs:
119119
namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir)
120120

121121

122-
def extract_wheel(wheel_file: str, extras: Dict[str, Set[str]], pip_data_exclude: List[str]) -> str:
122+
def extract_wheel(
123+
wheel_file: str, extras: Dict[str, Set[str]], pip_data_exclude: List[str]
124+
) -> str:
123125
"""Extracts wheel into given directory and creates a py_library target.
124126
125127
Args:

‎extract_wheels/lib/namespace_pkgs.py

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,11 @@
11
"""Utility functions to discover python package types"""
22
import os
3-
import sys
43
import textwrap
54
from typing import Set, List, Optional
65

76
from extract_wheels.lib import wheel
87

98

10-
def pkg_resources_style_namespace_packages(wheel_dir: str) -> Set[str]:
11-
"""Discovers namespace packages implemented using the 'pkg_resources-style namespace packages' method.
12-
13-
"While this approach is no longer recommended, it is widely present in most existing namespace packages." - PyPA
14-
See https://packaging.python.org/guides/packaging-namespace-packages/#pkg-resources-style-namespace-packages
15-
"""
16-
namespace_pkg_dirs = set()
17-
18-
dist_info = wheel.get_dist_info(wheel_dir)
19-
namespace_packages_record_file = os.path.join(dist_info, "namespace_packages.txt")
20-
if os.path.exists(namespace_packages_record_file):
21-
with open(namespace_packages_record_file) as nspkg:
22-
for line in nspkg.readlines():
23-
namespace = line.strip().replace(".", os.sep)
24-
if namespace:
25-
namespace_pkg_dirs.add(os.path.join(wheel_dir, namespace))
26-
return namespace_pkg_dirs
27-
28-
29-
def native_namespace_packages_supported() -> bool:
30-
"""Returns true if this version of Python supports native namespace packages."""
31-
return (sys.version_info.major, sys.version_info.minor) >= (3, 3)
32-
33-
349
def implicit_namespace_packages(
3510
directory: str, ignored_dirnames: Optional[List[str]] = None
3611
) -> Set[str]:
@@ -84,8 +59,7 @@ def add_pkgutil_style_namespace_pkg_init(dir_path: str) -> None:
8459

8560
if os.path.isfile(ns_pkg_init_filepath):
8661
raise ValueError("%s already contains an __init__.py file." % dir_path)
87-
if not os.path.exists(dir_path):
88-
os.makedirs(dir_path)
62+
8963
with open(ns_pkg_init_filepath, "w") as ns_pkg_init_f:
9064
# See https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages
9165
ns_pkg_init_f.write(

0 commit comments

Comments
 (0)
0