8000 Namespace packages (PEP 420) (#5691) · python/mypy@f59a871 · GitHub
[go: up one dir, main page]
Skip to content

Commit f59a871

Browse files
gvanrossumgvanrossum
authored
Namespace packages (PEP 420) (#5691)
Tentative implementation of PEP 420. Fixes #1645. Clarification of the implementation: - `candidate_base_dirs` is a list of `(dir, verify)` tuples, laboriously pre-computed. - The old code just looped over this list, looking for `dir/<module>`, and if found, it would verify that there were `__init__.py` files in all the right places (except if `verify` is false); the first success would be the hit; - In PEP 420 mode, if the above approach finds no hits, we do something different for those paths that failed due to `__init__` verification; essentially we narrow down the list of candidate paths by checking for `__init__` files from the top down. Hopefully the last test added clarifies this.
1 parent 88dc51a commit f59a871

File tree

2 files changed

+139
-0
lines changed

2 files changed

+139
-0
lines changed

‎mypy/modulefinder.py

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,7 @@ def _find_module(self, id: str) -> Optional[str]:
170170
# Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories.
171171
seplast = os.sep + components[-1] # so e.g. '/baz'
172172
sepinit = os.sep + '__init__'
173+
near_misses = [] # Collect near misses for namespace mode (see below).
173174
for base_dir, verify in candidate_base_dirs:
174175
base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz'
175176
# Prefer package over module, i.e. baz/__init__.py* over baz.py*.
@@ -178,19 +179,49 @@ def _find_module(self, id: str) -> Optional[str]:
178179
path_stubs = base_path + '-stubs' + sepinit + extension
179180
if fscache.isfile_case(path):
180181
if verify and not verify_module(fscache, id, path):
182+
near_misses.append(path)
181183
continue
182184
return path
183185
elif fscache.isfile_case(path_stubs):
184186
if verify and not verify_module(fscache, id, path_stubs):
187+
near_misses.append(path_stubs)
185188
continue
186189
return path_stubs
187190
# No package, look for module.
188191
for extension in PYTHON_EXTENSIONS:
189192
path = base_path + extension
190193
if fscache.isfile_case(path):
191194
if verify and not verify_module(fscache, id, path):
195+
near_misses.append(path)
192196
continue
193197
return path
198+
199+
# In namespace mode, re-check those entries that had 'verify'.
200+
# Assume search path entries xxx, yyy and zzz, and we're
201+
# looking for foo.bar.baz. Suppose near_misses has:
202+
#
203+
# - xxx/foo/bar/baz.py
204+
# - yyy/foo/bar/baz/__init__.py
205+
# - zzz/foo/bar/baz.pyi
206+
#
207+
# If any of the foo directories has __init__.py[i], it wins.
208+
# Else, we look for foo/bar/__init__.py[i], etc. If there are
209+
# none, the first hit wins. Note that this does not take into
210+
# account whether the lowest-level module is a file (baz.py),
211+
# a package (baz/__init__.py), or a stub file (baz.pyi) -- for
212+
# these the first one encountered along the search path wins.
213+
#
214+
# The helper function highest_init_level() returns an int that
215+
# indicates the highest level at which a __init__.py[i] file
216+
# is found; if no __init__ was found it returns 0, if we find
217+
# only foo/bar/__init__.py it returns 1, and if we have
218+
# foo/__init__.py it returns 2 (regardless of what's in
219+
# foo/bar). It doesn't look higher than that.
220+
if self.options and self.options.namespace_packages and near_misses:
221+
levels = [highest_init_level(fscache, id, path) for path in near_misses]
222+
index = levels.index(max(levels))
223+
return near_misses[index]
224+
194225
retu 10000 rn None
195226

196227
def find_modules_recursive(self, module: str) -> List[BuildSource]:
@@ -236,6 +267,19 @@ def verify_module(fscache: FileSystemCache, id: str, path: str) -> bool:
236267
return True
237268

238269

270+
def highest_init_level(fscache: FileSystemCache, id: str, path: str) -> int:
271+
"""Compute the highest level where an __init__ file is found."""
272+
if path.endswith(('__init__.py', '__init__.pyi')):
273+
path = os.path.dirname(path)
274+
level = 0
275+
for i in range(id.count('.')):
276+
path = os.path.dirname(path)
277+
if any(fscache.isfile_case(os.path.join(path, '__init__{}'.format(extension)))
278+
for extension in PYTHON_EXTENSIONS):
279+
level = i + 1
280+
return level
281+
282+
239283
def mypy_path() -> List[str]:
240284
path_env = os.getenv('MYPYPATH')
241285
if not path_env:

‎test-data/unit/check-modules.test

Lines changed: 95 additions & 0 deletions
< 6302 td data-grid-cell-id="diff-841b50838a13f5f73aa1699d00dbae63f0e4144cb0be0b8345af10acdb42e55f-2527-2536-2" data-line-anchor="diff-841b50838a13f5f73aa1699d00dbae63f0e4144cb0be0b8345af10acdb42e55fR2536" data-selected="false" role="gridcell" style="background-color:var(--diffBlob-additionLine-bgColor, var(--diffBlob-addition-bgColor-line));padding-right:24px" tabindex="-1" valign="top" class="focusable-grid-cell diff-text-cell right-side-diff-cell left-side">+
[file foo/bar.py]
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
-- Type checker test cases dealing with modules and imports.
2+
-- Towards the end there are tests for PEP 420 (namespace packages, i.e. __init__.py-less packages).
23

34
[case testAccessImportedDefinitions]
45
import m
@@ -2525,3 +2526,97 @@ def __radd__(self) -> int: ...
25252526

25262527
[case testFunctionWithInPlaceDunderName]
25272528
def __iadd__(self) -> int: ...
2529+
2530+
-- Tests for PEP 420 namespace packages.
2531+
2532+
[case testClassicPackage]
2533+
from foo.bar import x
2534+
[file foo/__init__.py]
2535+
# empty
2536
2537+
x = 0
2538+
2539+
[case testClassicNotPackage]
2540+
from foo.bar import x
2541+
[file foo/bar.py]
2542+
x = 0
2543+
[out]
2544+
main:1: error: Cannot find module named 'foo.bar'
2545+
main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
2546+
2547+
[case testNamespacePackage]
2548+
# flags: --namespace-packages
2549+
from foo.bar import x
2550+
reveal_type(x) # E: Revealed type is 'builtins.int'
2551+
[file foo/bar.py]
2552+
x = 0
2553+
2554+
[case testNamespacePackageWithMypyPath]
2555+
# flags: --namespace-packages --config-file tmp/mypy.ini
2556+
from foo.bax import x
2557+
from foo.bay import y
2558+
from foo.baz import z
2559+
reveal_type(x) # E: Revealed type is 'builtins.int'
2560+
reveal_type(y) # E: Revealed type is 'builtins.int'
2561+
reveal_type(z) # E: Revealed type is 'builtins.int'
2562+
[file xx/foo/bax.py]
2563+
x = 0
2564+
[file yy/foo/bay.py]
2565+
y = 0
2566+
[file foo/baz.py]
2567+
z = 0
2568+
[file mypy.ini]
2569+
[[mypy]
2570+
mypy_path = tmp/xx, tmp/yy
2571+
2572+
[case testClassicPackageIgnoresEarlierNamespacePackage]
2573+
# flags: --namespace-packages --config-file tmp/mypy.ini
2574+
from foo.bar import y
2575+
reveal_type(y) # E: Revealed type is 'builtins.int'
2576+
[file xx/foo/bar.py]
2577+
x = ''
2578+
[file yy/foo/bar.py]
2579+
y = 0
2580+
[file yy/foo/__init__.py]
2581+
[file mypy.ini]
2582+
[[mypy]
2583+
mypy_path = tmp/xx, tmp/yy
2584+
2585+
[case testNamespacePackagePickFirstOnMypyPath]
2586+
# flags: --namespace-packages --config-file tmp/mypy.ini
2587+
from foo.bar import x
2588+
reveal_type(x) # E: Revealed type is 'builtins.int'
2589+
[file xx/foo/bar.py]
2590+
x = 0
2591+
[file yy/foo/bar.py]
2592+
x = ''
2593+
[file mypy.ini]
2594+
[[mypy]
2595+
mypy_path = tmp/xx, tmp/yy
2596+
2597+
[case testNamespacePackageInsideClassicPackage]
2598+
# flags: --namespace-packages --config-file tmp/mypy.ini
2599+
from foo.bar.baz import x
2600+
reveal_type(x) # E: Revealed type is 'builtins.int'
2601+
[file xx/foo/bar/baz.py]
2602+
x = ''
2603+
[file yy/foo/bar/baz.py]
2604+
x = 0
2605+
[file yy/foo/__init__.py]
2606+
[file mypy.ini]
2607+
[[mypy]
2608+
mypy_path = tmp/xx, tmp/yy
2609+
2610+
[case testClassicPackageInsideNamespacePackage]
2611+
# flags: --namespace-packages --config-file tmp/mypy.ini
2612+
from foo.bar.baz.boo import x
2613+
reveal_type(x) # E: Revealed type is 'builtins.int'
2614+
[file xx/foo/bar/baz/boo.py]
2615+
x = ''
2616+
[file xx/foo/bar/baz/__init__.py]
2617+
[file yy/foo/bar/baz/boo.py]
2618+
x = 0
2619+
[file yy/foo/bar/__init__.py]
2620+
[file mypy.ini]
2621+
[[mypy]
2622+
mypy_path = tmp/xx, tmp/yy

0 commit comments

Comments
 (0)
0