8000 Filter to possible package paths before trying to resolve a module (#… · python/mypy@715b768 · GitHub
[go: up one dir, main page]
Skip to content

Commit 715b768

Browse files
falsedrowdmj-openaihauntsaninja
authored
Filter to possible package paths before trying to resolve a module (#18038)
With a long sys.path (it's got 300 entries), this removes 94% of stat syscalls from running mypy. With all the filesystem caching, that's only a small time savings, though it will depend on your filesystem. Local benchmarks showed a 20% time savings but they're pretty noisy from all the I/O. --------- Co-authored-by: Daniel Jacobowitz <dmj@openai.com> Co-authored-by: hauntsaninja <hauntsaninja@gmail.com>
1 parent 9365fbf commit 715b768

File tree

3 files changed

+19
-11
lines changed

3 files changed

+19
-11
lines changed

‎mypy/modulefinder.py

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -331,8 +331,6 @@ def _find_module_non_stub_helper(
331331
# If this is not a directory then we can't traverse further into it
332332
if not self.fscache.isdir(dir_path):
333333
break
334-
if approved_stub_package_exists(".".join(components)):
335-
return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED
336334
if plausible_match:
337335
return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS
338336
else:
@@ -414,9 +412,16 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult:
414412
third_party_inline_dirs: PackageDirs = []
415413
third_party_stubs_dirs: PackageDirs = []
416414
found_possible_third_party_missing_type_hints = False
417-
need_installed_stubs = False
418415
# Third-party stub/typed packages
416+
candidate_package_dirs = {
417+
package_dir[0]
418+
for component in (components[0], components[0] + "-stubs")
419+
for package_dir in self.find_lib_path_dirs(component, self.search_paths.package_path)
420+
}
419421
for pkg_dir in self.search_paths.package_path:
422+
pkg_dir = os.path.normpath(pkg_dir)
423+
if pkg_dir not in candidate_package_dirs:
424+
continue
420425
stub_name = components[0] + "-stubs"
421426
stub_dir = os_path_join(pkg_dir, stub_name)
422427
if fscache.isdir(stub_dir):
@@ -445,11 +450,10 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult:
445450
if isinstance(non_stub_match, ModuleNotFoundReason):
446451
if non_stub_match is ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS:
447452
found_possible_third_party_missing_type_hints = True
448-
elif non_stub_match is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED:
449-
need_installed_stubs = True
450453
else:
451454
third_party_inline_dirs.append(non_stub_match)
452455
self._update_ns_ancestors(components, non_stub_match)
456+
453457
if self.options and self.options.use_builtins_fixtures:
454458
# Everything should be in fixtures.
455459
third_party_inline_dirs.clear()
@@ -548,12 +552,11 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult:
548552
if ancestor is not None:
549553
return ancestor
550554

551-
if need_installed_stubs:
555+
if approved_stub_package_exists(id):
552556
return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED
553-
elif found_possible_third_party_missing_type_hints:
557+
if found_possible_third_party_missing_type_hints:
554558
return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS
555-
else:
556-
return ModuleNotFoundReason.NOT_FOUND
559+
return ModuleNotFoundReason.NOT_FOUND
557560

558561
def find_modules_recursive(self, module: str) -> list[BuildSource]:
559562
module_path = self.find_module(module, fast_path=True)

‎mypy/test/testmodulefinder.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ def setUp(self) -> None:
165165
self.fmc_nons = FindModuleCache(self.search_paths, fscache=None, options=options)
166166

167167
def path(self, *parts: str) -> str:
168-
return os.path.join(self.package_dir, *parts)
168+
return os.path.normpath(os.path.join(self.package_dir, *parts))
169169

170170
def test__packages_with_ns(self) -> None:
171171
cases = [

‎test-data/unit/check-modules.test

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3146,8 +3146,13 @@ main:1: note: (or run "mypy --install-types" to install all missing stub package
31463146
main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
31473147
main:2: error: Library stubs not installed for "bleach.abc"
31483148

3149-
[case testMissingSubmoduleOfInstalledStubPackageIgnored]
3149+
[case testMissingSubmoduleOfInstalledStubPackageIgnored-xfail]
31503150
# flags: --ignore-missing-imports
3151+
3152+
# TODO: testMissingSubmoduleOfInstalledStubPackageIgnored was regressed in
3153+
# https://github.com/python/mypy/pull/15347 but didn't cause failures because we don't have a
3154+
# package path in this unit test
3155+
31513156
import bleach.xyz
31523157
from bleach.abc import fgh
31533158
[file bleach/__init__.pyi]

0 commit comments

Comments
 (0)
0