diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py
index 0a98e8716f6a..4091e51f84e9 100644
--- a/mypy/modulefinder.py
+++ b/mypy/modulefinder.py
@@ -121,7 +121,7 @@ def _update_ns_ancestors(self, components: List[str], match: Tuple[str, bool]) -
         path, verify = match
         for i in range(1, len(components)):
             pkg_id = '.'.join(components[:-i])
-            if pkg_id not in self.ns_ancestors:
+            if pkg_id not in self.ns_ancestors and self.fscache.isdir(path):
                 self.ns_ancestors[pkg_id] = path
             path = os.path.dirname(path)