use regexes for exclude

python · JukkaL · Feb 10, 2021 · Jan 29, 2021 · Jan 29, 2021 · Jan 29, 2021
commit 7e92e9bdba8cfe9a9ce7e037b2effd6e70577b33
diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst
@@ -51,9 +51,9 @@ for full details, see :ref:`running-mypy`.
 
 .. option:: --exclude
 
-    Asks mypy to exclude a given file name, directory name or subpath while
-    recursively discovering files to check. This flag may be repeated multiple
-    times.
+    A regular expression that matches file names
8000
, directory names and paths
+    which mypy should ignore while recursively discovering files to check.
+    Use forward slashes on all platforms.
 
 
 Optional arguments

diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst
@@ -199,10 +199,11 @@ section of the command line docs.
 
 .. confval:: exclude
 
-    :type: comma-separated list of strings
+    :type: regular expression
 
-    A comma-separated list of file names, directory names or subpaths which mypy
-    should ignore while recursively discovering files to check.
+    A regular expression that matches file names, directory names and paths
+    which mypy should ignore while recursively discovering files to check.
+    Use forward slashes on all platforms.
 
     This option may only be set in the global section (``[mypy]``).
 

diff --git a/mypy/build.py b/mypy/build.py
@@ -2569,6 +2569,7 @@ def log_configuration(manager: BuildManager, sources: List[BuildSource]) -> None
         ("Current Executable", sys.executable),
         ("Cache Dir", manager.options.cache_dir),
         ("Compiled", str(not __file__.endswith(".py"))),
+        ("Exclude", manager.options.exclude),
     ]
 
     for conf_name, conf_value in configuration_vars:

diff --git a/mypy/config_parser.py b/mypy/config_parser.py
@@ -83,7 +83,6 @@ def check_follow_imports(choice: str) -> str:
     'custom_typing_module': str,
     'custom_typeshed_dir': expand_path,
     'mypy_path': lambda s: [expand_path(p.strip()) for p in re.split('[,:]', s)],
-    'exclude': lambda s: [expand_path(p.strip()) for p in s.split(",")],
     'files': split_and_match_files,
     'quickstart_file': expand_path,
     'junit_xml': expand_path,

diff --git a/mypy/find_sources.py b/mypy/find_sources.py
@@ -2,11 +2,12 @@
 
 import functools
 import os
+import re
 
 from typing import List, Sequence, Set, Tuple, Optional
 from typing_extensions import Final
 
-from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path, matches_exclude_pattern
+from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path
 from mypy.fscache import FileSystemCache
 from mypy.options import Options
 
@@ -103,16 +104,14 @@ def find_sources_in_dir(self, path: str) -> List[BuildSource]:
         seen = set()  # type: Set[str]
         names = sorted(self.fscache.listdir(path), key=keyfunc)
         for name in names:
-            # Skip certain names altogether
-            if (
-                name in ("__pycache__", "site-packages", "node_modules")
-                or name.startswith(".")
-                or name.endswith("~")
-            ):
-                continue
             subpath = os.path.join(path, name)
-            if any(matches_exclude_pattern(subpath, pattern) for pattern in self.exclude):
-                continue
+
+            if self.exclude:
+                subpath_str = "/" + os.path.abspath(subpath).replace(os.sep, "/")
+                if self.fscache.isdir(subpath):
+                    subpath_str += "/"
+                if re.search(self.exclude, subpath_str):
+                    continue
 
             if self.fscache.isdir(subpath):
                 sub_sources = self.find_sources_in_dir(subpath)

diff --git a/mypy/main.py b/mypy/main.py
@@ -424,6 +424,7 @@ def add_invertible_flag(flag: str,
     # Options object.  Options that require further processing should have
     # their `dest` prefixed with `special-opts:`, which will cause them to be
     # parsed into the separate special_opts namespace object.
+    options = Options()
 
     # Note: we have a style guide for formatting the mypy --help text. See
     # https://github.com/python/mypy/wiki/Documentation-Conventions
@@ -811,9 +812,11 @@ def add_invertible_flag(flag: str,
     code_group.add_argument(
         "--exclude",
         metavar="PATH",
-        action="append",
-        default=[],
-        help="File names, directory names or subpaths to avoid checking",
+        default=options.exclude,
+        help=(
+            "Regex to match file names, directory names or paths to avoid checking. "
+            "Defaults to '%(default)s'."
+        )
     )
     code_group.add_argument(
         '-m', '--module', action='append', metavar='MODULE',
@@ -843,8 +846,6 @@ def add_invertible_flag(flag: str,
     if config_file and not os.path.exists(config_file):
         parser.error("Cannot find config file '%s'" % config_file)
 
-    options = Options()
-
     def set_strict_flags() -> None:
         for dest, value in strict_flag_assignments:
             setattr(options, dest, value)

diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py
@@ -7,6 +7,7 @@
 import collections
 import functools
 import os
+import re
 import subprocess
 import sys
 from enum import Enum
@@ -450,10 +451,13 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]:
             ):
                 continue
             subpath = os.path.join(package_path, name)
-            if self.options and any(
-                matches_exclude_pattern(subpath, pattern) for pattern in self.options.exclude
-            ):
-                continue
+
+            if self.options and self.options.exclude:
+                subpath_str = "/" + os.path.abspath(subpath).replace(os.sep, "/")
+                if self.fscache.isdir(subpath):
+                    subpath_str += "/"
+                if re.search(self.options.exclude, subpath_str):
+                    continue
 
             if self.fscache.isdir(subpath):
                 # Only recurse into packages
@@ -475,18 +479,6 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]:
         return sources
 
 
-def matches_exclude_pattern(path: str, pattern: str) -> bool:
-    path = os.path.splitdrive(path)[1]
-    path_components = path.split(os.sep)
-    pattern_components = pattern.replace(os.sep, "/").split("/")
-    if len(path_components) < len(pattern_components):
-        return False
-    return all(
-        path == pattern
-        for path, pattern in zip(reversed(path_components), reversed(pattern_components))
-    )
-
-
 def verify_module(fscache: FileSystemCache, id: str, path: str, prefix: str) -> bool:
     """Check that all packages containing id have a __init__ file."""
     if path.endswith(('__init__.py', '__init__.pyi')):

diff --git a/mypy/options.py b/mypy/options.py
@@ -98,7 +98,7 @@ def __init__(self) -> None:
         # top-level __init__.py to your packages.
         self.explicit_package_bases = False
         # File names, directory names or subpaths to avoid checking
-        self.exclude = []  # type: List[str]
+        self.exclude = "/(__pycache__|site-packages|node_modules)/|/(\\.[^/]+|[^/]+~)/?$"  # type: str
 
         # disallow_any options
         self.disallow_any_generics = False

diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py
@@ -257,7 +257,7 @@ def test_find_sources_exclude(self) -> None:
         options = Options()
         options.namespace_packages = True
 
-        # special cased name
+        # default
         finder = SourceFinder(FakeFSCache({"/dir/a.py", "/dir/venv/site-packages/b.py"}), options)
         assert find_sources(finder, "/") == [("a", "/dir")]
 
@@ -270,7 +270,7 @@ def test_find_sources_exclude(self) -> None:
         }
 
         # file name
-        options.exclude = ["f.py"]
+        options.exclude = "/f.py"
         finder = SourceFinder(FakeFSCache(files), options)
         assert find_sources(finder, "/") == [
             ("a2", "/pkg"),
@@ -279,7 +279,7 @@ def test_find_sources_exclude(self) -> None:
         ]
 
         # directory name
-        options.exclude = ["a1"]
+        options.exclude = "/a1/"
         finder = SourceFinder(FakeFSCache(files), options)
         assert find_sources(finder, "/") == [
             ("a2", "/pkg"),
@@ -288,15 +288,15 @@ def test_find_sources_exclude(self) -> None:
         ]
 
         # paths
-        options.exclude = ["/pkg/a1"]
+        options.exclude = "/pkg/a1/"
         finder = SourceFinder(FakeFSCache(files), options)
         assert find_sources(finder, "/") == [
             ("a2", "/pkg"),
             ("a2.b.c.d.e", "/pkg"),
             ("a2.b.f", "/pkg"),
         ]
 
-        options.exclude = ["b/c"]
+        options.exclude = "b/c/"
         finder = SourceFinder(FakeFSCache(files), options)
         assert find_sources(finder, "/") == [
             ("a2", "/pkg"),
@@ -305,24 +305,19 @@ def test_find_sources_exclude(self) -> None:
         ]
 
         # nothing should be ignored as a result of this
-        options.exclude = [
-            "/pkg/a", "2", "1", "pk", "kg", "g.py", "bc", "/b", "/xxx/pkg/a2/b/f.py"
-            "xxx/pkg/a2/b/f.py"
-        ]
+        options.exclude = "|".join((
+            "/pkg/a/", "/2", "/1", "/pk/", "/kg", "/g.py", "/bc", "/xxx/pkg/a2/b/f.py"
+            "xxx/pkg/a2/b/f.py",
+        ))
         finder = SourceFinder(FakeFSCache(files), options)
         assert len(find_sources(finder, "/")) == len(files)
 
-        # nothing should be ignored as a result of this
         files = {
             "pkg/a1/b/c/d/e.py",
             "pkg/a1/b/f.py",
             "pkg/a2/__init__.py",
             "pkg/a2/b/c/d/e.py",
             "pkg/a2/b/f.py",
         }
-        options.exclude = [
-            "/pkg/a", "2", "1", "pk", "kg", "g.py", "bc", "/b", "/xxx/pkg/a2/b/f.py",
-            "xxx/pkg/a2/b/f.py", "/pkg/a1", "/pkg/a2"
-        ]
         finder = SourceFinder(FakeFSCache(files), options)
         assert len(find_sources(finder, "/")) == len(files)
diff --git a/mypy_self_check.ini b/mypy_self_check.ini
@@ -19,4 +19,4 @@ pretty = True
 always_false = MYPYC
 plugins = misc/proper_plugin.py
 python_version = 3.5
-exclude = mypy/typeshed
+exclude = /(mypy/typeshed|__pycache__|site-packages|node_modules)/|/(\.[^/]+|[^/]+~)/?$