diff --git a/mypy/build.py b/mypy/build.py index 41434adf6c79..d49e12e2bf69 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -42,7 +42,7 @@ from mypy.stats import dump_type_stats from mypy.types import Type from mypy.version import __version__ -from mypy.plugin import DefaultPlugin +from mypy.plugin import PluginManager, Plugin, DefaultPlugin # We need to know the location of this file to load data, but @@ -112,7 +112,9 @@ def is_source(self, file: MypyFile) -> bool: def build(sources: List[BuildSource], options: Options, alt_lib_path: str = None, - bin_dir: str = None) -> BuildResult: + bin_dir: str = None, + plugin: Optional[Plugin] = None, + ) -> BuildResult: """Analyze a program. A single call to build performs parsing, semantic analysis and optionally @@ -174,6 +176,9 @@ def build(sources: List[BuildSource], source_set = BuildSourceSet(sources) + if plugin is None: + plugin = DefaultPlugin(options.python_version) + # Construct a build manager object to hold state during the build. # # Ignore current directory prefix in error messages. @@ -183,6 +188,7 @@ def build(sources: List[BuildSource], reports=reports, options=options, version_id=__version__, + plugin=plugin, ) try: @@ -364,7 +370,8 @@ def __init__(self, data_dir: str, source_set: BuildSourceSet, reports: Reports, options: Options, - version_id: str) -> None: + version_id: str, + plugin: Plugin) -> None: self.start_time = time.time() self.data_dir = data_dir self.errors = Errors(options.show_error_context, options.show_column_numbers) @@ -374,6 +381,7 @@ def __init__(self, data_dir: str, self.reports = reports self.options = options self.version_id = version_id + self.plugin = plugin self.modules = {} # type: Dict[str, MypyFile] self.missing_modules = set() # type: Set[str] self.semantic_analyzer = SemanticAnalyzer(self.modules, self.missing_modules, @@ -1506,9 +1514,8 @@ def type_check_first_pass(self) -> None: if self.options.semantic_analysis_only: return with self.wrap_context(): - plugin = DefaultPlugin(self.options.python_version) self.type_checker = TypeChecker(manager.errors, manager.modules, self.options, - self.tree, self.xpath, plugin) + self.tree, self.xpath, manager.plugin) self.type_checker.check_first_pass() def type_check_second_pass(self) -> bool: diff --git a/mypy/main.py b/mypy/main.py index 2d0fe9ec7607..9c18dca3337a 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -8,7 +8,7 @@ import sys import time -from typing import Any, Dict, List, Mapping, Optional, Set, Tuple +from typing import Any, cast, Dict, List, Mapping, Optional, Set, Tuple from mypy import build from mypy import defaults @@ -17,6 +17,7 @@ from mypy.build import BuildSource, BuildResult, PYTHON_EXTENSIONS from mypy.errors import CompileError from mypy.options import Options, BuildType +from mypy.plugin import Plugin, PluginRegistry, PluginManager, DefaultPlugin, locate from mypy.report import reporter_classes from mypy.version import __version__ @@ -44,10 +45,10 @@ def main(script_path: str, args: List[str] = None) -> None: sys.setrecursionlimit(2 ** 14) if args is None: args = sys.argv[1:] - sources, options = process_options(args) + sources, options, plugin = process_options(args) serious = False try: - res = type_check_only(sources, bin_dir, options) + res = type_check_only(sources, bin_dir, options, plugin) a = res.errors except CompileError as e: a = e.messages @@ -90,11 +91,13 @@ def readlinkabs(link: str) -> str: return os.path.join(os.path.dirname(link), path) -def type_check_only(sources: List[BuildSource], bin_dir: str, options: Options) -> BuildResult: +def type_check_only(sources: List[BuildSource], bin_dir: str, options: Options, + plugin: Plugin) -> BuildResult: # Type-check the program and dependencies and translate to Python. return build.build(sources=sources, bin_dir=bin_dir, - options=options) + options=options, + plugin=plugin) FOOTER = """environment variables: @@ -172,9 +175,37 @@ def invert_flag_name(flag: str) -> str: return '--no-{}'.format(flag[2:]) +def load_plugin(prefix: str, name: str, location: str, + python_version: Tuple[int, int]) -> Optional[Plugin]: + try: + mod = __import__(location) + except BaseException as err: + print("%s: Error importing plugin module %s: %s" % + (prefix, location, err), file=sys.stderr) + return None + try: + register = getattr(mod, 'register_plugin') + except AttributeError: + print("%s: Could not find %s.register_plugin" % + (prefix, location), file=sys.stderr) + return None + try: + plugin = register(python_version) + except BaseException as err: + print("%s: Error calling %s.register_plugin: %s" % + (prefix, location, err), file=sys.stderr) + return None + + if not isinstance(plugin, Plugin): + print("%s: Result of calling %s.register_plugin is not a plugin: %r" % + (prefix, location, plugin), file=sys.stderr) + return None + return plugin + + def process_options(args: List[str], require_targets: bool = True - ) -> Tuple[List[BuildSource], Options]: + ) -> Tuple[List[BuildSource], Options, Plugin]: """Parse command line arguments.""" parser = argparse.ArgumentParser(prog='mypy', epilog=FOOTER, @@ -456,11 +487,21 @@ def disallow_any_argument_type(raw_options: str) -> List[str]: if options.quick_and_dirty: options.incremental = True + # Load plugins + plugins = [] # type: List[Plugin] + for registry in options.plugins: + plugin = load_plugin('[mypy]', registry.name, registry.location, options.python_version) + if plugin is not None: + plugins.append(plugin) + # always add the default last + plugins.append(DefaultPlugin(options.python_version)) + plugin_manager = PluginManager(options.python_version, plugins) + # Set target. if special_opts.modules: options.build_type = BuildType.MODULE targets = [BuildSource(None, m, None) for m in special_opts.modules] - return targets, options + return targets, options, plugin_manager elif special_opts.package: if os.sep in special_opts.package or os.altsep and os.altsep in special_opts.package: fail("Package name '{}' cannot have a slash in it." @@ -470,11 +511,11 @@ def disallow_any_argument_type(raw_options: str) -> List[str]: targets = build.find_modules_recursive(special_opts.package, lib_path) if not targets: fail("Can't find package '{}'".format(special_opts.package)) - return targets, options + return targets, options, plugin_manager elif special_opts.command: options.build_type = BuildType.PROGRAM_TEXT targets = [BuildSource(None, None, '\n'.join(special_opts.command))] - return targets, options + return targets, options, plugin_manager else: targets = [] for f in special_opts.files: @@ -495,7 +536,7 @@ def disallow_any_argument_type(raw_options: str) -> List[str]: else: mod = os.path.basename(f) if options.scripts_are_modules else None targets.append(BuildSource(f, mod, None)) - return targets, options + return targets, options, plugin_manager def keyfunc(name: str) -> Tuple[int, str]: @@ -645,19 +686,29 @@ def parse_config_file(options: Options, filename: Optional[str]) -> None: else: section = parser['mypy'] prefix = '%s: [%s]' % (file_read, 'mypy') - updates, report_dirs = parse_section(prefix, options, section) + updates, report_dirs, plugins = parse_section(prefix, options, section) for k, v in updates.items(): setattr(options, k, v) + + for k, v in plugins: + # look for an options section for this plugin + plug_opts = dict(parser[k]) if k in parser else {} + options.plugins.append(PluginRegistry(k, v, plug_opts)) + options.report_dirs.update(report_dirs) for name, section in parser.items(): if name.startswith('mypy-'): prefix = '%s: [%s]' % (file_read, name) - updates, report_dirs = parse_section(prefix, options, section) + updates, report_dirs, plugins = parse_section(prefix, options, section) if report_dirs: print("%s: Per-module sections should not specify reports (%s)" % (prefix, ', '.join(s + '_report' for s in sorted(report_dirs))), file=sys.stderr) + if plugins: + print("%s: Per-module sections should not specify plugins (%s)" % + (prefix, ', '.join([p[0] for p in plugins])), + file=sys.stderr) if set(updates) - Options.PER_MODULE_OPTIONS: print("%s: Per-module sections should only specify per-module flags (%s)" % (prefix, ', '.join(sorted(set(updates) - Options.PER_MODULE_OPTIONS))), @@ -674,16 +725,23 @@ def parse_config_file(options: Options, filename: Optional[str]) -> None: def parse_section(prefix: str, template: Options, - section: Mapping[str, str]) -> Tuple[Dict[str, object], Dict[str, str]]: + section: Mapping[str, str]) -> Tuple[Dict[str, object], + Dict[str, str], List[Tuple[str, str]]]: """Parse one section of a config file. Returns a dict of option values encountered, and a dict of report directories. """ results = {} # type: Dict[str, object] report_dirs = {} # type: Dict[str, str] + plugins = [] # type: List[Tuple[str, str]] for key in section: key = key.replace('-', '_') - if key in config_types: + if key.startswith('plugins.'): + dv = section.get(key) + key = key[8:] + plugins.append((key, dv)) + continue + elif key in config_types: ct = config_types[key] else: dv = getattr(template, key, None) @@ -731,7 +789,7 @@ def parse_section(prefix: str, template: Options, if 'follow_imports' not in results: results['follow_imports'] = 'error' results[key] = v - return results, report_dirs + return results, report_dirs, plugins def fail(msg: str) -> None: diff --git a/mypy/options.py b/mypy/options.py index 69f99cce9501..e55979f6890e 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -3,6 +3,7 @@ import sys from typing import Mapping, Optional, Tuple, List, Pattern, Dict +from mypy.plugin import PluginRegistry from mypy import defaults @@ -113,6 +114,9 @@ def __init__(self) -> None: self.debug_cache = False self.quick_and_dirty = False + # plugins + self.plugins = [] # type: List[PluginRegistry] + # Per-module options (raw) self.per_module_options = {} # type: Dict[Pattern[str], Dict[str, object]] diff --git a/mypy/plugin.py b/mypy/plugin.py index 5015f7b4c940..b4aadecd31f9 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -1,20 +1,87 @@ -from typing import Callable, List, Tuple, Optional, NamedTuple +from typing import Callable, Dict, List, Tuple, Optional, NamedTuple +from types import ModuleType from mypy.nodes import Expression, StrExpr, IntExpr, UnaryExpr, Context from mypy.types import ( Type, Instance, CallableType, TypedDictType, UnionType, NoneTyp, FunctionLike, TypeVarType, AnyType ) -from mypy.messages import MessageBuilder +MYPY = False +if MYPY: + from mypy.messages import MessageBuilder + + +def _safeimport(path: str) -> Optional[ModuleType]: + try: + module = __import__(path) + except ImportError as err: + if err.name == path: # type: ignore # ImportError stubs need to be updated + # No such module in the path. + return None + else: + raise + for part in path.split('.')[1:]: + try: + module = getattr(module, part) + except AttributeError: + return None + return module + + +def locate(path: str) -> Optional[object]: + """Locate an object by string identifier, importing as necessary. + + The two identifiers supported are: + + dotted path: + package.module.object.child + + file path followed by dotted object path] + /path/to/module.py:object.child + """ + if ':' in path: + raise RuntimeError + # not supported in python 3.3 + # file_path, obj_path = path.split(':', 1) + # mod_name = os.path.splitext(os.path.basename(file_path))[0] + # spec = importlib.util.spec_from_file_location(mod_name, file_path) + # module = importlib.util.module_from_spec(spec) + # spec.loader.exec_module(module) + # parts = obj_path.split('.') + else: + parts = [part for part in path.split('.') if part] + module, n = None, 0 + while n < len(parts): + nextmodule = _safeimport('.'.join(parts[:n + 1])) + if nextmodule: + module, n = nextmodule, n + 1 + else: + break + parts = parts[n:] + + if not module: + return None + + obj = module + for part in parts: + try: + obj = getattr(obj, part) + except AttributeError: + return None + return obj # Create an Instance given full name of class and type arguments. NamedInstanceCallback = Callable[[str, List[Type]], Type] +PluginRegistry = NamedTuple('PluginRegistry', [('name', str), + ('location', str), + ('options', Dict[str, str])]) + # Some objects and callbacks that plugins can use to get information from the # type checker or to report errors. PluginContext = NamedTuple('PluginContext', [('named_instance', NamedInstanceCallback), - ('msg', MessageBuilder), + ('msg', 'MessageBuilder'), ('context', Context)]) @@ -109,6 +176,46 @@ def get_method_hook(self, fullname: str) -> Optional[MethodHook]: return None +class PluginManager(Plugin): + def __init__(self, python_version: Tuple[int, int], + plugins: List[Plugin]) -> None: + super().__init__(python_version) + self.plugins = plugins + self._function_hooks = {} # type: Dict[str, Optional[FunctionHook]] + self._method_signature_hooks = {} # type: Dict[str, Optional[MethodSignatureHook]] + self._method_hooks = {} # type: Dict[str, Optional[MethodHook]] + + def get_function_hook(self, fullname: str) -> Optional[FunctionHook]: + hook = self._function_hooks.get(fullname) + if hook is None: + for plugin in self.plugins: + hook = plugin.get_function_hook(fullname) + if hook is not None: + break + self._function_hooks[fullname] = hook + return hook + + def get_method_signature_hook(self, fullname: str) -> Optional[MethodSignatureHook]: + hook = self._method_signature_hooks.get(fullname) + if hook is None: + for plugin in self.plugins: + hook = plugin.get_method_signature_hook(fullname) + if hook is not None: + break + self._method_signature_hooks[fullname] = hook + return hook + + def get_method_hook(self, fullname: str) -> Optional[MethodHook]: + hook = self._method_hooks.get(fullname) + if hook is None: + for plugin in self.plugins: + hook = plugin.get_method_hook(fullname) + if hook is not None: + break + self._method_hooks[fullname] = hook + return hook + + def open_callback( arg_types: List[List[Type]], args: List[List[Expression]], diff --git a/mypy/test/testargs.py b/mypy/test/testargs.py index 4e27e37a7e45..9f3ec63ffeb2 100644 --- a/mypy/test/testargs.py +++ b/mypy/test/testargs.py @@ -14,5 +14,5 @@ class ArgSuite(Suite): def test_coherence(self) -> None: options = Options() - _, parsed_options = process_options([], require_targets=False) + parsed_options = process_options([], require_targets=False)[1] assert_equal(options, parsed_options) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 91a818ac0f01..ffe70a295f9d 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -346,7 +346,7 @@ def parse_options(self, program_text: str, testcase: DataDrivenTestCase, flag_list = None if flags: flag_list = flags.group(1).split() - targets, options = process_options(flag_list, require_targets=False) + targets, options, _ = process_options(flag_list, require_targets=False) if targets: # TODO: support specifying targets via the flags pragma raise RuntimeError('Specifying targets via the flags pragma is not supported.') diff --git a/mypy/test/testgraph.py b/mypy/test/testgraph.py index 7a9062914f89..aeaea58c3a3b 100644 --- a/mypy/test/testgraph.py +++ b/mypy/test/testgraph.py @@ -6,8 +6,10 @@ from mypy.build import BuildManager, State, BuildSourceSet from mypy.build import topsort, strongly_connected_components, sorted_components, order_ascc from mypy.version import __version__ +from mypy.plugin import DefaultPlugin from mypy.options import Options from mypy.report import Reports +from mypy import defaults class GraphSuite(Suite): @@ -42,6 +44,7 @@ def _make_manager(self) -> BuildManager: reports=Reports('', {}), options=Options(), version_id=__version__, + plugin=DefaultPlugin(defaults.PYTHON3_VERSION) ) return manager