8000 User-plugin system by chadrik · Pull Request #3512 · python/mypy · GitHub
[go: up one dir, main page]
Skip to content

User-plugin system #3512

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 6 commits into from
Closed

User-plugin system #3512

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
906d03c
Initial sketch of user-plugin system
chadrik Jun 8, 2017
79e0494
Avoid a known bug in mypy where reference cycles are not handled corr…
chadrik Jun 8, 2017
8ec21e5
Move the creation of PluginManager earlier.
chadrik Jun 9, 2017
67262c4
fix name
chadrik Jun 9, 2017
5e784bc
it's called plugins now not hooks
chadrik Jun 9, 2017
8aaebd3
Expect plugins to provide a register_plugin function.
chadrik Jun 9, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 12 additions & 5 deletions mypy/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
from mypy.stats import dump_type_stats
from mypy.types import Type
from mypy.version import __version__
from mypy.plugin import DefaultPlugin
from mypy.plugin import PluginManager, Plugin, DefaultPlugin


# We need to know the location of this file to load data, but
Expand Down Expand Up @@ -112,7 +112,9 @@ def is_source(self, file: MypyFile) -> bool:
def build(sources: List[BuildSource],
options: Options,
alt_lib_path: str = None,
bin_dir: str = None) -> BuildResult:
bin_dir: str = None,
plugin: Optional[Plugin] = None,
) -> BuildResult:
"""Analyze a program.

A single call to build performs parsing, semantic analysis and optionally
Expand Down Expand Up @@ -174,6 +176,9 @@ def build(sources: List[BuildSource],

source_set = BuildSourceSet(sources)

if plugin is None:
plugin = DefaultPlugin(options.python_version)

# Construct a build manager object to hold state during the build.
#
# Ignore current directory prefix in error messages.
Expand All @@ -183,6 +188,7 @@ def build(sources: List[BuildSource],
reports=reports,
options=options,
version_id=__version__,
plugin=plugin,
)

try:
Expand Down Expand Up @@ -364,7 +370,8 @@ def __init__(self, data_dir: str,
source_set: BuildSourceSet,
reports: Reports,
options: Options,
version_id: str) -> None:
version_id: str,
plugin: Plugin) -> None:
self.start_time = time.time()
self.data_dir = data_dir
self.errors = Errors(options.show_error_context, options.show_column_numbers)
Expand All @@ -374,6 +381,7 @@ def __init__(self, data_dir: str,
self.reports = reports
self.options = options
self.version_id = version_id
self.plugin = plugin
self.modules = {} # type: Dict[str, MypyFile]
self.missing_modules = set() # type: Set[str]
self.semantic_analyzer = SemanticAnalyzer(self.modules, self.missing_modules,
Expand Down Expand Up @@ -1506,9 +1514,8 @@ def type_check_first_pass(self) -> None:
if self.options.semantic_analysis_only:
return
with self.wrap_context():
plugin = DefaultPlugin(self.options.python_version)
self.type_checker = TypeChecker(manager.errors, manager.modules, self.options,
self.tree, self.xpath, plugin)
self.tree, self.xpath, manager.plugin)
self.type_checker.check_first_pass()

def type_check_second_pass(self) -> bool:
Expand Down
88 changes: 73 additions & 15 deletions mypy/main.py
View file
Open in desktop
B41A
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import sys
import time

from typing import Any, Dict, List, Mapping, Optional, Set, Tuple
from typing import Any, cast, Dict, List, Mapping, Optional, Set, Tuple

from mypy import build
from mypy import defaults
Expand All @@ -17,6 +17,7 @@
from mypy.build import BuildSource, BuildResult, PYTHON_EXTENSIONS
from mypy.errors import CompileError
from mypy.options import Options, BuildType
from mypy.plugin import Plugin, PluginRegistry, PluginManager, DefaultPlugin, locate
from mypy.report import reporter_classes

from mypy.version import __version__
Expand Down Expand Up @@ -44,10 +45,10 @@ def main(script_path: str, args: List[str] = None) -> None:
sys.setrecursionlimit(2 ** 14)
if args is None:
args = sys.argv[1:]
sources, options = process_options(args)
sources, options, plugin = process_options(args)
serious = False
try:
res = type_check_only(sources, bin_dir, options)
res = type_check_only(sources, bin_dir, options, plugin)
a = res.errors
except CompileError as e:
a = e.messages
Expand Down Expand Up @@ -90,11 +91,13 @@ def readlinkabs(link: str) -> str:
return os.path.join(os.path.dirname(link), path)


def type_check_only(sources: List[BuildSource], bin_dir: str, options: Options) -> BuildResult:
def type_check_only(sources: List[BuildSource], bin_dir: str, options: Options,
plugin: Plugin) -> BuildResult:
# Type-check the program and dependencies and translate to Python.
return build.build(sources=sources,
bin_dir=bin_dir,
options=options)
options=options,
plugin=plugin)


FOOTER = """environment variables:
Expand Down Expand Up @@ -172,9 +175,37 @@ def invert_flag_name(flag: str) -> str:
return '--no-{}'.format(flag[2:])


def load_plugin(prefix: str, name: str, location: str,
python_version: Tuple[int, int]) -> Optional[Plugin]:
try:
mod = __import__(location)
except BaseException as err:
print("%s: Error importing plugin module %s: %s" %
(prefix, location, err), file=sys.stderr)
return None
try:
register = getattr(mod, 'register_plugin')
except AttributeError:
print("%s: Could not find %s.register_plugin" %
(prefix, location), file=sys.stderr)
return None
try:
plugin = register(python_version)
except BaseException as err:
print("%s: Error calling %s.register_plugin: %s" %
(prefix, location, err), file=sys.stderr)
return None

if not isinstance(plugin, Plugin):
print("%s: Result of calling %s.register_plugin is not a plugin: %r" %
(prefix, location, plugin), file=sys.stderr)
return None
return plugin


def process_options(args: List[str],
require_targets: bool = True
) -> Tuple[List[BuildSource], Options]:
) -> Tuple[List[BuildSource], Options, Plugin]:
"""Parse command line arguments."""

parser = argparse.ArgumentParser(prog='mypy', epilog=FOOTER,
Expand Down Expand Up @@ -456,11 +487,21 @@ def disallow_any_argument_type(raw_options: str) -> List[str]:
if options.quick_and_dirty:
options.incremental = True

# Load plugins
plugins = [] # type: List[Plugin]
for registry in options.plugins:
plugin = load_plugin('[mypy]', registry.name, registry.location, options.python_version)
if plugin is not None:
plugins.append(plugin)
# always add the default last
plugins.append(DefaultPlugin(options.python_version))
plugin_manager = PluginManager(options.python_version, plugins)

# Set target.
if special_opts.modules:
options.build_type = BuildType.MODULE
targets = [BuildSource(None, m, None) for m in special_opts.modules]
return targets, options
return targets, options, plugin_manager
elif special_opts.package:
if os.sep in special_opts.package or os.altsep and os.altsep in special_opts.package:
fail("Package name '{}' cannot have a slash in it."
Expand All @@ -470,11 +511,11 @@ def disallow_any_argument_type(raw_options: str) -> List[str]:
targets = build.find_modules_recursive(special_opts.package, lib_path)
if not targets:
fail("Can't find package '{}'".format(special_opts.package))
return targets, options
return targets, options, plugin_manager
elif special_opts.command:
options.build_type = BuildType.PROGRAM_TEXT
targets = [BuildSource(None, None, '\n'.join(special_opts.command))]
return targets, options
return targets, options, plugin_manager
else:
targets = []
for f in special_opts.files:
Expand All @@ -495,7 +536,7 @@ def disallow_any_argument_type(raw_options: str) -> List[str]:
else:
mod = os.path.basename(f) if options.scripts_are_modules else None
targets.append(BuildSource(f, mod, None))
return targets, options
return targets, options, plugin_manager


def keyfunc(name: str) -> Tuple[int, str]:
Expand Down Expand Up @@ -645,19 +686,29 @@ def parse_config_file(options: Options, filename: Optional[str]) -> None:
else:
section = parser['mypy']
prefix = '%s: [%s]' % (file_read, 'mypy')
updates, report_dirs = parse_section(prefix, options, section)
updates, report_dirs, plugins = parse_section(prefix, options, section)
for k, v in updates.items():
setattr(options, k, v)

for k, v in plugins:
# look for an options section for this plugin
plug_opts = dict(parser[k]) if k in parser else {}
options.plugins.append(PluginRegistry(k, v, plug_opts))

options.report_dirs.update(report_dirs)

for name, section in parser.items():
if name.startswith('mypy-'):
prefix = '%s: [%s]' % (file_read, name)
updates, report_dirs = parse_section(prefix, options, section)
updates, report_dirs, plugins = parse_section(prefix, options, section)
if report_dirs:
print("%s: Per-module sections should not specify reports (%s)" %
(prefix, ', '.join(s + '_report' for s in sorted(report_dirs))),
file=sys.stderr)
if plugins:
print("%s: Per-module sections should not specify plugins (%s)" %
(prefix, ', '.join([p[0] for p in plugins])),
file=sys.stderr)
if set(updates) - Options.PER_MODULE_OPTIONS:
print("%s: Per-module sections should only specify per-module flags (%s)" %
(prefix, ', '.join(sorted(set(updates) - Options.PER_MODULE_OPTIONS))),
Expand All @@ -674,16 +725,23 @@ def parse_config_file(options: Options, filename: Optional[str]) -> None:


def parse_section(prefix: str, template: Options,
section: Mapping[str, str]) -> Tuple[Dict[str, object], Dict[str, str]]:
section: Mapping[str, str]) -> Tuple[Dict[str, object],
Dict[str, str], List[Tuple[str, str]]]:
"""Parse one section of a config file.

Returns a dict of option values encountered, and a dict of report directories.
"""
results = {} # type: Dict[str, object]
report_dirs = {} # type: Dict[str, str]
plugins = [] # type: List[Tuple[str, str]]
for key in section:
key = key.replace('-', '_')
if key in config_types:
if key.startswith('plugins.'):
dv = section.get(key)
key = key[8:]
plugins.append((key, dv))
continue
elif key in config_types:
ct = config_types[key]
else:
dv = getattr(template, key, None)
Expand Down Expand Up @@ -731,7 +789,7 @@ def parse_section(prefix: str, template: Options,
if 'follow_imports' not in results:
results['follow_imports'] = 'error'
results[key] = v
return results, report_dirs
return results, report_dirs, plugins


def fail(msg: str) -> None:
Expand Down
4 changes: 4 additions & 0 deletions mypy/options.py
View file
Open in desktop
F438
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import sys

from typing import Mapping, Optional, Tuple, List, Pattern, Dict
from mypy.plugin import PluginRegistry

from mypy import defaults

Expand Down Expand Up @@ -113,6 +114,9 @@ def __init__(self) -> None:
self.debug_cache = False
self.quick_and_dirty = False

# plugins
self.plugins = [] # type: List[PluginRegistry]
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is producing an error when running the tests that I don't understand:

mypy/options.py:121: error: Invalid type "mypy.plugin.PluginRegistry"

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My guess is that this is because of a reference cycle. There is a known bug that mypy does not treat named tuples in cycles correctly.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the heads up. I suspected a cycle too, but knew that they seemed to be working in general. I'll see if I can resolve the cycle elsewhere.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just for information here is the issue about cycles #3054


# Per-module options (raw)
self.per_module_options = {} # type: Dict[Pattern[str], Dict[str, object]]

Expand Down
Loading
0