8000 Update the fast parser to work with Python 2.7 by ddfisher · Pull Request #1418 · python/mypy · GitHub
[go: up one dir, main page]

Skip to content

Update the fast parser to work with Python 2.7 #1418

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Apr 22, 2016
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Update the fast parser to work with Python 2.7
  • Loading branch information
ddfisher committed Apr 22, 2016
commit 35c8be481eeb6af5edb510c0030c79ff6b979f64
2 changes: 1 addition & 1 deletion mypy/checkexpr.py
Original file line number Diff line number Diff line change
Expand Up @@ -861,7 +861,7 @@ def visit_op_expr(self, e: OpExpr) -> Type:
if e.op == '*' and isinstance(e.left, ListExpr):
# Expressions of form [...] * e get special type inference.
return self.check_list_multiply(e)
if e.op == '%' and isinstance(e.left, StrExpr):
if e.op == '%' and isinstance(e.left, (StrExpr, BytesExpr)):
return self.strfrm_checker.check_str_interpolation(cast(StrExpr, e.left), e.right)
left_type = self.accept(e.left)

Expand Down
6 changes: 3 additions & 3 deletions mypy/checkstrformat.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
Type, AnyType, TupleType, Instance, UnionType
)
from mypy.nodes import (
Node, StrExpr, TupleExpr, DictExpr, Context
Node, StrExpr, BytesExpr, TupleExpr, DictExpr, Context
)
if False:
# break import cycle only needed for mypy
Expand Down Expand Up @@ -136,7 +136,7 @@ def check_simple_str_interpolation(self, specifiers: List[ConversionSpecifier],
def check_mapping_str_interpolation(self, specifiers: List[ConversionSpecifier],
replacements: Node) -> None:
dict_with_only_str_literal_keys = (isinstance(replacements, DictExpr) and
all(isinstance(k, StrExpr)
all(isinstance(k, (StrExpr, BytesExpr))
for k, v in cast(DictExpr, replacements).items))
if dict_with_only_str_literal_keys:
mapping = {} # type: Dict[str, Type]
Expand Down Expand Up @@ -255,7 +255,7 @@ def check_type(type: Type = None) -> None:
def check_node(node: Node) -> None:
"""int, or str with length 1"""
type = self.accept(node, expected_type)
if isinstance(node, StrExpr) and len(cast(StrExpr, node).value) != 1:
if isinstance(node, (StrExpr, BytesExpr)) and len(cast(StrExpr, node).value) != 1:
self.msg.requires_int_or_char(context)
check_type(type)

Expand Down
4 changes: 2 additions & 2 deletions mypy/exprtotype.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""Translate an expression (Node) to a Type value."""

from mypy.nodes import (
Node, NameExpr, MemberExpr, IndexExpr, TupleExpr, ListExpr, StrExpr, EllipsisExpr
Node, NameExpr, MemberExpr, IndexExpr, TupleExpr, ListExpr, StrExpr, BytesExpr, EllipsisExpr
)
from mypy.parsetype import parse_str_as_type, TypeParseError
from mypy.types import Type, UnboundType, TypeList, EllipsisType
Expand Down Expand Up @@ -42,7 +42,7 @@ def expr_to_unanalyzed_type(expr: Node) -> Type:
elif isinstance(expr, ListExpr):
return TypeList([expr_to_unanalyzed_type(t) for t in expr.items],
line=expr.line)
elif isinstance(expr, StrExpr):
elif isinstance(expr, (StrExpr, BytesExpr)):
# Parse string literal type.
try:
result = parse_str_as_type(expr.value, expr.line)
Expand Down
64 changes: 52 additions & 12 deletions mypy/fastparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl,
WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt,
TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr,
DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr,
DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr,
FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr,
UnaryExpr, FuncExpr, ComparisonExpr,
StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension,
Expand All @@ -21,7 +21,9 @@
from mypy.errors import Errors

try:
from typed_ast import ast27 # type: ignore
from typed_ast import ast35 # type: ignore
from typed_ast import conversions # type: ignore
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe you could provide stubs for these? The untypedness here kind of concerns me.

except ImportError:
print('You must install the typed_ast module before you can run mypy with `--fast-parser`.\n'
'The typed_ast module can be found at https://github.com/ddfisher/typed_ast',
Expand All @@ -43,15 +45,21 @@ def parse(source: Union[str, bytes], fnam: str = None, errors: Errors = None,
"""
is_stub_file = bool(fnam) and fnam.endswith('.pyi')
try:
ast = ast35.parse(source, fnam, 'exec')
if pyversion[0] == 3 or is_stub_file:
ast = ast35.parse(source, fnam, 'exec')
else:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd assert that pyversion[0] == 2 here, or else use >= 3 two lines up (which is what every other pyversion test in mypy uses).

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changed to use >= 3.

ast = ast27.parse(source, fnam, 'exec')
ast = conversions.py2to3(ast)
except SyntaxError as e:
if errors:
errors.set_file('<input>' if fnam is None else fnam)
errors.report(e.lineno, e.msg) # type: ignore
else:
raise
else:
tree = ASTConverter().visit(ast)
tree = ASTConverter(pyversion=pyversion,
custom_typing_module=custom_typing_module,
).visit(ast)
tree.path = fnam
tree.is_stub = is_stub_file
return tree
Expand Down Expand Up @@ -85,10 +93,13 @@ def find(f: Callable[[T], bool], seq: Sequence[T]) -> T:


class ASTConverter(ast35.NodeTransformer):
def __init__(self):
self.in_class = False
def __init__(self, pyversion, custom_typing_module = None):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why no type annotations here?

self.class_nesting = 0
self.imports = []

self.pyversion = pyversion
self.custom_typing_module = custom_typing_module

def generic_visit(self, node):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or here, or in any other method here?

raise RuntimeError('AST node not implemented: ' + str(type(node)))

Expand Down Expand Up @@ -178,6 +189,22 @@ def fix_function_overloads(self, stmts):
ret.append(OverloadedFuncDef(current_overload))
return ret

def in_class(self):
return self.class_nesting > 0

def translate_module_id(self, id: str) -> str:
"""Return the actual, internal module id for a source text id.

For example, translate '__builtin__' in Python 2 to 'builtins'.
"""
if id == self.custom_typing_module:
return 'typing'
elif id == '__builtin__' and self.pyversion[0] == 2:
# HACK: __builtin__ in Python 2 is aliases to builtins. However, the implementation
# is named __builtin__.py (there is another layer of translation elsewhere).
return 'builtins'
return id

def visit_Module(self, mod):
body = self.fix_function_overloads(self.visit(mod.body))

Expand All @@ -200,12 +227,17 @@ def visit_FunctionDef(self, n):
arg_names = [arg.variable.name() for arg in args]
if n.type_comment is not None:
func_type_ast = ast35.parse(n.type_comment, '<func_type>', 'func_type')
arg_types = [a if a is not None else AnyType() for
a in TypeConverter(line=n.lineno).visit(func_type_ast.argtypes)]
# for ellipsis arg
if (len(func_type_ast.argtypes) == 1 and
isinstance(func_type_ast.argtypes[0], ast35.Ellipsis)):
arg_types = [AnyType() for a in args]
else:
arg_types = [a if a is not None else AnyType() for
a in TypeConverter(line=n.lineno).visit(func_type_ast.argtypes)]
return_type = TypeConverter(line=n.lineno).visit(func_type_ast.returns)

# add implicit self type
if self.in_class and len(arg_types) < len(args):
if self.in_class() and len(arg_types) < len(args):
arg_types.insert(0, AnyType())
else:
arg_types = [a.type_annotation for a in args]
Expand Down Expand Up @@ -290,7 +322,7 @@ def stringify_name(self, n):
# expr* decorator_list)
@with_line
def visit_ClassDef(self, n):
self.in_class = True
self.class_nesting += 1
metaclass_arg = find(lambda x: x.arg == 'metaclass', n.keywords)
metaclass = None
if metaclass_arg:
Expand All @@ -302,7 +334,7 @@ def visit_ClassDef(self, n):
self.visit(n.bases),
metaclass=metaclass)
cdef.decorators = self.visit(n.decorator_list)
self.in_class = False
self.class_nesting -= 1
return cdef

# Return(expr? value)
Expand Down Expand Up @@ -397,7 +429,7 @@ def visit_Assert(self, n):
# Import(alias* names)
@with_line
def visit_Import(self, n):
i = Import([(a.name, a.asname) for a in n.names])
i = Import([(self.translate_module_id(a.name), a.asname) for a in n.names])
self.imports.append(i)
return i

Expand All @@ -407,7 +439,7 @@ def visit_ImportFrom(self, n):
if len(n.names) == 1 and n.names[0].name == '*':
i = ImportAll(n.module, n.level)
else:
i = ImportFrom(n.module if n.module is not None else '',
i = ImportFrom(self.translate_module_id(n.module) if n.module is not None else '',
n.level,
[(a.name, a.asname) for a in n.names])
self.imports.append(i)
Expand Down Expand Up @@ -608,12 +640,20 @@ def visit_Num(self, num):
@with_line
def visit_Str(self, n):
return StrExpr(n.s)
# if self.pyversion[0] > 2:
# return StrExpr(n.s)
# else:
# return UnicodeExpr(n.s)

# Bytes(bytes s)
@with_line
def visit_Bytes(self, n):
# TODO: this is kind of hacky
return BytesExpr(str(n.s)[2:-1])
# if self.pyversion[0] > 2:
# return BytesExpr(str(n.s)[2:-1])
# else:
# return StrExpr(str(n.s)[2:-1])

# NameConstant(singleton value)
def visit_NameConstant(self, n):
Expand Down
4 changes: 0 additions & 4 deletions mypy/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,10 +199,6 @@ def parse_version(v):
parser.error('Python version 2 (or --py2) specified, '
'but --use-python-path will search in sys.path of Python 3')

if args.fast_parser and args.python_version and args.python_version[0] == 2:
parser.error('The experimental fast parser is only compatible with Python 3, '
'but Python 2 specified.')

# Set options.
options = Options()
options.dirty_stubs = args.dirty_stubs
Expand Down
12 changes: 6 additions & 6 deletions mypy/semanal.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@
SliceExpr, CastExpr, TypeApplication, Context, SymbolTable,
SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr,
FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr,
StrExpr, PrintStmt, ConditionalExpr, PromoteExpr,
StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr,
ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases,
YieldFromExpr, NamedTupleExpr, NonlocalDecl,
SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr,
Expand Down Expand Up @@ -1279,7 +1279,7 @@ def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> boo
if len(call.args) < 1:
self.fail("Too few arguments for TypeVar()", context)
return False
if not isinstance(call.args[0], StrExpr) or not call.arg_kinds[0] == ARG_POS:
if not isinstance(call.args[0], (StrExpr, BytesExpr)) or not call.arg_kinds[0] == ARG_POS:
self.fail("TypeVar() expects a string literal as first argument", context)
return False
if cast(StrExpr, call.args[0]).value != name:
Expand Down Expand Up @@ -1425,13 +1425,13 @@ def parse_namedtuple_args(self, call: CallExpr,
return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call)
if call.arg_kinds != [ARG_POS, ARG_POS]:
return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call)
if not isinstance(args[0], StrExpr):
if not isinstance(args[0], (StrExpr, BytesExpr)):
return self.fail_namedtuple_arg(
"namedtuple() expects a string literal as the first argument", call)
types = [] # type: List[Type]
ok = True
if not isinstance(args[1], ListExpr):
if fullname == 'collections.namedtuple' and isinstance(args[1], StrExpr):
if fullname == 'collections.namedtuple' and isinstance(args[1], (StrExpr, BytesExpr)):
str_expr = cast(StrExpr, args[1])
items = str_expr.value.split()
else:
Expand All @@ -1441,7 +1441,7 @@ def parse_namedtuple_args(self, call: CallExpr,
listexpr = cast(ListExpr, args[1])
if fullname == 'collections.namedtuple':
# The fields argument contains just names, with implicit Any types.
if any(not isinstance(item, StrExpr) for item in listexpr.items):
if any(not isinstance(item, (StrExpr, BytesExpr)) for item in listexpr.items):
return self.fail_namedtuple_arg("String literal expected as namedtuple() item",
call)
items = [cast(StrExpr, item).value for item in listexpr.items]
Expand All @@ -1462,7 +1462,7 @@ def parse_namedtuple_fields_with_types(self, nodes: List[Node],
return self.fail_namedtuple_arg("Invalid NamedTuple field definition",
item)
name, type_node = item.items
if isinstance(name, StrExpr):
if isinstance(name, (StrExpr, BytesExpr)):
items.append(name.value)
else:
return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item)
Expand Down
0