From c1dce83afce89c80017403887d1a50c736127dba Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Tue, 12 Nov 2019 16:55:42 -0800 Subject: [PATCH 1/5] [mypyc] Add a mypyc_attr to support interpreted children This operates by generating a "shadow vtable" containing pointers to glue methods that dispatch to the appropriate method via the C API. We then install those shadow vtables in interpreted subclasses so that overridden methods will be called. This does not support directly inheriting from traits, which I think will require generating vtables dynamically (and maybe some more nonsense too.) Closes #296. (Though I will file a follow-up for traits.) --- mypy-requirements.txt | 2 +- mypyc/emitclass.py | 71 +++++++---- mypyc/genops.py | 126 ++++++++++++++++---- mypyc/ops.py | 21 +++- mypyc/test-data/commandline.test | 14 ++- mypyc/test-data/run-classes.test | 82 +++++++++++++ setup.py | 2 +- test-data/unit/lib-stub/mypy_extensions.pyi | 7 +- 8 files changed, 276 insertions(+), 49 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 25e5d743202e..66d15c1516f3 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,3 +1,3 @@ typing_extensions>=3.7.4 -mypy_extensions>=0.4.0,<0.5.0 +mypy_extensions>=0.4.3,<0.5.0 typed_ast>=1.4.0,<1.5.0 diff --git a/mypyc/emitclass.py b/mypyc/emitclass.py index a7d2497ae272..23305e9628bd 100644 --- a/mypyc/emitclass.py +++ b/mypyc/emitclass.py @@ -202,7 +202,7 @@ def emit_line() -> None: fields['tp_basicsize'] = base_size if generate_full: - emitter.emit_line('static PyObject *{}(void);'.format(setup_name)) + emitter.emit_line('static PyObject *{}(PyTypeObject *type);'.format(setup_name)) assert cl.ctor is not None emitter.emit_line(native_function_header(cl.ctor, emitter) + ';') @@ -216,7 +216,15 @@ def emit_line() -> None: generate_dealloc_for_class(cl, dealloc_name, clear_name, emitter) emit_line() generate_native_getters_and_setters(cl, emitter) - vtable_name = generate_vtables(cl, vtable_setup_name, vtable_name, emitter) + + if cl.allow_interpreted_children: + shadow_vtable_name = generate_vtables( + cl, vtable_setup_name + "_shadow", vtable_name + "_shadow", emitter, shadow=True + ) # type: Optional[str] + emit_line() + else: + shadow_vtable_name = None + vtable_name = generate_vtables(cl, vtable_setup_name, vtable_name, emitter, shadow=False) emit_line() if needs_getseters: generate_getseter_declarations(cl, emitter) @@ -241,7 +249,8 @@ def emit_line() -> None: emitter.emit_line() if generate_full: - generate_setup_for_class(cl, setup_name, defaults_fn, vtable_name, emitter) + generate_setup_for_class( + cl, setup_name, defaults_fn, vtable_name, shadow_vtable_name, emitter) emitter.emit_line() generate_constructor_for_class( cl, cl.ctor, init_fn, setup_name, vtable_name, emitter) @@ -344,7 +353,8 @@ def generate_native_getters_and_setters(cl: ClassIR, def generate_vtables(base: ClassIR, vtable_setup_name: str, vtable_name: str, - emitter: Emitter) -> str: + emitter: Emitter, + shadow: bool) -> str: """Emit the vtables and vtable setup functions for a class. This includes both the primary vtable and any trait implementation vtables. @@ -359,8 +369,9 @@ def generate_vtables(base: ClassIR, """ def trait_vtable_name(trait: ClassIR) -> str: - return '{}_{}_trait_vtable'.format( - base.name_prefix(emitter.names), trait.name_prefix(emitter.names)) + return '{}_{}_trait_vtable{}'.format( + base.name_prefix(emitter.names), trait.name_prefix(emitter.names), + '_shadow' if shadow else '') # Emit array definitions with enough space for all the entries emitter.emit_line('static CPyVTableItem {}[{}];'.format( @@ -376,13 +387,16 @@ def trait_vtable_name(trait: ClassIR) -> str: emitter.emit_line('{}{}(void)'.format(NATIVE_PREFIX, vtable_setup_name)) emitter.emit_line('{') + if base.allow_interpreted_children and not shadow: + emitter.emit_line('{}{}_shadow();'.format(NATIVE_PREFIX, vtable_setup_name)) + subtables = [] for trait, vtable in base.trait_vtables.items(): name = trait_vtable_name(trait) - generate_vtable(vtable, name, emitter, []) + generate_vtable(vtable, name, emitter, [], shadow) subtables.append((trait, name)) - generate_vtable(base.vtable_entries, vtable_name, emitter, subtables) + generate_vtable(base.vtable_entries, vtable_name, emitter, subtables, shadow) emitter.emit_line('return 1;') emitter.emit_line('}') @@ -393,7 +407,8 @@ def trait_vtable_name(trait: ClassIR) -> str: def generate_vtable(entries: VTableEntries, vtable_name: str, emitter: Emitter, - subtables: List[Tuple[ClassIR, str]]) -> None: + subtables: List[Tuple[ClassIR, str]], + shadow: bool) -> None: emitter.emit_line('CPyVTableItem {}_scratch[] = {{'.format(vtable_name)) if subtables: emitter.emit_line('/* Array of trait vtables */') @@ -404,10 +419,11 @@ def generate_vtable(entries: VTableEntries, for entry in entries: if isinstance(entry, VTableMethod): + method = entry.shadow_method if shadow and entry.shadow_method else entry.method emitter.emit_line('(CPyVTableItem){}{}{},'.format( emitter.get_group_prefix(entry.method.decl), NATIVE_PREFIX, - entry.method.cname(emitter.names))) + method.cname(emitter.names))) else: cl, attr, is_setter = entry namer = native_setter_name if is_setter else native_getter_name @@ -425,18 +441,27 @@ def generate_setup_for_class(cl: ClassIR, func_name: str, defaults_fn: Optional[FuncIR], vtable_name: str, + shadow_vtable_name: Optional[str], emitter: Emitter) -> None: """Generate a native function that allocates an instance of a class.""" emitter.emit_line('static PyObject *') - emitter.emit_line('{}(void)'.format(func_name)) + emitter.emit_line('{}(PyTypeObject *type)'.format(func_name)) emitter.emit_line('{') emitter.emit_line('{} *self;'.format(cl.struct_name(emitter.names))) - emitter.emit_line('self = ({struct} *){type_struct}->tp_alloc({type_struct}, 0);'.format( - struct=cl.struct_name(emitter.names), - type_struct=emitter.type_struct_name(cl))) + emitter.emit_line('self = ({struct} *)type->tp_alloc(type, 0);'.format( + struct=cl.struct_name(emitter.names))) emitter.emit_line('if (self == NULL)') emitter.emit_line(' return NULL;') - emitter.emit_line('self->vtable = {};'.format(vtable_name)) + + if shadow_vtable_name: + emitter.emit_line('if (type != {}) {{'.format(emitter.type_struct_name(cl))) + emitter.emit_line('self->vtable = {};'.format(shadow_vtable_name)) + emitter.emit_line('} else {') + emitter.emit_line('self->vtable = {};'.format(vtable_name)) + emitter.emit_line('}') + else: + emitter.emit_line('self->vtable = {};'.format(vtable_name)) + for base in reversed(cl.base_mro): for attr, rtype in base.attributes.items(): emitter.emit_line('self->{} = {};'.format( @@ -464,7 +489,7 @@ def generate_constructor_for_class(cl: ClassIR, """Generate a native function that allocates and initializes an instance of a class.""" emitter.emit_line('{}'.format(native_function_header(fn, emitter))) emitter.emit_line('{') - emitter.emit_line('PyObject *self = {}();'.format(setup_name)) + emitter.emit_line('PyObject *self = {}({});'.format(setup_name, emitter.type_struct_name(cl))) emitter.emit_line('if (self == NULL)') emitter.emit_line(' return NULL;') args = ', '.join(['self'] + [REG_PREFIX + arg.name for arg in fn.sig.args]) @@ -525,13 +550,15 @@ def generate_new_for_class(cl: ClassIR, '{}(PyTypeObject *type, PyObject *args, PyObject *kwds)'.format(func_name)) emitter.emit_line('{') # TODO: Check and unbox arguments - emitter.emit_line('if (type != {}) {{'.format(emitter.type_struct_name(cl))) - emitter.emit_line( - 'PyErr_SetString(PyExc_TypeError, "interpreted classes cannot inherit from compiled");') - emitter.emit_line('return NULL;') - emitter.emit_line('}') + if not cl.allow_interpreted_children: + emitter.emit_line('if (type != {}) {{'.format(emitter.type_struct_name(cl))) + emitter.emit_line( + 'PyErr_SetString(PyExc_TypeError, "interpreted classes cannot inherit from compiled");' + ) + emitter.emit_line('return NULL;') + emitter.emit_line('}') - emitter.emit_line('return {}();'.format(setup_name)) + emitter.emit_line('return {}(type);'.format(setup_name)) emitter.emit_line('}') diff --git a/mypyc/genops.py b/mypyc/genops.py index b7e5fd497304..ca3cf850040c 100644 --- a/mypyc/genops.py +++ b/mypyc/genops.py @@ -239,10 +239,55 @@ def is_dataclass(cdef: ClassDef) -> bool: return any(is_dataclass_decorator(d) for d in cdef.decorators) -def is_extension_class(cdef: ClassDef) -> bool: +def get_mypyc_attr_literal(e: Expression) -> Any: + """Convert an expression from a mypyc_attr decorator to a value. + + Supports a pretty limited range.""" + if isinstance(e, (StrExpr, IntExpr, FloatExpr)): + return e.value + elif isinstance(e, RefExpr) and e.fullname == 'builtins.True': + return True + elif isinstance(e, RefExpr) and e.fullname == 'builtins.False': + return False + elif isinstance(e, RefExpr) and e.fullname == 'builtins.None': + return None + return NotImplemented + + +def get_mypyc_attr_call(d: Expression) -> Optional[CallExpr]: + """Check if an expression is a call to mypyc_attr and return it if so.""" + if ( + isinstance(d, CallExpr) + and isinstance(d.callee, RefExpr) + and d.callee.fullname == 'mypy_extensions.mypyc_attr' + ): + return d + return None + + +def get_mypyc_attrs(stmt: Union[ClassDef, Decorator]) -> Dict[str, Any]: + """Collect all the mypyc_attr attributes on a class definition or a function.""" + attrs = {} # type: Dict[str, Any] + for dec in stmt.decorators: + d = get_mypyc_attr_call(dec) + if d: + for name, arg in zip(d.arg_names, d.args): + if name is None: + if isinstance(arg, StrExpr): + attrs[arg.value] = True + else: + attrs[name] = get_mypyc_attr_literal(arg) + + return attrs + - if any(not is_trait_decorator(d) and not is_dataclass_decorator(d) - for d in cdef.decorators): +def is_extension_class(cdef: ClassDef) -> bool: + if any( + not is_trait_decorator(d) + and not is_dataclass_decorator(d) + and not get_mypyc_attr_call(d) + for d in cdef.decorators + ): return False elif (cdef.info.metaclass_type and cdef.info.metaclass_type.type.fullname not in ( 'abc.ABCMeta', 'typing.TypingMeta', 'typing.GenericMeta')): @@ -285,10 +330,11 @@ def specialize_parent_vtable(cls: ClassIR, parent: ClassIR) -> VTableEntries: # TODO: emit a wrapper for __init__ that raises or something if (is_same_method_signature(orig_parent_method.sig, child_method.sig) or orig_parent_method.name == '__init__'): - entry = VTableMethod(entry.cls, entry.name, child_method) + entry = VTableMethod(entry.cls, entry.name, child_method, entry.shadow_method) else: entry = VTableMethod(entry.cls, entry.name, - defining_cls.glue_methods[(entry.cls, entry.name)]) + defining_cls.glue_methods[(entry.cls, entry.name)], + entry.shadow_method) else: # If it is an attribute from a trait, we need to find out # the real class it got mixed in at and point to that. @@ -346,7 +392,8 @@ def compute_vtable(cls: ClassIR) -> None: # TODO: don't generate a new entry when we overload without changing the type if fn == cls.get_method(fn.name): cls.vtable[fn.name] = len(entries) - entries.append(VTableMethod(t, fn.name, fn)) + shadow = cls.glue_methods.get((cls, fn.name)) + entries.append(VTableMethod(t, fn.name, fn, shadow)) # Compute vtables for all of the traits that the class implements if not cls.is_trait: @@ -546,6 +593,10 @@ def prepare_class_def(path: str, module_name: str, cdef: ClassDef, ir = mapper.type_to_ir[cdef.info] info = cdef.info + attrs = get_mypyc_attrs(cdef) + if attrs.get("allow_interpreted_children") is True: + ir.allow_interpreted_children = True + # We sort the table for determinism here on Python 3.5 for name, node in sorted(info.names.items()): # Currenly all plugin generated methods are dummies and not included. @@ -1170,20 +1221,26 @@ def handle_ext_method(self, cdef: ClassDef, fdef: FuncDef) -> None: and not is_same_method_signature(class_ir.method_decls[name].sig, cls.method_decls[name].sig)): + if cls is class_ir and not cls.allow_interpreted_children: + continue + # TODO: Support contravariant subtyping in the input argument for # property setters. Need to make a special glue method for handling this, # similar to gen_glue_property. - if fdef.is_property: - f = self.gen_glue_property(cls.method_decls[name].sig, func_ir, class_ir, - cls, fdef.line) - else: - f = self.gen_glue_method(cls.method_decls[name].sig, func_ir, class_ir, - cls, fdef.line) - + f = self.gen_glue(cls.method_decls[name].sig, func_ir, class_ir, cls, fdef) class_ir.glue_methods[(cls, name)] = f self.functions.append(f) + # If the class allows interpreted children, create glue + # methods that dispatch via the Python API. These will go in a + # "shadow vtable" that will be assigned to interpreted + # children. + if class_ir.allow_interpreted_children: + f = self.gen_glue(func_ir.sig, func_ir, class_ir, class_ir, fdef, do_py_ops=True) + class_ir.glue_methods[(class_ir, name)] = f + self.functions.append(f) + def handle_non_ext_method( self, non_ext: NonExtClassInfo, cdef: ClassDef, fdef: FuncDef) -> None: # Perform the function of visit_method for methods inside non-extension classes. @@ -1482,6 +1539,11 @@ def visit_class_def(self, cdef: ClassDef) -> None: if any(ir.base_mro[i].base != ir. base_mro[i + 1] for i in range(len(ir.base_mro) - 1)): self.error("Non-trait MRO must be linear", cdef.line) + if ir.allow_interpreted_children and any( + not parent.allow_interpreted_children for parent in ir.mro + ): + self.error("Parents must allow interpreted children also", cdef.line) + # Currently, we only create non-extension classes for classes that are # decorated or inherit from Enum. Classes decorated with @trait do not # apply here, and are handled in a different way. @@ -1708,8 +1770,24 @@ def visit_import_all(self, node: ImportAll) -> None: return self.gen_import(node.id, node.line) + def gen_glue(self, sig: FuncSignature, target: FuncIR, + cls: ClassIR, base: ClassIR, fdef: FuncItem, + *, + do_py_ops: bool = False + ) -> FuncIR: + if fdef.is_property: + return self.gen_glue_property( + cls.method_decls[fdef.name].sig, target, cls, base, fdef.line, do_py_ops + ) + else: + return self.gen_glue_method( + cls.method_decls[fdef.name].sig, target, cls, base, fdef.line, do_py_ops + ) + def gen_glue_method(self, sig: FuncSignature, target: FuncIR, - cls: ClassIR, base: ClassIR, line: int) -> FuncIR: + cls: ClassIR, base: ClassIR, line: int, + do_pycall: bool, + ) -> FuncIR: """Generate glue methods that mediate between different method types in subclasses. For example, if we have: @@ -1746,7 +1824,11 @@ def f(self, x: object) -> int: ... arg_names = [arg.name for arg in rt_args] arg_kinds = [concrete_arg_kind(arg.kind) for arg in rt_args] - retval = self.call(target.decl, args, arg_kinds, arg_names, line) + if do_pycall: + retval = self.py_method_call( + args[0], target.name, args[1:], line, arg_kinds[1:], arg_names[1:]) + else: + retval = self.call(target.decl, args, arg_kinds, arg_names, line) retval = self.coerce(retval, sig.ret_type, line) self.add(Return(retval)) @@ -1759,7 +1841,8 @@ def f(self, x: object) -> int: ... blocks, env) def gen_glue_property(self, sig: FuncSignature, target: FuncIR, cls: ClassIR, base: ClassIR, - line: int) -> FuncIR: + line: int, + do_pygetattr: bool) -> FuncIR: """Similarly to methods, properties of derived types can be covariantly subtyped. Thus, properties also require glue. However, this only requires the return type to change. Further, instead of a method call, an attribute get is performed.""" @@ -1768,7 +1851,10 @@ def gen_glue_property(self, sig: FuncSignature, target: FuncIR, cls: ClassIR, ba rt_arg = RuntimeArg(SELF_NAME, RInstance(cls)) arg = self.read(self.add_self_to_env(cls), line) self.ret_types[-1] = sig.ret_type - retval = self.add(GetAttr(arg, target.name, line)) + if do_pygetattr: + retval = self.py_get_attr(arg, target.name, line) + else: + retval = self.add(GetAttr(arg, target.name, line)) retbox = self.coerce(retval, sig.ret_type, line) self.add(Return(retbox)) @@ -3104,7 +3190,7 @@ def py_call(self, arg_values: List[Value], line: int, arg_kinds: Optional[List[int]] = None, - arg_names: Optional[List[Optional[str]]] = None) -> Value: + arg_names: Optional[Sequence[Optional[str]]] = None) -> Value: """Use py_call_op or py_call_with_kwargs_op for function call.""" # If all arguments are positional, we can use py_call_op. if (arg_kinds is None) or all(kind == ARG_POS for kind in arg_kinds): @@ -3153,8 +3239,8 @@ def py_method_call(self, method_name: str, arg_values: List[Value], line: int, - arg_kinds: Optional[List[int]] = None, - arg_names: Optional[List[Optional[str]]] = None) -> Value: + arg_kinds: Optional[List[int]], + arg_names: Optional[Sequence[Optional[str]]]) -> Value: if (arg_kinds is None) or all(kind == ARG_POS for kind in arg_kinds): method_name_reg = self.load_static_unicode(method_name) return self.primitive_op(py_method_call_op, [obj, method_name_reg] + arg_values, line) diff --git a/mypyc/ops.py b/mypyc/ops.py index caab7b5b1ad7..b643138186bf 100644 --- a/mypyc/ops.py +++ b/mypyc/ops.py @@ -1677,15 +1677,24 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> 'FuncIR': # The arrow points to the "start" of the vtable (what vtable pointers # point to) and the bars indicate which parts correspond to the parent # class A's vtable layout. +# +# Classes that allow interpreted code to subclass them also have a +# "shadow vtable" that contains implementations that delegate to +# making a pycall, so that overridden methods in interpreted children +# will be called. (A better strategy could dynamically generate these +# vtables based on which methods are overridden in the children.) # Descriptions of method and attribute entries in class vtables. # The 'cls' field is the class that the method/attr was defined in, # which might be a parent class. +# The 'shadow_method', if present, contains the method that should be +# placed in the class's shadow vtable (if it has one). VTableMethod = NamedTuple( 'VTableMethod', [('cls', 'ClassIR'), ('name', str), - ('method', FuncIR)]) + ('method', FuncIR), + ('shadow_method', Optional[FuncIR])]) VTableAttr = NamedTuple( @@ -1705,6 +1714,7 @@ def serialize_vtable_entry(entry: VTableEntry) -> JsonDict: 'cls': entry.cls.fullname, 'name': entry.name, 'method': entry.method.decl.fullname, + 'shadow_method': entry.shadow_method.decl.fullname if entry.shadow_method else None, } else: return { @@ -1721,7 +1731,9 @@ def serialize_vtable(vtable: VTableEntries) -> List[JsonDict]: def deserialize_vtable_entry(data: JsonDict, ctx: DeserMaps) -> VTableEntry: if data['.class'] == 'VTableMethod': - return VTableMethod(ctx.classes[data['cls']], data['name'], ctx.functions[data['method']]) + return VTableMethod( + ctx.classes[data['cls']], data['name'], ctx.functions[data['method']], + ctx.functions[data['shadow_method']] if data['shadow_method'] else None) elif data['.class'] == 'VTableAttr': return VTableAttr(ctx.classes[data['cls']], data['name'], data['is_setter']) assert False, "Bogus vtable .class: %s" % data['.class'] @@ -1750,6 +1762,7 @@ def __init__(self, name: str, module_name: str, is_trait: bool = False, self.is_augmented = False self.inherits_python = False self.has_dict = False + self.allow_interpreted_children = False # If this a subclass of some built-in python class, the name # of the object for that class. We currently only support this # in a few ad-hoc cases. @@ -1878,7 +1891,7 @@ def get_method(self, name: str) -> Optional[FuncIR]: def subclasses(self) -> Optional[Set['ClassIR']]: """Return all subclassses of this class, both direct and indirect.""" - if self.children is None: + if self.children is None or self.allow_interpreted_children: return None result = set(self.children) for child in self.children: @@ -1914,6 +1927,7 @@ def serialize(self) -> JsonDict: 'is_augmented': self.is_augmented, 'inherits_python': self.inherits_python, 'has_dict': self.has_dict, + 'allow_interpreted_children': self.allow_interpreted_children, 'builtin_base': self.builtin_base, 'ctor': self.ctor.serialize(), # We serialize dicts as lists to ensure order is preserved @@ -1963,6 +1977,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> 'ClassIR': ir.is_augmented = data['is_augmented'] ir.inherits_python = data['inherits_python'] ir.has_dict = data['has_dict'] + ir.allow_interpreted_children = data['allow_interpreted_children'] ir.builtin_base = data['builtin_base'] ir.ctor = FuncDecl.deserialize(data['ctor'], ctx) ir.attributes = OrderedDict( diff --git a/mypyc/test-data/commandline.test b/mypyc/test-data/commandline.test index d6384b182b77..a82b42138211 100644 --- a/mypyc/test-data/commandline.test +++ b/mypyc/test-data/commandline.test @@ -100,7 +100,7 @@ def f(x: int) -> int: [file test.py] from typing import List, Any from typing_extensions import Final -from mypy_extensions import trait +from mypy_extensions import trait, mypyc_attr def busted(b: bool) -> None: for i in range(1, 10, 0): # E: range() step can't be zero @@ -137,6 +137,10 @@ class NeverMetaclass(type): # E: Inheriting from most builtin types is unimplem class Concrete1: pass +@trait +class PureTrait: + pass + @trait class Trait1(Concrete1): pass @@ -180,3 +184,11 @@ def f(l: List[object]) -> None: for i in l: if x is None: x = i + +@mypyc_attr(allow_interpreted_children=True) +class AllowInterp1(Concrete1): # E: Parents must allow interpreted children also + pass + +@mypyc_attr(allow_interpreted_children=True) +class AllowInterp2(PureTrait): # E: Parents must allow interpreted children also + pass diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 1d17a2b71dff..cb7df5febe74 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -1216,3 +1216,85 @@ test(b, 20) test(d, 30) test(B, -1) test(D, -2) + +[case testInterpretedInherit] +from mypy_extensions import mypyc_attr, trait + +@mypyc_attr(allow_interpreted_children=True) +class Top: + def spam(self) -> str: + return "grandparent" + +@mypyc_attr(allow_interpreted_children=True) +@trait +class Trait: + def trait_method(self) -> str: + return "trait" + +@mypyc_attr(allow_interpreted_children=True) +class Foo(Top, Trait): + def __init__(self, x: int) -> None: + self.x = 10 + + def foo(self) -> str: + return "parent foo: " + self.bar(self.x) + + def bar(self, x: int) -> str: + return "parent bar: {}".format(x + self.x) + + @property + def read_property(self) -> str: + return "parent prop" + +def foo(x: Foo) -> str: + return x.foo() + +def bar(x: Foo, y: int) -> str: + return x.bar(y) + +def spam(x: Top) -> str: + return x.spam() + +def prop(x: Foo) -> str: + return x.read_property + +def trait_method(x: Trait) -> str: + return x.trait_method() + +[file interp.py] +from native import Foo + +class Bar(Foo): + def bar(self, x: int) -> str: + return "child bar: {}".format(x + self.x) + + def spam(self) -> str: + return "child" + + @property + def read_property(self) -> str: + return "child prop" + + def trait_method(self) -> str: + return "child" + +[file driver.py] +from native import Foo, foo, bar, spam, prop, trait_method +from interp import Bar + +x = Foo(10) +y = Bar(20) + +assert isinstance(y, Bar) +assert y.x == 10 +assert y.bar(10) == "child bar: 20" +assert y.foo() == "parent foo: child bar: 20" +assert foo(y) == "parent foo: child bar: 20" +assert bar(y, 30) == "child bar: 40" +assert spam(y) == "child" +assert y.read_property == "child prop" +assert prop(x) == "parent prop" +assert prop(y) == "child prop" + +assert y.trait_method() == "child" +assert trait_method(y) == "child" diff --git a/setup.py b/setup.py index bcb6d2a0119e..b11cc5bf0560 100644 --- a/setup.py +++ b/setup.py @@ -192,7 +192,7 @@ def run(self): # When changing this, also update mypy-requirements.txt. install_requires=['typed_ast >= 1.4.0, < 1.5.0', 'typing_extensions>=3.7.4', - 'mypy_extensions >= 0.4.0, < 0.5.0', + 'mypy_extensions >= 0.4.3, < 0.5.0', ], # Same here. extras_require={'dmypy': 'psutil >= 4.0'}, diff --git a/test-data/unit/lib-stub/mypy_extensions.pyi b/test-data/unit/lib-stub/mypy_extensions.pyi index 8fb5942cd983..306d217f478e 100644 --- a/test-data/unit/lib-stub/mypy_extensions.pyi +++ b/test-data/unit/lib-stub/mypy_extensions.pyi @@ -1,6 +1,6 @@ # NOTE: Requires fixtures/dict.pyi from typing import ( - Dict, Type, TypeVar, Optional, Any, Generic, Mapping, NoReturn as NoReturn, Iterator + Any, Dict, Type, TypeVar, Optional, Any, Generic, Mapping, NoReturn as NoReturn, Iterator ) import sys @@ -42,4 +42,9 @@ def TypedDict(typename: str, fields: Dict[str, Type[_T]], *, total: Any = ...) - # when a Type[_T] is expected, so we can't give it the type we want. def trait(cls: Any) -> Any: ... +# The real type is in the comment but it isn't safe to use **kwargs in +# a lib-stub because the fixtures might not have dict. Argh! +# def mypyc_attr(*attrs: str, **kwattrs: object) -> Callable[[_T], _T]: ... +mypyc_attr: Any + class FlexibleAlias(Generic[_T, _U]): ... From e09d9311d9087aa711528b9e704791a11b6f300a Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Mon, 25 Nov 2019 11:25:12 -0800 Subject: [PATCH 2/5] Fix error introduced in refactor --- mypyc/genops.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/mypyc/genops.py b/mypyc/genops.py index ca3cf850040c..a2ae27270346 100644 --- a/mypyc/genops.py +++ b/mypyc/genops.py @@ -1216,20 +1216,17 @@ def handle_ext_method(self, cdef: ClassDef, fdef: FuncDef) -> None: # If this overrides a parent class method with a different type, we need # to generate a glue method to mediate between them. - for cls in class_ir.mro[1:]: - if (name in cls.method_decls and name != '__init__' + for base in class_ir.mro[1:]: + if (name in base.method_decls and name != '__init__' and not is_same_method_signature(class_ir.method_decls[name].sig, - cls.method_decls[name].sig)): - - if cls is class_ir and not cls.allow_interpreted_children: - continue + base.method_decls[name].sig)): # TODO: Support contravariant subtyping in the input argument for # property setters. Need to make a special glue method for handling this, # similar to gen_glue_property. - f = self.gen_glue(cls.method_decls[name].sig, func_ir, class_ir, cls, fdef) - class_ir.glue_methods[(cls, name)] = f + f = self.gen_glue(base.method_decls[name].sig, func_ir, class_ir, base, fdef) + class_ir.glue_methods[(base, name)] = f self.functions.append(f) # If the class allows interpreted children, create glue @@ -1776,13 +1773,9 @@ def gen_glue(self, sig: FuncSignature, target: FuncIR, do_py_ops: bool = False ) -> FuncIR: if fdef.is_property: - return self.gen_glue_property( - cls.method_decls[fdef.name].sig, target, cls, base, fdef.line, do_py_ops - ) + return self.gen_glue_property(sig, target, cls, base, fdef.line, do_py_ops) else: - return self.gen_glue_method( - cls.method_decls[fdef.name].sig, target, cls, base, fdef.line, do_py_ops - ) + return self.gen_glue_method(sig, target, cls, base, fdef.line, do_py_ops) def gen_glue_method(self, sig: FuncSignature, target: FuncIR, cls: ClassIR, base: ClassIR, line: int, From 8ed1ea473099bbd9269d3eb40212de2bcb8dfa84 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Wed, 27 Nov 2019 14:04:26 -0800 Subject: [PATCH 3/5] doc tweaks --- mypyc/emitclass.py | 7 +++++++ mypyc/genops.py | 23 +++++++++++++++++++++-- mypyc/ops.py | 7 ++++++- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/mypyc/emitclass.py b/mypyc/emitclass.py index 23305e9628bd..6c704cfeeb77 100644 --- a/mypyc/emitclass.py +++ b/mypyc/emitclass.py @@ -202,6 +202,9 @@ def emit_line() -> None: fields['tp_basicsize'] = base_size if generate_full: + # Declare setup method that allocates and initializes an object. type is the + # type of the class being initialized, which could be another class if there + # is an interpreted subclass. emitter.emit_line('static PyObject *{}(PyTypeObject *type);'.format(setup_name)) assert cl.ctor is not None emitter.emit_line(native_function_header(cl.ctor, emitter) + ';') @@ -364,8 +367,12 @@ def generate_vtables(base: ClassIR, emit empty array definitions to store the vtables and a function to populate them. + If shadow is True, generate "shadow vtables" that point to the + shadow glue methods (which should dispatch via the Python C-API). + Returns the expression to use to refer to the vtable, which might be different than the name, if there are trait vtables. + """ def trait_vtable_name(trait: ClassIR) -> str: diff --git a/mypyc/genops.py b/mypyc/genops.py index a2ae27270346..f45d73f89ed4 100644 --- a/mypyc/genops.py +++ b/mypyc/genops.py @@ -392,6 +392,8 @@ def compute_vtable(cls: ClassIR) -> None: # TODO: don't generate a new entry when we overload without changing the type if fn == cls.get_method(fn.name): cls.vtable[fn.name] = len(entries) + # If the class contains a glue method referring to itself, that is a + # shadow glue method to support interpreted subclasses. shadow = cls.glue_methods.get((cls, fn.name)) entries.append(VTableMethod(t, fn.name, fn, shadow)) @@ -1772,6 +1774,14 @@ def gen_glue(self, sig: FuncSignature, target: FuncIR, *, do_py_ops: bool = False ) -> FuncIR: + """Generate glue methods that mediate between different method types in subclasses. + + Works on both properties and methods. See gen_glue_methods below for more details. + + If do_py_ops is True, then the glue methods should use generic + C API operations instead of direct calls, to enable generating + "shadow" glue methods that work with interpreted subclasses. + """ if fdef.is_property: return self.gen_glue_property(sig, target, cls, base, fdef.line, do_py_ops) else: @@ -1802,6 +1812,9 @@ def f(self, x: object) -> int: ... we need to generate glue methods that mediate between the different versions by coercing the arguments and return values. + + If do_pycall is True, then make the call using the C API + instead of a native call. """ self.enter(FuncInfo()) self.ret_types[-1] = sig.ret_type @@ -1836,9 +1849,15 @@ def f(self, x: object) -> int: ... def gen_glue_property(self, sig: FuncSignature, target: FuncIR, cls: ClassIR, base: ClassIR, line: int, do_pygetattr: bool) -> FuncIR: - """Similarly to methods, properties of derived types can be covariantly subtyped. Thus, + """Generate glue methods for properties that mediate between different subclass types. + + Similarly to methods, properties of derived types can be covariantly subtyped. Thus, properties also require glue. However, this only requires the return type to change. - Further, instead of a method call, an attribute get is performed.""" + Further, instead of a method call, an attribute get is performed. + + If do_pygetattr is True, then get the attribute using the C + API instead of a native call. + """ self.enter(FuncInfo()) rt_arg = RuntimeArg(SELF_NAME, RInstance(cls)) diff --git a/mypyc/ops.py b/mypyc/ops.py index b643138186bf..e560438d598f 100644 --- a/mypyc/ops.py +++ b/mypyc/ops.py @@ -1762,6 +1762,7 @@ def __init__(self, name: str, module_name: str, is_trait: bool = False, self.is_augmented = False self.inherits_python = False self.has_dict = False + # Do we allow interpreted subclasses? Derived from a mypyc_attr. self.allow_interpreted_children = False # If this a subclass of some built-in python class, the name # of the object for that class. We currently only support this @@ -1890,7 +1891,11 @@ def get_method(self, name: str) -> Optional[FuncIR]: return res[0] if res else None def subclasses(self) -> Optional[Set['ClassIR']]: - """Return all subclassses of this class, both direct and indirect.""" + """Return all subclassses of this class, both direct and indirect. + + Return None if it is impossible to identify all subclasses, for example + because we are performing separate compilation. + """ if self.children is None or self.allow_interpreted_children: return None result = set(self.children) From 632af66f310843f0be5580251913a6df349b76dc Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Wed, 27 Nov 2019 15:55:58 -0800 Subject: [PATCH 4/5] change attribute name --- mypyc/emitclass.py | 6 +++--- mypyc/genops.py | 16 +++++++++------- mypyc/ops.py | 8 ++++---- mypyc/test-data/commandline.test | 8 ++++---- mypyc/test-data/run-classes.test | 6 +++--- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/mypyc/emitclass.py b/mypyc/emitclass.py index 6c704cfeeb77..37edcb0c2b78 100644 --- a/mypyc/emitclass.py +++ b/mypyc/emitclass.py @@ -220,7 +220,7 @@ def emit_line() -> None: emit_line() generate_native_getters_and_setters(cl, emitter) - if cl.allow_interpreted_children: + if cl.allow_interpreted_subclasses: shadow_vtable_name = generate_vtables( cl, vtable_setup_name + "_shadow", vtable_name + "_shadow", emitter, shadow=True ) # type: Optional[str] @@ -394,7 +394,7 @@ def trait_vtable_name(trait: ClassIR) -> str: emitter.emit_line('{}{}(void)'.format(NATIVE_PREFIX, vtable_setup_name)) emitter.emit_line('{') - if base.allow_interpreted_children and not shadow: + if base.allow_interpreted_subclasses and not shadow: emitter.emit_line('{}{}_shadow();'.format(NATIVE_PREFIX, vtable_setup_name)) subtables = [] @@ -557,7 +557,7 @@ def generate_new_for_class(cl: ClassIR, '{}(PyTypeObject *type, PyObject *args, PyObject *kwds)'.format(func_name)) emitter.emit_line('{') # TODO: Check and unbox arguments - if not cl.allow_interpreted_children: + if not cl.allow_interpreted_subclasses: emitter.emit_line('if (type != {}) {{'.format(emitter.type_struct_name(cl))) emitter.emit_line( 'PyErr_SetString(PyExc_TypeError, "interpreted classes cannot inherit from compiled");' diff --git a/mypyc/genops.py b/mypyc/genops.py index f45d73f89ed4..bbc5724d7b36 100644 --- a/mypyc/genops.py +++ b/mypyc/genops.py @@ -596,8 +596,8 @@ def prepare_class_def(path: str, module_name: str, cdef: ClassDef, info = cdef.info attrs = get_mypyc_attrs(cdef) - if attrs.get("allow_interpreted_children") is True: - ir.allow_interpreted_children = True + if attrs.get("allow_interpreted_subclasses") is True: + ir.allow_interpreted_subclasses = True # We sort the table for determinism here on Python 3.5 for name, node in sorted(info.names.items()): @@ -1235,7 +1235,7 @@ def handle_ext_method(self, cdef: ClassDef, fdef: FuncDef) -> None: # methods that dispatch via the Python API. These will go in a # "shadow vtable" that will be assigned to interpreted # children. - if class_ir.allow_interpreted_children: + if class_ir.allow_interpreted_subclasses: f = self.gen_glue(func_ir.sig, func_ir, class_ir, class_ir, fdef, do_py_ops=True) class_ir.glue_methods[(class_ir, name)] = f self.functions.append(f) @@ -1538,10 +1538,12 @@ def visit_class_def(self, cdef: ClassDef) -> None: if any(ir.base_mro[i].base != ir. base_mro[i + 1] for i in range(len(ir.base_mro) - 1)): self.error("Non-trait MRO must be linear", cdef.line) - if ir.allow_interpreted_children and any( - not parent.allow_interpreted_children for parent in ir.mro - ): - self.error("Parents must allow interpreted children also", cdef.line) + if ir.allow_interpreted_subclasses: + for parent in ir.mro: + if not parent.allow_interpreted_subclasses: + self.error( + 'Base class "{}" does not allow interpreted subclasses'.format( + parent.fullname), cdef.line) # Currently, we only create non-extension classes for classes that are # decorated or inherit from Enum. Classes decorated with @trait do not diff --git a/mypyc/ops.py b/mypyc/ops.py index e560438d598f..9c56ebed0feb 100644 --- a/mypyc/ops.py +++ b/mypyc/ops.py @@ -1763,7 +1763,7 @@ def __init__(self, name: str, module_name: str, is_trait: bool = False, self.inherits_python = False self.has_dict = False # Do we allow interpreted subclasses? Derived from a mypyc_attr. - self.allow_interpreted_children = False + self.allow_interpreted_subclasses = False # If this a subclass of some built-in python class, the name # of the object for that class. We currently only support this # in a few ad-hoc cases. @@ -1896,7 +1896,7 @@ def subclasses(self) -> Optional[Set['ClassIR']]: Return None if it is impossible to identify all subclasses, for example because we are performing separate compilation. """ - if self.children is None or self.allow_interpreted_children: + if self.children is None or self.allow_interpreted_subclasses: return None result = set(self.children) for child in self.children: @@ -1932,7 +1932,7 @@ def serialize(self) -> JsonDict: 'is_augmented': self.is_augmented, 'inherits_python': self.inherits_python, 'has_dict': self.has_dict, - 'allow_interpreted_children': self.allow_interpreted_children, + 'allow_interpreted_subclasses': self.allow_interpreted_subclasses, 'builtin_base': self.builtin_base, 'ctor': self.ctor.serialize(), # We serialize dicts as lists to ensure order is preserved @@ -1982,7 +1982,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> 'ClassIR': ir.is_augmented = data['is_augmented'] ir.inherits_python = data['inherits_python'] ir.has_dict = data['has_dict'] - ir.allow_interpreted_children = data['allow_interpreted_children'] + ir.allow_interpreted_subclasses = data['allow_interpreted_subclasses'] ir.builtin_base = data['builtin_base'] ir.ctor = FuncDecl.deserialize(data['ctor'], ctx) ir.attributes = OrderedDict( diff --git a/mypyc/test-data/commandline.test b/mypyc/test-data/commandline.test index a82b42138211..b77c3dd9ffd5 100644 --- a/mypyc/test-data/commandline.test +++ b/mypyc/test-data/commandline.test @@ -185,10 +185,10 @@ def f(l: List[object]) -> None: if x is None: x = i -@mypyc_attr(allow_interpreted_children=True) -class AllowInterp1(Concrete1): # E: Parents must allow interpreted children also +@mypyc_attr(allow_interpreted_subclasses=True) +class AllowInterp1(Concrete1): # E: Base class "test.Concrete1" does not allow interpreted subclasses pass -@mypyc_attr(allow_interpreted_children=True) -class AllowInterp2(PureTrait): # E: Parents must allow interpreted children also +@mypyc_attr(allow_interpreted_subclasses=True) +class AllowInterp2(PureTrait): # E: Base class "test.PureTrait" does not allow interpreted subclasses pass diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index cb7df5febe74..d20b58fa55a0 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -1220,18 +1220,18 @@ test(D, -2) [case testInterpretedInherit] from mypy_extensions import mypyc_attr, trait -@mypyc_attr(allow_interpreted_children=True) +@mypyc_attr(allow_interpreted_subclasses=True) class Top: def spam(self) -> str: return "grandparent" -@mypyc_attr(allow_interpreted_children=True) +@mypyc_attr(allow_interpreted_subclasses=True) @trait class Trait: def trait_method(self) -> str: return "trait" -@mypyc_attr(allow_interpreted_children=True) +@mypyc_attr(allow_interpreted_subclasses=True) class Foo(Top, Trait): def __init__(self, x: int) -> None: self.x = 10 From 985ce438b26e0fe35e4ed3674350368872f033be Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Wed, 27 Nov 2019 18:38:45 -0800 Subject: [PATCH 5/5] add a bunch of tests --- mypyc/test-data/run-classes.test | 84 +++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 8 deletions(-) diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index d20b58fa55a0..a909f64def12 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -1218,8 +1218,12 @@ test(B, -1) test(D, -2) [case testInterpretedInherit] +from typing import TypeVar, Any, overload from mypy_extensions import mypyc_attr, trait +T = TypeVar('T') +def dec(x: T) -> T: return x + @mypyc_attr(allow_interpreted_subclasses=True) class Top: def spam(self) -> str: @@ -1234,7 +1238,7 @@ class Trait: @mypyc_attr(allow_interpreted_subclasses=True) class Foo(Top, Trait): def __init__(self, x: int) -> None: - self.x = 10 + self.x = x def foo(self) -> str: return "parent foo: " + self.bar(self.x) @@ -1242,10 +1246,23 @@ class Foo(Top, Trait): def bar(self, x: int) -> str: return "parent bar: {}".format(x + self.x) + @dec + def decorated(self) -> str: + return "decorated parent" + @property def read_property(self) -> str: return "parent prop" + @overload + def overloaded(self, index: int) -> int: ... + + @overload + def overloaded(self, index: str) -> str: ... + + def overloaded(self, index: Any) -> Any: + return index + def foo(x: Foo) -> str: return x.foo() @@ -1255,13 +1272,20 @@ def bar(x: Foo, y: int) -> str: def spam(x: Top) -> str: return x.spam() +def decorated(x: Foo) -> str: + return x.decorated() + def prop(x: Foo) -> str: return x.read_property def trait_method(x: Trait) -> str: return x.trait_method() +def overloaded(x: Foo, s: str) -> str: + return x.overloaded(s) + [file interp.py] +from typing import Any from native import Foo class Bar(Foo): @@ -1269,32 +1293,76 @@ class Bar(Foo): return "child bar: {}".format(x + self.x) def spam(self) -> str: + assert super().spam() == "grandparent" return "child" @property def read_property(self) -> str: return "child prop" + def decorated(self) -> str: + return "decorated child" + def trait_method(self) -> str: return "child" + def overloaded(self, index: Any) -> Any: + return index + index + + +class InterpBase: + def eggs(self) -> str: + return "eggs" + +class Baz(InterpBase, Bar): + def __init__(self) -> None: + super().__init__(1000) + self.z = self.read_property + [file driver.py] -from native import Foo, foo, bar, spam, prop, trait_method -from interp import Bar +from native import Foo, foo, bar, spam, decorated, overloaded, prop, trait_method +from interp import Bar, Baz +from unittest.mock import patch +from testutil import assertRaises x = Foo(10) y = Bar(20) +z = Baz() assert isinstance(y, Bar) -assert y.x == 10 -assert y.bar(10) == "child bar: 20" -assert y.foo() == "parent foo: child bar: 20" -assert foo(y) == "parent foo: child bar: 20" -assert bar(y, 30) == "child bar: 40" +assert y.x == 20 +assert y.bar(10) == "child bar: 30" +assert y.foo() == "parent foo: child bar: 40" +assert foo(y) == "parent foo: child bar: 40" +assert bar(y, 30) == "child bar: 50" +y.x = 30 +assert bar(y, 30) == "child bar: 60" + assert spam(y) == "child" assert y.read_property == "child prop" assert prop(x) == "parent prop" assert prop(y) == "child prop" +assert y.decorated() == "decorated child" +assert decorated(y) == "decorated child" +assert y.overloaded("test") == "testtest" +assert overloaded(y, "test") == "testtest" assert y.trait_method() == "child" assert trait_method(y) == "child" + +assert z.bar(10) == "child bar: 1010" +assert bar(z, 10) == "child bar: 1010" +assert z.z == "child prop" +assert z.eggs() == "eggs" + +with patch("interp.Bar.spam", lambda self: "monkey patched"): + assert y.spam() == "monkey patched" + spam(y) == "monkey patched" + +with patch("interp.Bar.spam", lambda self: 20): + assert y.spam() == 20 + with assertRaises(TypeError, "str object expected; got int"): + spam(y) + +with assertRaises(TypeError, "int object expected; got str"): + y.x = "test"