10000 [mypyc] Support new syntax for generic functions and classes (PEP 695… · python/mypy@415d49f · GitHub
[go: up one dir, main page]
Skip to content

Commit 415d49f

Browse files
JukkaLJukkaL
authored
[mypyc] Support new syntax for generic functions and classes (PEP 695) (#17357)
Generate an implicit `Generic` base class for new-style generic classes. For this to work, also create C statics that can be used to access type variable objects (e.g. `T` or `Ts`) at runtime. These are needed when evaluating base classes. Import `TypeVar` and friends from the `_typing` C extension instead of `typing`, since the latter is pretty slow to import, and we don't want to add a hidden new runtime dependency in case the full `typing` module isn't needed. Generic functions don't need any changes, since they don't support indexing with a type, and type variable types aren't valid in runtime contexts. Type erasure seems sufficient, especially considering that mypyc doesn't support classes nested within functions. (I'm not 100% sure about this though, and we might need to put function type variables into statics eventually.) Update builtins test fixtures used in mypyc tests to not defined type variables such as `T`, since these leak into tests and can produce unexpected or unrealistic results. Ignore upper bounds and value restrictions. These are only used for type checking. This should only affect introspection of type variables, which isn't properly supported in compiled code anyway. New type alias syntax is not supported in this PR.
1 parent b8a0260 commit 415d49f

18 files changed

+382
-81
lines changed

‎mypy/nodes.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2535,8 +2535,9 @@ def __init__(
25352535
default: mypy.types.Type,
25362536
variance: int = INVARIANT,
25372537
is_new_style: bool = False,
2538+
line: int = -1,
25382539
) -> None:
2539-
super().__init__()
2540+
super().__init__(line=line)
25402541
self._name = name
25412542
self._fullname = fullname
25422543
self.upper_bound = upper_bound
@@ -2582,8 +2583,9 @@ def __init__(
25822583
default: mypy.types.Type,
25832584
variance: int = INVARIANT,
25842585
is_new_style: bool = False,
2586+
line: int = -1,
25852587
) -> None:
2586-
super().__init__(name, fullname, upper_bound, default, variance, is_new_style)
2588+
super().__init__(name, fullname, upper_bound, default, variance, is_new_style, line=line)
25872589
self.values = values
25882590

25892591
def accept(self, visitor: ExpressionVisitor[T]) -> T:
@@ -2661,8 +2663,9 @@ def __init__(
26612663
default: mypy.types.Type,
26622664
variance: int = INVARIANT,
26632665
is_new_style: bool = False,
2666+
line: int = -1,
26642667
) -> None:
2665-
super().__init__(name, fullname, upper_bound, default, variance, is_new_style)
2668+
super().__init__(name, fullname, upper_bound, default, variance, is_new_style, line=line)
26662669
self.tuple_fallback = tuple_fallback
26672670

26682671
def accept(self, visitor: ExpressionVisitor[T]) -> T:

‎mypy/semanal.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1709,7 +1709,7 @@ def push_type_args(
17091709
self.scope_stack.append(SCOPE_ANNOTATION)
17101710
tvs: list[tuple[str, TypeVarLikeExpr]] = []
17111711
for p in type_args:
1712-
tv = self.analyze_type_param(p)
1712+
tv = self.analyze_type_param(p, context)
17131713
if tv is None:
17141714
return None
17151715
tvs.append((p.name, tv))
@@ -1732,7 +1732,9 @@ def is_defined_type_param(self, name: str) -> bool:
17321732
return True
17331733
return False
17341734

1735-
def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None:
1735+
def analyze_type_param(
1736+
self, type_param: TypeParam, context: Context
1737+
) -> TypeVarLikeExpr | None:
17361738
fullname = self.qualified_name(type_param.name)
17371739
if type_param.upper_bound:
17381740
upper_bound = self.anal_type(type_param.upper_bound)
@@ -1757,6 +1759,7 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None:
17571759
default=default,
17581760
variance=VARIANCE_NOT_READY,
17591761
is_new_style=True,
1762+
line=context.line,
17601763
)
17611764
elif type_param.kind == PARAM_SPEC_KIND:
17621765
return ParamSpecExpr(
@@ -1765,6 +1768,7 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None:
17651768
upper_bound=upper_bound,
17661769
default=default,
17671770
is_new_style=True,
1771+
line=context.line,
17681772
)
17691773
else:
17701774
assert type_param.kind == TYPE_VAR_TUPLE_KIND
@@ -1777,6 +1781,7 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None:
17771781
tuple_fallback=tuple_fallback,
17781782
default=default,
17791783
is_new_style=True,
1784+
line=context.line,
17801785
)
17811786

17821787
def pop_type_args(self, type_args: list[TypeParam] | None) -> None:

‎mypyc/codegen/emitfunc.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,22 @@
66

77
from mypyc.analysis.blockfreq import frequently_executed_blocks
88
from mypyc.codegen.emit import DEBUG_ERRORS, Emitter, TracebackAndGotoHandler, c_array_initializer
9-
from mypyc.common import MODULE_PREFIX, NATIVE_PREFIX, REG_PREFIX, STATIC_PREFIX, TYPE_PREFIX
9+
from mypyc.common import (
10+
MODULE_PREFIX,
11+
NATIVE_PREFIX,
12+
REG_PREFIX,
13+
STATIC_PREFIX,
14+
TYPE_PREFIX,
15+
TYPE_VAR_PREFIX,
16+
)
1017
from mypyc.ir.class_ir import ClassIR
1118
from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR, all_values
1219
from mypyc.ir.ops import (
1320
ERR_FALSE,
1421
NAMESPACE_MODULE,
1522
NAMESPACE_STATIC,
1623
NAMESPACE_TYPE,
24+
NAMESPACE_TYPE_VAR,
1725
Assign,
1826
AssignMulti,
1927
BasicBlock,
@@ -477,6 +485,7 @@ def visit_set_attr(self, op: SetAttr) -> None:
477485
NAMESPACE_STATIC: STATIC_PREFIX,
478486
NAMESPACE_TYPE: TYPE_PREFIX,
479487
NAMESPACE_MODULE: MODULE_PREFIX,
488+
NAMESPACE_TYPE_VAR: TYPE_VAR_PREFIX,
480489
}
481490

482491
def visit_load_static(self, op: LoadStatic) -> None:

‎mypyc/codegen/emitmodule.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
PREFIX,
4242
RUNTIME_C_FILES,
4343
TOP_LEVEL_NAME,
44+
TYPE_VAR_PREFIX,
4445
shared_lib_name,
4546
short_id_from_name,
4647
use_vectorcall,
@@ -590,6 +591,7 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]:
590591
self.declare_finals(module_name, module.final_names, declarations)
591592
for cl in module.classes:
592593
generate_class_type_decl(cl, emitter, ext_declarations, declarations)
594+
self.declare_type_vars(module_name, module.type_var_names, declarations)
593595
for fn in module.functions:
594596
generate_function_declaration(fn, declarations)
595597

@@ -1063,6 +1065,15 @@ def declare_static_pyobject(self, identifier: str, emitter: Emitter) -> None:
10631065
symbol = emitter.static_name(identifier, None)
10641066
self.declare_global("PyObject *", symbol)
10651067

1068+
def declare_type_vars(self, module: str, type_var_names: list[str], emitter: Emitter) -> None:
1069+
for name in type_var_names:
1070+
static_name = emitter.static_name(name, module, prefix=TYPE_VAR_PREFIX)
1071+
emitter.context.declarations[static_name] = HeaderDeclaration(
1072+
f"PyObject *{static_name};",
1073+
[f"PyObject *{static_name} = NULL;"],
1074+
needs_export=False,
1075+
)
1076+
10661077

10671078
def sort_classes(classes: list[tuple[str, ClassIR]]) -> list[tuple[str, ClassIR]]:
10681079
mod_name = {ir: name for name, ir in classes}

‎mypyc/common.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
STATIC_PREFIX: Final = "CPyStatic_" # Static variables (for literals etc.)
1414
TYPE_PREFIX: Final = "CPyType_" # Type object struct
1515
MODULE_PREFIX: Final = "CPyModule_" # Cached modules
16+
TYPE_VAR_PREFIX: Final = "CPyTypeVar_" # Type variables when using new-style Python 3.12 syntax
1617
ATTR_PREFIX: Final = "_" # Attributes
1718

1819
ENV_ATTR_NAME: Final = "__mypyc_env__"

‎mypyc/ir/module_ir.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,17 @@ def __init__(
2121
functions: list[FuncIR],
2222
classes: list[ClassIR],
2323
final_names: list[tuple[str, RType]],
24+
type_var_names: list[str],
2425
) -> None:
2526
self.fullname = fullname
2627
self.imports = imports.copy()
2728
self.functions = functions
2829
self.classes = classes
2930
self.final_names = final_names
31+
# Names of C statics used for Python 3.12 type variable objects.
32+
# These are only visible in the module that defined them, so no need
33+
# to serialize.
34+
self.type_var_names = type_var_names
3035

3136
def serialize(self) -> JsonDict:
3237
return {
@@ -45,6 +50,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ModuleIR:
4550
[ctx.functions[FuncDecl.get_id_from_json(f)] for f in data["functions"]],
4651
[ClassIR.deserialize(c, ctx) for c in data["classes"]],
4752
[(k, deserialize_type(t, ctx)) for k, t in data["final_names"]],
53+
[],
4854
)
4955

5056

‎mypyc/ir/ops.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -789,6 +789,9 @@ def accept(self, visitor: OpVisitor[T]) -> T:
789789
# Namespace for modules
790790
NAMESPACE_MODULE: Final = "module"
791791

792+
# Namespace for Python 3.12 type variable objects (implicitly created TypeVar instances, etc.)
793+
NAMESPACE_TYPE_VAR: Final = "typevar"
794+
792795

793796
class LoadStatic(RegisterOp):
794797
"""Load a static name (name :: static).

‎mypyc/irbuild/builder.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
from mypyc.ir.func_ir import INVALID_FUNC_DEF, FuncDecl, FuncIR, FuncSignature, RuntimeArg
7070
from mypyc.ir.ops import (
7171
NAMESPACE_MODULE,
72+
NAMESPACE_TYPE_VAR,
7273
Assign,
7374
BasicBlock,
7475
Branch,
@@ -179,6 +180,7 @@ def __init__(
179180
self.function_names: set[tuple[str | None, str]] = set()
180181
self.classes: list[ClassIR] = []
181182
self.final_names: list[tuple[str, RType]] = []
183+
self.type_var_names: list[str] = []
182184
self.callable_class_names: set[str] = set()
183185
self.options = options
184186

@@ -541,6 +543,21 @@ def load_final_static(
541543
error_msg=f'value for final name "{error_name}" was not set',
542544
)
543545

546+
def init_type_var(self, value: Value, name: str, line: int) -> None:
547+
unique_name = name + "___" + str(line)
548+
self.type_var_names.append(unique_name)
549+
self.add(InitStatic(value, unique_name, self.module_name, namespace=NAMESPACE_TYPE_VAR))
550+
551+
def load_type_var(self, name: str, line: int) -> Value:
552+
return self.add(
553+
LoadStatic(
554+
object_rprimitive,
555+
name + "___" + str(line),
556+
self.module_name,
557+
namespace=NAMESPACE_TYPE_VAR,
558+
)
559+
)
560+
544561
def load_literal_value(self, val: int | str | bytes | float | complex | bool) -> Value:
545562
"""Load value of a final name, class-level attribute, or constant folded expression."""
546563
if isinstance(val, bool):

‎mypyc/irbuild/classdef.py

Lines changed: 59 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
from typing import Callable, Final
88

99
from mypy.nodes import (
10+
PARAM_SPEC_KIND,
11+
TYPE_VAR_KIND,
12+
TYPE_VAR_TUPLE_KIND,
1013
AssignmentStmt,
1114
CallExpr,
1215
ClassDef,
@@ -22,6 +25,7 @@
2225
StrExpr,
2326
TempNode,
2427
TypeInfo,
28+
TypeParam,
2529
is_class_var,
2630
)
2731
from mypy.types import ENUM_REMOVED_PROPS, Instance, RawExpressionType, get_proper_type
@@ -63,9 +67,16 @@
6367
)
6468
from mypyc.irbuild.util import dataclass_type, get_func_def, is_constant, is_dataclass_decorator
6569
from mypyc.primitives.dict_ops import dict_new_op, dict_set_item_op
66-
from mypyc.primitives.generic_ops import py_hasattr_op, py_setattr_op
70+
from mypyc.primitives.generic_ops import (
71+
iter_op,
72+
next_op,
73+
py_get_item_op,
74+
py_hasattr_op,
75+
py_setattr_op,
76+
)
6777
from mypyc.primitives.misc_ops import (
6878
dataclass_sleight_of_hand,
79+
import_op,
6980
not_implemented_op,
7081
py_calc_meta_op,
7182
pytype_from_template_op,
@@ -405,8 +416,14 @@ def get_type_annotation(self, stmt: AssignmentStmt) -> TypeInfo | None:
405416
def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value:
406417
# OK AND NOW THE FUN PART
407418
base_exprs = cdef.base_type_exprs + cdef.removed_base_type_exprs
408-
if base_exprs:
409-
bases = [builder.accept(x) for x in base_exprs]
419+
new_style_type_args = cdef.type_args
420+
if new_style_type_args:
421+
bases = [make_generic_base_class(builder, cdef.fullname, new_style_type_args, cdef.line)]
422+
else:
423+
bases = []
424+
425+
if base_exprs or new_style_type_args:
426+
bases.extend([builder.accept(x) for x in base_exprs])
410427
tp_bases = builder.new_tuple(bases, cdef.line)
411428
else:
412429
tp_bases = builder.add(LoadErrorValue(object_rprimitive, is_borrowed=True))
@@ -453,6 +470,45 @@ def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value:
453470
return tp
454471

455472

473+
def make_generic_base_class(
474+
builder: IRBuilder, fullname: str, type_args: list[TypeParam], line: int
475+
) -> Value:
476+
"""Construct Generic[...] base class object for a new-style generic class (Python 3.12)."""
477+
mod = builder.call_c(import_op, [builder.load_str("_typing")], line)
478+
tvs = []
479+
type_var_imported: Value | None = None
480+
for type_param in type_args:
481+
unpack = False
482+
if type_param.kind == TYPE_VAR_KIND:
483+
if type_var_imported:
484+
# Reuse previously imported value as a minor optimization
485+
tvt = type_var_imported
486+
else:
487+
tvt = builder.py_get_attr(mod, "TypeVar", line)
488+
type_var_imported = tvt
489+
elif type_param.kind == TYPE_VAR_TUPLE_KIND:
490+
tvt = builder.py_get_attr(mod, "TypeVarTuple", line)
491+
unpack = True
492+
else:
493+
assert type_param.kind == PARAM_SPEC_KIND
494+
tvt = builder.py_get_attr(mod, "ParamSpec", line)
495+
tv = builder.py_call(tvt, [builder.load_str(type_param.name)], line)
496+
builder.init_type_var(tv, type_param.name, line)
497+
if unpack:
498+
# Evaluate *Ts for a TypeVarTuple
499+
it = builder.call_c(iter_op, [tv], line)
500+
tv = builder.call_c(next_op, [it], line)
501+
tvs.append(tv)
502+
gent = builder.py_get_attr(mod, "Generic", line)
503+
if len(tvs) == 1:
504+
arg = tvs[0]
505+
else:
506+
arg = builder.new_tuple(tvs, line)
507+
508+
base = builder.call_c(py_get_item_op, [gent, arg], line)
509+
return base
510+
511+
456512
# Mypy uses these internally as base classes of TypedDict classes. These are
457513
# lies and don't have any runtime equivalent.
458514
MAGIC_TYPED_DICT_CLASSES: Final[tuple[str, ...]] = (

‎mypyc/irbuild/expression.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
TupleExpr,
4545
TypeApplication,
4646
TypeInfo,
47+
TypeVarLikeExpr,
4748
UnaryExpr,
4849
Var,
4950
)
@@ -106,6 +107,10 @@
106107

107108

108109
def transform_name_expr(builder: IRBuilder, expr: NameExpr) -> Value:
110+
if isinstance(expr.node, TypeVarLikeExpr) and expr.node.is_new_style:
111+
# Reference to Python 3.12 implicit TypeVar/TupleVarTuple/... object.
112+
# These are stored in C statics and not visible in Python namespaces.
113+
return builder.load_type_var(expr.node.name, expr.node.line)
109114
if expr.node is None:
110115
builder.add(
111116
RaiseStandardError(

‎mypyc/irbuild/main.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ def build_ir(
9999
builder.functions,
100100
builder.classes,
101101
builder.final_names,
102+
builder.type_var_names,
102103
)
103104
result[module.fullname] = module_ir
104105
class_irs.extend(builder.classes)

‎mypyc/primitives/generic_ops.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@
178178
)
179179

180180
# obj1[obj2]
181-
method_op(
181+
py_get_item_op = method_op(
182182
name="__getitem__",
183183
arg_types=[object_rprimitive, object_rprimitive],
184184
return_type=object_rprimitive,

0 commit comments

Comments
 (0)
0