8000 [3.11] gh-91162: Support splitting of unpacked arbitrary-length tuple… · python/cpython@8dc5df4 · GitHub
[go: up one dir, main page]
Skip to content

Commit 8dc5df4

Browse files
miss-islingtonserhiy-storchaka
miss-islington
and
serhiy-storchaka
authored
[3.11] gh-91162: Support splitting of unpacked arbitrary-length tuple over TypeVar and TypeVarTuple parameters (alt) (GH-93412) (GH-93746)
For example: A[T, *Ts][*tuple[int, ...]] -> A[int, *tuple[int, ...]] A[*Ts, T][*tuple[int, ...]] -> A[*tuple[int, ...], int] (cherry picked from commit 3473817) Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
1 parent 7aa4038 commit 8dc5df4

File tree

7 files changed

+2288
-2293
lines changed

7 files changed

+2288
-2293
lines changed

‎Doc/data/python3.11.abi

+2,182-2,166
Large diffs are not rendered by default.

‎Include/internal/pycore_global_strings.h

+1
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,7 @@ struct _Py_global_strings {
202202
STRUCT_FOR_ID(__truediv__)
203203
STRUCT_FOR_ID(__trunc__)
204204
STRUCT_FOR_ID(__typing_is_unpacked_typevartuple__)
205+
STRUCT_FOR_ID(__typing_prepare_subst__)
205206
STRUCT_FOR_ID(__typing_subst__)
206207
STRUCT_FOR_ID(__typing_unpacked_tuple_args__)
207208
STRUCT_FOR_ID(__warningregistry__)

‎Include/internal/pycore_runtime_init.h

+1
Original file line numberDiff line numberDiff line change
@@ -825,6 +825,7 @@ extern "C" {
825825
INIT_ID(__truediv__), \
826826
INIT_ID(__trunc__), \
827827
INIT_ID(__typing_is_unpacked_typevartuple__), \
828+
INIT_ID(__typing_prepare_subst__), \
828829
INIT_ID(__typing_subst__), \
829830
INIT_ID(__typing_unpacked_tuple_args__), \
830831
INIT_ID(__warningregistry__), \

‎Lib/test/test_typing.py

+10-7
Original file line numberDiff line numberDiff line change
@@ -753,14 +753,11 @@ class C(Generic[*Ts]): pass
753753
('generic[*Ts]', '[*tuple_type[int]]', 'generic[int]'),
754754
('generic[*Ts]', '[*tuple_type[*Ts]]', 'generic[*Ts]'),
755755
('generic[*Ts]', '[*tuple_type[int, str]]', 'generic[int, str]'),
756+
('generic[*Ts]', '[str, *tuple_type[int, ...], bool]', 'generic[str, *tuple_type[int, ...], bool]'),
756757
('generic[*Ts]', '[tuple_type[int, ...]]', 'generic[tuple_type[int, ...]]'),
757758
('generic[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'generic[tuple_type[int, ...], tuple_type[str, ...]]'),
758759
('generic[*Ts]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...]]'),
759-
760-
# Technically, multiple unpackings are forbidden by PEP 646, but we
761-
# choose to be less restrictive at runtime, to allow folks room
762-
# to experiment. So all three of these should be valid.
763-
('generic[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'generic[*tuple_type[int, ...], *tuple_type[str, ...]]'),
760+
('generic[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'TypeError'),
764761

765762
('generic[*Ts]', '[*Ts]', 'generic[*Ts]'),
766763
('generic[*Ts]', '[T, *Ts]', 'generic[T, *Ts]'),
@@ -772,15 +769,21 @@ class C(Generic[*Ts]): pass
772769
('generic[list[T], *Ts]', '[int, str]', 'generic[list[int], str]'),
773770
('generic[list[T], *Ts]', '[int, str, bool]', 'generic[list[int], str, bool]'),
774771

775-
('generic[T, *Ts]', '[*tuple[int, ...]]', 'TypeError'), # Should be generic[int, *tuple[int, ...]]
776-
777772
('generic[*Ts, T]', '[int]', 'generic[int]'),
778773
('generic[*Ts, T]', '[int, str]', 'generic[int, str]'),
779774
('generic[*Ts, T]', '[int, str, bool]', 'generic[int, str, bool]'),
780775
('generic[*Ts, list[T]]', '[int]', 'generic[list[int]]'),
781776
('generic[*Ts, list[T]]', '[int, str]', 'generic[int, list[str]]'),
782777
('generic[*Ts, list[T]]', '[int, str, bool]', 'generic[int, str, list[bool]]'),
783778

779+
('generic[T, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...]]'),
780+
('generic[*Ts, T]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], int]'),
781+
('generic[T1, *Ts, T2]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...], int]'),
782+
('generic[T, str, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, str, *tuple_type[int, ...]]'),
783+
('generic[*Ts, str, T]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], str, int]'),
784+
('generic[list[T], *Ts]', '[*tuple_type[int, ...]]', 'generic[list[int], *tuple_type[int, ...]]'),
785+
('generic[*Ts, list[T]]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], list[int]]'),
786+
784787
('generic[T, *tuple_type[int, ...]]', '[str]', 'generic[str, *tuple_type[int, ...]]'),
785788
('generic[T1, T2, *tuple_type[int, ...]]', '[str, bool]', 'generic[str, bool, *tuple_type[int, ...]]'),
786789
('generic[T1, *tuple_type[int, ...], T2]', '[str, bool]', 'generic[str, *tuple_type[int, ...], bool]'),

‎Lib/typing.py

+46-64
Original file line numberDiff line numberDiff line change
@@ -1065,6 +1065,42 @@ def __repr__(self):
10651065
def __typing_subst__(self, arg):
10661066
raise TypeError("Substitution of bare TypeVarTuple is not supported")
10671067

1068+
def __typing_prepare_subst__(self, alias, args):
1069+
params = alias.__parameters__
1070+
typevartuple_index = params.index(self)
1071+
for param in enumerate(params[typevartuple_index + 1:]):
1072+
if isinstance(param, TypeVarTuple):
1073+
raise TypeError(f"More than one TypeVarTuple parameter in {alias}")
1074+
1075+
alen = len(args)
1076+
plen = len(params)
1077+
left = typevartuple_index
1078+
right = plen - typevartuple_index - 1
1079+
var_tuple_index = None
1080+
fillarg = None
1081+
for k, arg in enumerate(args):
1082+
if not (isinstance(arg, type) and not isinstance(arg, GenericAlias)):
1083+
subargs = getattr(arg, '__typing_unpacked_tuple_args__', None)
1084+
if subargs and len(subargs) == 2 and subargs[-1] is ...:
1085+
if var_tuple_index is not None:
1086+
raise TypeError("More than one unpacked arbitrary-length tuple argument")
1087+
var_tuple_index = k
1088+
fillarg = subargs[0]
1089+
if var_tuple_index is not None:
1090+
left = min(left, var_tuple_index)
1091+
right = min(right, alen - var_tuple_index - 1)
1092+
elif left + right > alen:
1093+
raise TypeError(f"Too few arguments for {alias};"
1094+
f" actual {alen}, expected at least {plen-1}")
1095+
1096+
return (
1097+
*args[:left],
1098+
*([fillarg]*(typevartuple_index - left)),
1099+
tuple(args[left: alen - right]),
1100+
*([fillarg]*(plen - right - left - typevartuple_index - 1)),
1101+
*args[alen - right:],
1102+
)
1103+
10681104

10691105
class ParamSpecArgs(_Final, _Immutable, _root=True):
10701106
"""The args for a ParamSpec object.
@@ -1184,6 +1220,8 @@ def __typing_subst__(self, arg):
11841220
f"ParamSpec, or Concatenate. Got {arg}")
11851221
return arg
11861222

1223+
def __typing_prepare_subst__(self, alias, args):
1224+
return _prepare_paramspec_params(alias, args)
11871225

11881226
def _is_dunder(attr):
11891227
return attr.startswith('__') and attr.endswith('__')
@@ -1255,44 +1293,6 @@ def __dir__(self):
12551293
+ [attr for attr in dir(self.__origin__) if not _is_dunder(attr)]))
12561294

12571295

1258-
def _is_unpacked_tuple(x: Any) -> bool:
1259-
# Is `x` something like `*tuple[int]` or `*tuple[int, ...]`?
1260-
if not isinstance(x, _UnpackGenericAlias):
1261-
return False
1262-
# Alright, `x` is `Unpack[something]`.
1263-
1264-
# `x` will always have `__args__`, because Unpack[] and Unpack[()]
1265-
# aren't legal.
1266-
unpacked_type = x.__args__[0]
1267-
1268-
return getattr(unpacked_type, '__origin__', None) is tuple
1269-
1270-
1271-
def _is_unpacked_arbitrary_length_tuple(x: Any) -> bool:
1272-
if not _is_unpacked_tuple(x):
1273-
return False
1274-
unpacked_tuple = x.__args__[0]
1275-
1276-
if not hasattr(unpacked_tuple, '__args__'):
1277-
# It's `Unpack[tuple]`. We can't make any assumptions about the length
1278-
# of the tuple, so it's effectively an arbitrary-length tuple.
1279-
return True
1280-
1281-
tuple_args = unpacked_tuple.__args__
1282-
if not tuple_args:
1283-
# It's `Unpack[tuple[()]]`.
1284-
return False
1285-
1286-
last_arg = tuple_args[-1]
1287-
if last_arg is Ellipsis:
1288-
# It's `Unpack[tuple[something, ...]]`, which is arbitrary-length.
1289-
return True
1290-
1291-
# If the arguments didn't end with an ellipsis, then it's not an
1292-
# arbitrary-length tuple.
1293-
return False
1294-
1295-
12961296
# Special typing constructs Union, Optional, Generic, Callable and Tuple
12971297
# use three special attributes for internal bookkeeping of generic types:
12981298
# * __parameters__ is a tuple of unique free type parameters of a generic
@@ -1385,10 +1385,6 @@ def __getitem__(self, args):
13851385
args = (args,)
13861386
args = tuple(_type_convert(p) for p in args)
13871387
args = _unpack_args(args)
1388-
if (self._paramspec_tvars
1389-
and any(isinstance(t, ParamSpec) for t in self.__parameters__)):
1390-
args = _prepare_paramspec_params(self, args)
1391-
13921388
new_args = self._determine_new_args(args)
13931389
r = self.copy_with(new_args)
13941390
return r
@@ -1410,30 +1406,16 @@ def _determine_new_args(self, args):
14101406

14111407
params = self.__parameters__
14121408
# In the example above, this would be {T3: str}
1413-
new_arg_by_param = {}
1414-
typevartuple_index = None
1415-
for i, param in enumerate(params):
1416-
if isinstance(param, TypeVarTuple):
1417-
if typevartuple_index is not None:
1418-
raise TypeError(f"More than one TypeVarTuple parameter in {self}")
1419-
typevartuple_index = i
1420-
1409+
for param in params:
1410+
prepare = getattr(param, '__typing_prepare_subst__', None)
1411+
if prepare is not None:
1412+
args = prepare(self, args)
14211413
alen = len(args)
14221414
plen = len(params)
1423-
if typevartuple_index is not None:
1424-
i = typevartuple_index
1425-
j = alen - (plen - i - 1)
1426-
if j < i:
1427-
raise TypeError(f"Too few arguments for {self};"
1428-
f" actual {alen}, expected at least {plen-1}")
1429-
new_arg_by_param.update(zip(params[:i], args[:i]))
1430-
new_arg_by_param[params[i]] = tuple(args[i: j])
1431-
new_arg_by_param.update(zip(params[i + 1:], args[j:]))
1432-
else:
1433-
if alen != plen:
1434-
raise TypeError(f"Too {'many' if alen > plen else 'few'} arguments for {self};"
1435-
f" actual {alen}, expected {plen}")
1436-
new_arg_by_param.update(zip(params, args))
1415+
if alen != plen:
1416+
raise TypeError(f"Too {'many' if alen > plen else 'few'} arguments for {self};"
1417+
f" actual {alen}, expected {plen}")
1418+
new_arg_by_param = dict(zip(params, args))
14371419

14381420
new_args = []
14391421
for old_arg in self.__args__:

‎Misc/NEWS.d/next/Library/2022-06-01-11-24-13.gh-issue-91162.NxvU_u.rst

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Support splitting of unpacked arbitrary-length tuple over ``TypeVar`` and
2+
``TypeVarTuple`` parameters. For example:
3+
4+
* ``A[T, *Ts][*tuple[int, ...]]`` -> ``A[int, *tuple[int, ...]]``
5+
* ``A[*Ts, T][*tuple[int, ...]]`` -> ``A[*tuple[int, ...], int]``

‎Objects/genericaliasobject.c

+43-56
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ _Py_make_parameters(PyObject *args)
269269
a non-empty tuple, return a new reference to obj. */
270270
static PyObject *
271271
subs_tvars(PyObject *obj, PyObject *params,
272-
PyObject **argitems, Py_ssize_t nargs, Py_ssize_t varparam)
272+
PyObject **argitems, Py_ssize_t nargs)
273273
{
274274
PyObject *subparams;
275275
if (_PyObject_LookupAttr(obj, &_Py_ID(__parameters__), &subparams) < 0) {
@@ -283,28 +283,28 @@ subs_tvars(PyObject *obj, PyObject *params,
283283
Py_DECREF(subparams);
284284
return NULL;
285285
}
286-
for (Py_ssize_t i = 0, j = 0; i < nsubargs; ++i) {
286+
Py_ssize_t j = 0;
287+
for (Py_ssize_t i = 0; i < nsubargs; ++i) {
287288
PyObject *arg = PyTuple_GET_ITEM(subparams, i);
288289
Py_ssize_t iparam = tuple_index(params, nparams, arg);
289-
if (iparam == varparam) {
290-
j = tuple_extend(&subargs, j,
291-
argitems + iparam, nargs - nparams + 1);
292-
if (j < 0) {
293-
return NULL;
294-
}
295-
}
296-
else {
297-
if (iparam >= 0) {
298-
if (iparam > varparam) {
299-
iparam += nargs - nparams;
290+
if (iparam >= 0) {
291+
PyObject *param = PyTuple_GET_ITEM(params, iparam);
292+
arg = argitems[iparam];
293+
if (Py_TYPE(param)->tp_iter && PyTuple_Check(arg)) { // TypeVarTuple
294+
j = tuple_extend(&subargs, j,
295+
&PyTuple_GET_ITEM(arg, 0),
296+
PyTuple_GET_SIZE(arg));
297+
if (j < 0) {
298+
return NULL;
300299
}
301-
arg = argitems[iparam];
300+
continue;
302301
}
303-
Py_INCREF(arg);
304-
PyTuple_SET_ITEM(subargs, j, arg);
305-
j++;
306302
}
303+
Py_INCREF(arg);
304+
PyTuple_SET_ITEM(subargs, j, arg);
305+
j++;
307306
}
307+
assert(j == PyTuple_GET_SIZE(subargs));
308308

309309
obj = PyObject_GetItem(obj, subargs);
310310

@@ -409,39 +409,37 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
409409
self);
410410
}
411411
item = _unpack_args(item);
412-
int is_tuple = PyTuple_Check(item);
413-
Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1;
414-
PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item;
415-
Py_ssize_t varparam = nparams;
416412
for (Py_ssize_t i = 0; i < nparams; i++) {
417413
PyObject *param = PyTuple_GET_ITEM(parameters, i);
418-
if (Py_TYPE(param)->tp_iter) { // TypeVarTuple
419-
if (varparam < nparams) {
420-
Py_DECREF(item);
421-
return PyErr_Format(PyExc_TypeError,
422-
"More than one TypeVarTuple parameter in %S",
423-
self);
424-
}
425-
varparam = i;
426-
}
427-
}
428-
if (varparam < nparams) {
429-
if (nitems < nparams - 1) {
414+
PyObject *prepare, *tmp;
415+
if (_PyObject_LookupAttr(param, &_Py_ID(__typing_prepare_subst__), &prepare) < 0) {
430416
Py_DECREF(item);
431-
return PyErr_Format(PyExc_TypeError,
432-
"Too few arguments for %R",
433-
self);
417+
return NULL;
434418
}
435-
}
436-
else {
437-
if (nitems != nparams) {
438-
Py_DECREF(item);
439-
return PyErr_Format(PyExc_TypeError,
440-
"Too %s arguments for %R; actual %zd, expected %zd",
441-
nitems > nparams ? "many" : "few",
442-
self, nitems, nparams);
419+
if (prepare && prepare != Py_None) {
420+
if (PyTuple_Check(item)) {
421+
tmp = PyObject_CallFunction(prepare, "OO", self, item);
422+
}
423+
else {
424+
tmp = PyObject_CallFunction(prepare, "O(O)", self, item);
425+
}
426+
Py_DECREF(prepare);
427+
Py_SETREF(item, tmp);
428+
if (item == NULL) {
429+
return NULL;
430+
}
443431
}
444432
}
433+
int is_tuple = PyTuple_Check(item);
434+
Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1;
435+
PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item;
436+
if (nitems != nparams) {
437+
Py_DECREF(item);
438+
return PyErr_Format(PyExc_TypeError,
439+
"Too %s arguments for %R; actual %zd, expected %zd",
440+
nitems > nparams ? "many" : "few",
441+
self, nitems, nparams);
442+
}
445443
/* Replace all type variables (specified by parameters)
446444
with corresponding values specified by argitems.
447445
t = list[T]; t[int] -> newargs = [int]
@@ -471,22 +469,11 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
471469
if (subst) {
472470
Py_ssize_t iparam = tuple_index(parameters, nparams, arg);
473471
assert(iparam >= 0);
474-
if (iparam == varparam) {
475-
Py_DECREF(subst);
476-
Py_DECREF(newargs);
477-
Py_DECREF(item);
478-
PyErr_SetString(PyExc_TypeError,
479-
"Substitution of bare TypeVarTuple is not supported");
480-
return NULL;
481-
}
482-
if (iparam > varparam) {
483-
iparam += nitems - nparams;
484-
}
485472
arg = PyObject_CallOneArg(subst, argitems[iparam]);
486473
Py_DECREF(subst);
487474
}
488475
else {
489-
arg = subs_tvars(arg, parameters, argitems, nitems, varparam);
476+
arg = subs_tvars(arg, parameters, argitems, nitems);
490477
}
491478
if (arg == NULL) {
492479
Py_DECREF(newargs);

0 commit comments

Comments
 (0)
0