8000 bpo-42195: Ensure consistency of Callable's __args__ in collections.a… · python/cpython@463c7d3 · GitHub
[go: up one dir, main page]
Skip to content

Commit 463c7d3

Browse files
Fidget-SpinnerFidget-Spinner
authored
bpo-42195: Ensure consistency of Callable's __args__ in collections.abc and typing (GH-23060)
1 parent 43c4fb6 commit 463c7d3

File tree

9 files changed

+212
-63
lines changed

9 files changed

+212
-63
lines changed

‎Lib/_collections_abc.py

Lines changed: 68 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@
1010
import sys
1111

1212
GenericAlias = type(list[int])
13+
EllipsisType = type(...)
14+
def _f(): pass
15+
FunctionType = type(_f)
16+
del _f
1317

1418
__all__ = ["Awaitable", "Coroutine",
1519
"AsyncIterable", "AsyncIterator", "AsyncGenerator",
@@ -409,6 +413,69 @@ def __subclasshook__(cls, C):
409413
return NotImplemented
410414

411415

416+
class _CallableGenericAlias(GenericAlias):
417+
""" Represent `Callable[argtypes, resulttype]`.
418+
419+
This sets ``__args__`` to a tuple containing the flattened``argtypes``
420+
followed by ``resulttype``.
421+
422+
Example: ``Callable[[int, str], float]`` sets ``__args__`` to
423+
``(int, str, float)``.
424+
"""
425+
426+
__slots__ = ()
427+
428+
def __new__(cls, origin, args):
429+
return cls.__create_ga(origin, args)
430+
431+
@classmethod
432+
def __create_ga(cls, origin, args):
433+
if not isinstance(args, tuple) or len(args) != 2:
434+
raise TypeError(
435+
"Callable must be used as Callable[[arg, ...], result].")
436+
t_args, t_result = args
437+
if isinstance(t_args, list):
438+
ga_args = tuple(t_args) + (t_result,)
439+
# This relaxes what t_args can be on purpose to allow things like
440+
# PEP 612 ParamSpec. Responsibility for whether a user is using
441+
# Callable[...] properly is deferred to static type checkers.
442+
else:
443+
ga_args = args
444+
return super().__new__(cls, origin, ga_args)
445+
446+
def __repr__(self):
447+
if len(self.__args__) == 2 and self.__args__[0] is Ellipsis:
448+
return super().__repr__()
449+
return (f'collections.abc.Callable'
450+
f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], '
451+
f'{_type_repr(self.__args__[-1])}]')
452+
453+
def __reduce__(self):
454+
args = self.__args__
455+
if not (len(args) == 2 and args[0] is Ellipsis):
456+
args = list(args[:-1]), args[-1]
457+
return _CallableGenericAlias, (Callable, args)
458+
459+
460+
def _type_repr(obj):
461+
"""Return the repr() of an object, special-casing types (internal helper).
462+
463+
Copied from :mod:`typing` since collections.abc
464+
shouldn't depend on that module.
465+
"""
466+
if isinstance(obj, GenericAlias):
467+
return repr(obj)
468+
if isinstance(obj, type):
469+
if obj.__module__ == 'builtins':
470+
return obj.__qualname__
471+
return f'{obj.__module__}.{obj.__qualname__}'
472+
if obj is Ellipsis:
473+
return '...'
474+
if isinstance(obj, FunctionType):
475+
return obj.__name__
476+
return repr(obj)
477+
478+
412479
class Callable(metaclass=ABCMeta):
413480

414481
__slots__ = ()
@@ -423,7 +490,7 @@ def __subclasshook__(cls, C):
423490
return _check_methods(C, "__call__")
424491
return NotImplemented
425492

426-
__class_getitem__ = classmethod(GenericAlias)
493+
__class_getitem__ = classmethod(_CallableGenericAlias)
427494

428495

429496
### SETS ###

‎Lib/collections/abc.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
from _collections_abc import *
22
from _collections_abc import __all__
3+
from _collections_abc import _CallableGenericAlias

‎Lib/test/test_genericalias.py

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,6 @@ class BaseTest(unittest.TestCase):
6262
Iterable, Iterator,
6363
Reversible,
6464
Container, Collection,
65-
Callable,
6665
Mailbox, _PartialFile,
6766
ContextVar, Token,
6867
Field,
@@ -307,6 +306,63 @@ def test_no_kwargs(self):
307306
with self.assertRaises(TypeError):
308307
GenericAlias(bad=float)
309308

309+
def test_subclassing_types_genericalias(self):
310+
class SubClass(GenericAlias): ...
311+
alias = SubClass(list, int)
312+
class Bad(GenericAlias):
313+
def __new__(cls, *args, **kwargs):
314+
super().__new__(cls, *args, **kwargs)
315+
316+
self.assertEqual(alias, list[int])
317+
with self.assertRaises(TypeError):
318+
Bad(list, int, bad=int)
319+
320+
def test_abc_callable(self):
321+
# A separate test is needed for Callable since it uses a subclass of
322+
# GenericAlias.
323+
alias = Callable[[int, str], float]
324+
with self.subTest("Testing subscription"):
325+
self.assertIs(alias.__origin__, Callable)
326+
self.assertEqual(alias.__args__, (int, str, float))
327+
self.assertEqual(alias.__parameters__, ())
328+
329+
with self.subTest("Testing instance checks"):
330+
self.assertIsInstance(alias, GenericAlias)
331+
332+
with self.subTest("Testing weakref"):
333+
self.assertEqual(ref(alias)(), alias)
334+
335+
with self.subTest("Testing pickling"):
336+
s = pickle.dumps(alias)
337+
loaded = pickle.loads(s)
338+
self.assertEqual(alias.__origin__, loaded.__origin__)
339+
self.assertEqual(alias.__args__, loaded.__args__)
340+
self.assertEqual(alias.__parameters__, loaded.__parameters__)
341+
342+
with self.subTest("Testing TypeVar substitution"):
343+
C1 = Callable[[int, T], T]
344+
C2 = Callable[[K, T], V]
345+
C3 = Callable[..., T]
346+
self.assertEqual(C1[str], Callable[[int, str], str])
347+
self.assertEqual(C2[int, float, str], Callable[[int, float], str])
348+
self.assertEqual(C3[int], Callable[..., int])
349+
350+
with self.subTest("Testing type erasure"):
351+
class C1(Callable):
352+
def __call__(self):
353+
return None
354+
a = C1[[int], T]
355+
self.assertIs(a().__class__, C1)
356+
self.assertEqual(a().__orig_class__, C1[[int], T])
357+
358+
# bpo-42195
359+
with self.subTest("Testing collections.abc.Callable's consistency "
360+
"with typing.Callable"):
361+
c1 = typing.Callable[[int, str], dict]
362+
c2 = Callable[[int, str], dict]
363+
self.assertEqual(c1.__args__, c2.__args__)
364+
self.assertEqual(hash(c1.__args__), hash(c2.__args__))
365+
310366

311367
if __name__ == "__main__":
312368
unittest.main()

‎Lib/test/test_types.py

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -717,14 +717,16 @@ def test_or_type_operator_with_genericalias(self):
717717
a = list[int]
718718
b = list[str]
719719
c = dict[float, str]
720+
class SubClass(types.GenericAlias): ...
721+
d = SubClass(list, float)
720722
# equivalence with typing.Union
721-
self.assertEqual(a | b | c, typing.Union[a, b, c])
723+
self.assertEqual(a | b | c | d, typing.Union[a, b, c, d])
722724
# de-duplicate
723-
self.assertEqual(a | c | b | b | a | c, a | b | c)
725+
self.assertEqual(a | c | b | b | a | c | d | d, a | b | c | d)
724726
# order shouldn't matter
725-
self.assertEqual(a | b, b | a)
726-
self.assertEqual(repr(a | b | c),
727-
"list[int] | list[str] | dict[float, str]")
727+
self.assertEqual(a | b | d, b | a | d)
728+
self.assertEqual(repr(a | b | c | d),
729+
"list[int] | list[str] | dict[float, str] | list[float]")
728730

729731
class BadType(type):
730732
def __eq__(self, other):

‎Lib/test/test_typing.py

Lines changed: 6 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -446,14 +446,6 @@ def test_cannot_instantiate(self):
446446
type(c)()
447447

448448
def test_callable_wrong_forms(self):
449-
with self.assertRaises(TypeError):
450-
Callable[[...], int]
451-
with self.assertRaises(TypeError):
452-
Callable[(), int]
453-
with self.assertRaises(TypeError):
454-
Callable[[()], int]
455-
with self.assertRaises(TypeError):
456-
Callable[[int, 1], 2]
457449
with self.assertRaises(TypeError):
458450
Callable[int]
459451

@@ -1807,10 +1799,9 @@ def barfoo2(x: CT): ...
18071799
def test_extended_generic_rules_subclassing(self):
18081800
class T1(Tuple[T, KT]): ...
18091801
class T2(Tuple[T, ...]): ...
1810-
class C1(Callable[[T], T]): ...
1811-
class C2(Callable[..., int]):
1812-
def __call__(self):
1813-
return None
1802+
class C1(typing.Container[T]):
1803+
def __contains__(self, item):
1804+
return False
18141805

18151806
self.assertEqual(T1.__parameters__, (T, KT))
18161807
self.assertEqual(T1[int, str].__args__, (int, str))
@@ -1824,10 +1815,9 @@ def __call__(self):
18241815
## T2[int, str]
18251816

18261817
self.assertEqual(repr(C1[int]).split('.')[-1], 'C1[int]')
1827-
self.assertEqual(C2.__parameters__, ())
1828-
self.assertIsInstance(C2(), collections.abc.Callable)
1829-
self.assertIsSubclass(C2, collections.abc.Callable)
1830-
self.assertIsSubclass(C1, collections.abc.Callable)
1818+
self.assertEqual(C1.__parameters__, (T,))
1819+
self.assertIsInstance(C1(), collections.abc.Container)
1820+
self.assertIsSubclass(C1, collections.abc.Container)
18311821
self.assertIsInstance(T1(), tuple)
18321822
self.assertIsSubclass(T2, tuple)
18331823
with self.assertRaises(TypeError):
@@ -1861,10 +1851,6 @@ def test_type_erasure_special(self):
18611851
class MyTup(Tuple[T, T]): ...
18621852
self.< 10000 span class=pl-c1>assertIs(MyTup[int]().__class__, MyTup)
18631853
self.assertEqual(MyTup[int]().__orig_class__, MyTup[int])
1864-
class MyCall(Callable[..., T]):
1865-
def __call__(self): return None
1866-
self.assertIs(MyCall[T]().__class__, MyCall)
1867-
self.assertEqual(MyCall[T]().__orig_class__, MyCall[T])
18681854
class MyDict(typing.Dict[T, T]): ...
18691855
self.assertIs(MyDict[int]().__class__, MyDict)
18701856
self.assertEqual(MyDict[int]().__orig_class__, MyDict[int])

‎Lib/typing.py

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,16 @@
120120
# namespace, but excluded from __all__ because they might stomp on
121121
# legitimate imports of those modules.
122122

123+
124+
def _type_convert(arg):
125+
"""For converting None to type(None), and strings to ForwardRef."""
126+
if arg is None:
127+
return type(None)
128+
if isinstance(arg, str):
129+
return ForwardRef(arg)
130+
return arg
131+
132+
123133
def _type_check(arg, msg, is_argument=True):
124134
"""Check that the argument is a type, and return it (internal helper).
125135
@@ -136,10 +146,7 @@ def _type_check(arg, msg, is_argument=True):
136146
if is_argument:
137147
invalid_generic_forms = invalid_generic_forms + (ClassVar, Final)
138148

139-
if arg is None:
140-
return type(None)
141-
if isinstance(arg, str):
142-
return ForwardRef(arg)
149+
arg = _type_convert(arg)
143150
if (isinstance(arg, _GenericAlias) and
144151
arg.__origin__ in invalid_generic_forms):
145152
raise TypeError(f"{arg} is not valid as type argument")
@@ -900,13 +907,13 @@ def __getitem__(self, params):
900907
raise TypeError("Callable must be used as "
901908
"Callable[[arg, ...], result].")
902909
args, result = params
903-
if args is Ellipsis:
904-
params = (Ellipsis, result)
905-
else:
906-
if not isinstance(args, list):
907-
raise TypeError(f"Callable[args, result]: args must be a list."
908-
f" Got {args}")
910+
# This relaxes what args can be on purpose to allow things like
911+
# PEP 612 ParamSpec. Responsibility for whether a user is using
912+
# Callable[...] properly is deferred to static type checkers.
913+
if isinstance(args, list):
909914
params = (tuple(args), result)
915+
else:
916+
params = (args, result)
910917
return self.__getitem_inner__(params)
911918

912919
@_tp_cache
@@ -916,8 +923,9 @@ def __getitem_inner__(self, params):
916923
result = _type_check(result, msg)
917924
if args is Ellipsis:
918925
return self.copy_with((_TypingEllipsis, result))
919-
msg = "Callable[[arg, ...], result]: each arg must be a type."
920-
args = tuple(_type_check(arg, msg) for arg in args)
926+
if not isinstance(args, tuple):
927+
args = (args,)
928+
args = tuple(_type_convert(arg) for arg in args)
921929
params = args + (result,)
922930
return self.copy_with(params)
923931

‎Misc/NEWS.d/next/Core and Builtins/2020-11-20-00-57-47.bpo-42195.HeqcpS.rst

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
The ``__args__`` of the parameterized generics for :data:`typing.Callable`
2+
and :class:`collections.abc.Callable` are now consistent. The ``__args__``
3+
for :class:`collections.abc.Callable` are now flattened while
4+
:data:`typing.Callable`'s have not changed. To allow this change,
5+
:class:`types.GenericAlias` can now be subclassed and
6+
``collections.abc.Callable``'s ``__class_getitem__`` will now return a subclass
7+
of ``types.GenericAlias``. Tests for typing were also updated to not subclass
8+
things like ``Callable[..., T]`` as that is not a valid base class. Finally,
9+
both ``Callable``s no longer validate their ``argtypes``, in
10+
``Callable[[argtypes], resulttype]`` to prepare for :pep:`612`. Patch by Ken Jin.
11+

0 commit comments

Comments
 (0)
0