From 2a9f75b2c02cc5f077c867533f28f5bee222c423 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 8 Apr 2022 08:55:49 -0700 Subject: [PATCH 1/5] Add Required and NotRequired --- Lib/test/test_typing.py | 113 +++++++++++++++++- Lib/typing.py | 63 +++++++++- .../2022-04-08-08-55-36.bpo-47087.Q5C3EI.rst | 2 + 3 files changed, 173 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-04-08-08-55-36.bpo-47087.Q5C3EI.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index e09f8aa3fb8496..c0744fd91d892d 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -23,7 +23,7 @@ from typing import reveal_type from typing import no_type_check, no_type_check_decorator from typing import Type -from typing import NamedTuple, TypedDict +from typing import NamedTuple, NotRequired, Required, TypedDict from typing import IO, TextIO, BinaryIO from typing import Pattern, Match from typing import Annotated, ForwardRef @@ -3967,6 +3967,18 @@ class Options(TypedDict, total=False): log_level: int log_path: str +class TotalMovie(TypedDict): + title: str + year: NotRequired[int] + +class NontotalMovie(TypedDict, total=False): + title: Required[str] + year: int + +class AnnotatedMovie(TypedDict): + title: Annotated[Required[str], "foobar"] + year: NotRequired[Annotated[int, 2000]] + class HasForeignBaseClass(mod_generics_cache.A): some_xrepr: 'XRepr' other_a: 'mod_generics_cache.A' @@ -4254,6 +4266,18 @@ def test_top_level_class_var(self): ): get_type_hints(ann_module6) + def test_get_type_hints_typeddict(self): + assert get_type_hints(TotalMovie) == {'title': str, 'year': int} + assert get_type_hints(TotalMovie, include_extras=True) == { + 'title': str, + 'year': NotRequired[int], + } + + assert get_type_hints(AnnotatedMovie) == {'title': str, 'year': int} + assert get_type_hints(AnnotatedMovie, include_extras=True) == { + 'title': Annotated[Required[str], "foobar"], + 'year': NotRequired[Annotated[int, 2000]], + } class GetUtilitiesTestCase(TestCase): def test_get_origin(self): @@ -5273,6 +5297,13 @@ class Cat(Animal): 'voice': str, } + def test_required_notrequired_keys(self): + assert NontotalMovie.__required_keys__ == frozenset({'title'}) + assert NontotalMovie.__optional_keys__ == frozenset({'year'}) + + assert TotalMovie.__required_keys__ == frozenset({'title'}) + assert TotalMovie.__optional_keys__ == frozenset({'year'}) + def test_multiple_inheritance(self): class One(TypedDict): one: int @@ -5373,6 +5404,86 @@ def test_get_type_hints(self): ) +class RequiredTests(BaseTestCase): + + def test_basics(self): + with self.assertRaises(TypeError): + Required[NotRequired] + with self.assertRaises(TypeError): + Required[int, str] + with self.assertRaises(TypeError): + Required[int][str] + + def test_repr(self): + self.assertEqual(repr(Required), 'typing.Required') + cv = Required[int] + self.assertEqual(repr(cv), 'typing.Required[int]') + cv = Required[Employee] + self.assertEqual(repr(cv), f'typing.Required[{__name__}.Employee]') + + def test_cannot_subclass(self): + with self.assertRaises(TypeError): + class C(type(Required)): + pass + with self.assertRaises(TypeError): + class C(type(Required[int])): + pass + + def test_cannot_init(self): + with self.assertRaises(TypeError): + Required() + with self.assertRaises(TypeError): + type(Required)() + with self.assertRaises(TypeError): + type(Required[Optional[int]])() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, Required[int]) + with self.assertRaises(TypeError): + issubclass(int, Required) + + +class NotRequiredTests(BaseTestCase): + + def test_basics(self): + with self.assertRaises(TypeError): + NotRequired[Required] + with self.assertRaises(TypeError): + NotRequired[int, str] + with self.assertRaises(TypeError): + NotRequired[int][str] + + def test_repr(self): + self.assertEqual(repr(NotRequired), 'typing.NotRequired') + cv = NotRequired[int] + self.assertEqual(repr(cv), 'typing.NotRequired[int]') + cv = NotRequired[Employee] + self.assertEqual(repr(cv), f'typing.NotRequired[{__name__}.Employee]') + + def test_cannot_subclass(self): + with self.assertRaises(TypeError): + class C(type(NotRequired)): + pass + with self.assertRaises(TypeError): + class C(type(NotRequired[int])): + pass + + def test_cannot_init(self): + with self.assertRaises(TypeError): + NotRequired() + with self.assertRaises(TypeError): + type(NotRequired)() + with self.assertRaises(TypeError): + type(NotRequired[Optional[int]])() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, NotRequired[int]) + with self.assertRaises(TypeError): + issubclass(int, NotRequired) + + class IOTests(BaseTestCase): def test_io(self): diff --git a/Lib/typing.py b/Lib/typing.py index 26c6b8c278b734..8d36e2b2577be0 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -132,9 +132,11 @@ def _idfunc(_, x): 'no_type_check', 'no_type_check_decorator', 'NoReturn', + 'NotRequired', 'overload', 'ParamSpecArgs', 'ParamSpecKwargs', + 'Required', 'reveal_type', 'runtime_checkable', 'Self', @@ -2261,6 +2263,8 @@ def _strip_annotations(t): """ if isinstance(t, _AnnotatedAlias): return _strip_annotations(t.__origin__) + if hasattr(t, "__origin__") and t.__origin__ in (Required, NotRequired): + return _strip_annotations(t.__args__[0]) if isinstance(t, _GenericAlias): stripped_args = tuple(_strip_annotations(a) for a in t.__args__) if stripped_args == t.__args__: @@ -2785,10 +2789,22 @@ def __new__(cls, name, bases, ns, total=True): optional_keys.update(base.__dict__.get('__optional_keys__', ())) annotations.update(own_annotations) - if total: - required_keys.update(own_annotation_keys) - else: - optional_keys.update(own_annotation_keys) + for annotation_key, annotation_type in own_annotations.items(): + annotation_origin = get_origin(annotation_type) + if annotation_origin is Annotated: + annotation_args = get_args(annotation_type) + if annotation_args: + annotation_type = annotation_args[0] + annotation_origin = get_origin(annotation_type) + + if annotation_origin is Required: + required_keys.add(annotation_key) + elif annotation_origin is NotRequired: + optional_keys.add(annotation_key) + elif total: + required_keys.add(annotation_key) + else: + optional_keys.add(annotation_key) tp_dict.__annotations__ = annotations tp_dict.__required_keys__ = frozenset(required_keys) @@ -2873,6 +2889,45 @@ class body be required. TypedDict.__mro_entries__ = lambda bases: (_TypedDict,) +@_SpecialForm +def Required(self, parameters): + """A special typing construct to mark a key of a total=False TypedDict + as required. For example: + + class Movie(TypedDict, total=False): + title: Required[str] + year: int + + m = Movie( + title='The Matrix', # typechecker error if key is omitted + year=1999, + ) + + There is no runtime checking that a required key is actually provided + when instantiating a related TypedDict. + """ + item = _type_check(parameters, f'{self._name} accepts only single type') + return _GenericAlias(self, (item,)) + + +@_SpecialForm +def NotRequired(self, parameters): + """A special typing construct to mark a key of a TypedDict as + potentially missing. For example: + + class Movie(TypedDict): + title: str + year: NotRequired[int] + + m = Movie( + title='The Matrix', # typechecker error if key is omitted + year=1999, + ) + """ + item =_type_check(parameters, f'{self._name} accepts only single type') + return _GenericAlias(self, (item,)) + + class NewType: """NewType creates simple unique types with almost zero runtime overhead. NewType(name, tp) is considered a subtype of tp diff --git a/Misc/NEWS.d/next/Library/2022-04-08-08-55-36.bpo-47087.Q5C3EI.rst b/Misc/NEWS.d/next/Library/2022-04-08-08-55-36.bpo-47087.Q5C3EI.rst new file mode 100644 index 00000000000000..33996632140450 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-04-08-08-55-36.bpo-47087.Q5C3EI.rst @@ -0,0 +1,2 @@ +Implement ``typing.Required`` and ``typing.NotRequired`` (:pep:`655`). Patch +by Jelle Zijlstra. From ea8512009765d1e3e5af2ce6681e8449ac669444 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 8 Apr 2022 08:59:23 -0700 Subject: [PATCH 2/5] test get_origin/get_args --- Lib/test/test_typing.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index c0744fd91d892d..08d47ccca0f851 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4303,6 +4303,8 @@ class C(Generic[T]): pass self.assertIs(get_origin(list | str), types.UnionType) self.assertIs(get_origin(P.args), P) self.assertIs(get_origin(P.kwargs), P) + self.assertIs(get_origin(Required[int]), Required) + self.assertIs(get_origin(NotRequired[int]), NotRequired) def test_get_args(self): T = TypeVar('T') @@ -4340,6 +4342,8 @@ class C(Generic[T]): pass self.assertEqual(get_args(Callable[Concatenate[int, P], int]), (Concatenate[int, P], int)) self.assertEqual(get_args(list | str), (list, str)) + self.assertEqual(get_args(Required[int]), (int,)) + self.assertEqual(get_args(NotRequired[int]), (int,)) class CollectionsAbcTests(BaseTestCase): From 3cafefa03964fae96eb99f8c281e0beb0f0b22bb Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 10 Apr 2022 15:56:58 -0700 Subject: [PATCH 3/5] Apply feedback from David Foster --- Lib/test/test_typing.py | 21 +++++++++++++++++++++ Lib/typing.py | 4 ++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 08d47ccca0f851..25b4d779187fad 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3979,6 +3979,10 @@ class AnnotatedMovie(TypedDict): title: Annotated[Required[str], "foobar"] year: NotRequired[Annotated[int, 2000]] +class DeeplyAnnotatedMovie(TypedDict): + title: Annotated[Annotated[Required[str], "foobar"], "another level"] + year: NotRequired[Annotated[int, 2000]] + class HasForeignBaseClass(mod_generics_cache.A): some_xrepr: 'XRepr' other_a: 'mod_generics_cache.A' @@ -4278,6 +4282,11 @@ def test_get_type_hints_typeddict(self): 'title': Annotated[Required[str], "foobar"], 'year': NotRequired[Annotated[int, 2000]], } + assert get_type_hints(DeeplyAnnotatedMovie) == {'title': str, 'year': int} + assert get_type_hints(DeeplyAnnotatedMovie, include_extras=True) == { + 'title': Annotated[Annotated[Required[str], "foobar"], "another level"], + 'year': NotRequired[Annotated[int, 2000]], + } class GetUtilitiesTestCase(TestCase): def test_get_origin(self): @@ -5432,6 +5441,12 @@ class C(type(Required)): with self.assertRaises(TypeError): class C(type(Required[int])): pass + with self.assertRaises(TypeError): + class C(Required): + pass + with self.assertRaises(TypeError): + class C(Required[int]): + pass def test_cannot_init(self): with self.assertRaises(TypeError): @@ -5472,6 +5487,12 @@ class C(type(NotRequired)): with self.assertRaises(TypeError): class C(type(NotRequired[int])): pass + with self.assertRaises(TypeError): + class C(NotRequired): + pass + with self.assertRaises(TypeError): + class C(NotRequired[int]): + pass def test_cannot_init(self): with self.assertRaises(TypeError): diff --git a/Lib/typing.py b/Lib/typing.py index 8d36e2b2577be0..48a22e43c0f38b 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2906,7 +2906,7 @@ class Movie(TypedDict, total=False): There is no runtime checking that a required key is actually provided when instantiating a related TypedDict. """ - item = _type_check(parameters, f'{self._name} accepts only single type') + item = _type_check(parameters, f'{self._name} accepts only a single type') return _GenericAlias(self, (item,)) @@ -2924,7 +2924,7 @@ class Movie(TypedDict): year=1999, ) """ - item =_type_check(parameters, f'{self._name} accepts only single type') + item = _type_check(parameters, f'{self._name} accepts only a single type') return _GenericAlias(self, (item,)) From 86253cabe587f6c34714025493db6e5658f48574 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 10 Apr 2022 16:13:10 -0700 Subject: [PATCH 4/5] more tests --- Lib/test/_typed_dict_helper.py | 8 ++++- Lib/test/test_typing.py | 66 ++++++++++++++++++++++++++-------- 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/Lib/test/_typed_dict_helper.py b/Lib/test/_typed_dict_helper.py index d333db193183eb..3328330c995b30 100644 --- a/Lib/test/_typed_dict_helper.py +++ b/Lib/test/_typed_dict_helper.py @@ -6,13 +6,19 @@ class Bar(_typed_dict_helper.Foo, total=False): b: int + +In addition, it uses multiple levels of Annotated to test the interaction +between the __future__ import, Annotated, and Required. """ from __future__ import annotations -from typing import Optional, TypedDict +from typing import Annotated, Optional, Required, TypedDict OptionalIntType = Optional[int] class Foo(TypedDict): a: OptionalIntType + +class VeryAnnotated(TypedDict, total=False): + a: Annotated[Annotated[Annotated[Required[int], "a"], "b"], "c"] diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 25b4d779187fad..d102db8a387695 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3983,6 +3983,10 @@ class DeeplyAnnotatedMovie(TypedDict): title: Annotated[Annotated[Required[str], "foobar"], "another level"] year: NotRequired[Annotated[int, 2000]] +class WeirdlyQuotedMovie(TypedDict): + title: Annotated['Annotated[Required[str], "foobar"]', "another level"] + year: NotRequired['Annotated[int, 2000]'] + class HasForeignBaseClass(mod_generics_cache.A): some_xrepr: 'XRepr' other_a: 'mod_generics_cache.A' @@ -4271,22 +4275,35 @@ def test_top_level_class_var(self): get_type_hints(ann_module6) def test_get_type_hints_typeddict(self): - assert get_type_hints(TotalMovie) == {'title': str, 'year': int} - assert get_type_hints(TotalMovie, include_extras=True) == { + self.assertEqual(get_type_hints(TotalMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(TotalMovie, include_extras=True), { 'title': str, 'year': NotRequired[int], - } + }) - assert get_type_hints(AnnotatedMovie) == {'title': str, 'year': int} - assert get_type_hints(AnnotatedMovie, include_extras=True) == { + self.assertEqual(get_type_hints(AnnotatedMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(AnnotatedMovie, include_extras=True), { 'title': Annotated[Required[str], "foobar"], 'year': NotRequired[Annotated[int, 2000]], - } - assert get_type_hints(DeeplyAnnotatedMovie) == {'title': str, 'year': int} - assert get_type_hints(DeeplyAnnotatedMovie, include_extras=True) == { - 'title': Annotated[Annotated[Required[str], "foobar"], "another level"], + }) + + self.assertEqual(get_type_hints(DeeplyAnnotatedMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(DeeplyAnnotatedMovie, include_extras=True), { + 'title': Annotated[Required[str], "foobar", "another level"], 'year': NotRequired[Annotated[int, 2000]], - } + }) + + self.assertEqual(get_type_hints(WeirdlyQuotedMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(WeirdlyQuotedMovie, include_extras=True), { + 'title': Annotated[Required[str], "foobar", "another level"], + 'year': NotRequired[Annotated[int, 2000]], + }) + + self.assertEqual(get_type_hints(_typed_dict_helper.VeryAnnotated), {'a': int}) + self.assertEqual(get_type_hints(_typed_dict_helper.VeryAnnotated, include_extras=True), { + 'a': Annotated[Required[int], "a", "b", "c"] + }) + class GetUtilitiesTestCase(TestCase): def test_get_origin(self): @@ -5311,11 +5328,30 @@ class Cat(Animal): } def test_required_notrequired_keys(self): - assert NontotalMovie.__required_keys__ == frozenset({'title'}) - assert NontotalMovie.__optional_keys__ == frozenset({'year'}) - - assert TotalMovie.__required_keys__ == frozenset({'title'}) - assert TotalMovie.__optional_keys__ == frozenset({'year'}) + self.assertEqual(NontotalMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(NontotalMovie.__optional_keys__, + frozenset({"year"})) + + self.assertEqual(TotalMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(TotalMovie.__optional_keys__, + frozenset({"year"})) + + self.assertEqual(_typed_dict_helper.VeryAnnotated.__required_keys__, + frozenset()) + self.assertEqual(_typed_dict_helper.VeryAnnotated.__optional_keys__, + frozenset({"a"})) + + self.assertEqual(AnnotatedMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(AnnotatedMovie.__optional_keys__, + frozenset({"year"})) + + self.assertEqual(WeirdlyQuotedMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(WeirdlyQuotedMovie.__optional_keys__, + frozenset({"year"})) def test_multiple_inheritance(self): class One(TypedDict): From a963b13b8998adc33096bcf780f6add9150df072 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 10 Apr 2022 16:14:08 -0700 Subject: [PATCH 5/5] period in error --- Lib/typing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 48a22e43c0f38b..fade84619ff2cf 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2906,7 +2906,7 @@ class Movie(TypedDict, total=False): There is no runtime checking that a required key is actually provided when instantiating a related TypedDict. """ - item = _type_check(parameters, f'{self._name} accepts only a single type') + item = _type_check(parameters, f'{self._name} accepts only a single type.') return _GenericAlias(self, (item,)) @@ -2924,7 +2924,7 @@ class Movie(TypedDict): year=1999, ) """ - item = _type_check(parameters, f'{self._name} accepts only a single type') + item = _type_check(parameters, f'{self._name} accepts only a single type.') return _GenericAlias(self, (item,))