8000 Add eval_type_backport to handle union operator and builtin generic subscripting in older Pythons by alexmojaki · Pull Request #8209 · pydantic/pydantic · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
75986a9
Add eval_type_backport to handle union operator in older Pythons
alexmojaki Oct 29, 2023
ca60caa
Use modified get_type_hints in test_config
alexmojaki Oct 29, 2023
a163ea3
unskip a couple more tests in older pythons
alexmojaki Oct 29, 2023
8033556
Use pipe operator in a bunch of tests
alexmojaki Oct 29, 2023
0495f0f
various misc tidying up: use default localns=None, handle None values…
alexmojaki Nov 4, 2023
780b46a
explain asserts
alexmojaki Nov 4, 2023
4c536b6
type hints
alexmojaki Nov 4, 2023
d7f874b
inline node_to_ref
alexmojaki Nov 4, 2023
09d2e93
is_unsupported_types_for_union_error
alexmojaki Nov 4, 2023
d7b1462
tidying
alexmojaki Nov 4, 2023
c65ae3d
remove more type: ignore comments
alexmojaki Nov 4, 2023
3aa88da
docstrings and tidying
alexmojaki Nov 4, 2023
ca09a03
fix and tighten test_is_union
alexmojaki Nov 4, 2023
4890fc8
Merge branch 'main' of github.com:pydantic/pydantic into eval_type_ba…
alexmojaki Nov 22, 2023
c2d9d22
Use `eval_type_backport` package
alexmojaki Nov 22, 2023
40a41fb
Add test dependency
alexmojaki Nov 22, 2023
7da17e0
Merge branch 'main' of github.com:pydantic/pydantic into eval_type_ba…
alexmojaki Dec 16, 2023
a998d14
upgrade eval_type_backport
alexmojaki Dec 16, 2023
4f465a3
fix pdm.lock
alexmojaki Dec 16, 2023
71ca7cf
upgrade eval_type_backport to handle fussy typing._type_check
alexmojaki Dec 16, 2023
f060876
update is_backport_fixable_error and move down, update eval_type_back…
alexmojaki Dec 16, 2023
2b42d65
raise helpful error if eval_type_backport isn't installed. ensure tes…
alexmojaki Dec 17, 2023
dcbdd28
Restore skip, add another test for combination of backport and Pydant…
alexmojaki Dec 17, 2023
959c755
Test that eval_type_backport is being called in the right places
alexmojaki Dec 17, 2023
71c912e
test calling backport from get_type_hints
alexmojaki Dec 17, 2023
9847058
upgrade eval_type_backport to handle working union operator
alexmojaki Dec 17, 2023
6beab5b
unskip tests that can now pass in 3.8
alexmojaki Dec 17, 2023
b79692b
revert scattered test changes
alexmojaki Dec 17, 2023
6bc0ee4
unskip more tests
alexmojaki Dec 17, 2023
f423659
upgrade eval_type_backport to copy ForwardRef attributes, allowing un…
alexmojaki Dec 17, 2023
d3d5584
Merge branch 'main' of github.com:pydantic/pydantic into eval_type_ba…
alexmojaki Jan 15, 2024
aa21092
revert moving part of pyproject.toml
alexmojaki Jan 15, 2024
8da4294
Refine and test error raised when eval_type_backport isn't installed
alexmojaki Jan 16, 2024
60aa70f
use a type annotation that's unsupported in 3.9, not just 3.8
alexmojaki Jan 16, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Use pipe operator in a bunch of tests
  • Loading branch information
alexmojaki committed Nov 10, 2023
commit 80335569cb18d702947f3fe6c4238ad53a834d6c
2 changes: 1 addition & 1 deletion pydantic/_internal/_typing_extra.py
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ def visit_BinOp(self, node):
return node


def eval_type_backport(value: Any, globalns: dict[str, Any] | None, localns: dict[str, Any] | None):
def eval_type_backport(value: Any, globalns: dict[str, Any] | None = None, localns: dict[str, Any] | None = None):
try:
return typing._eval_type(value, globalns, localns) # type: ignore
except TypeError:
Expand Down
4 changes: 2 additions & 2 deletions tests/test_dataclasses.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from dataclasses import InitVar
from datetime import date, datetime
from pathlib import Path
from typing import Any, Callable, ClassVar, Dict, FrozenSet, Generic, List, Optional, Set, TypeVar, Union
from typing import Any, Callable, ClassVar, Dict, FrozenSet, Generic, List, Optional, Set, TypeVar

import pytest
from dirty_equals import HasRepr
Expand Down Expand Up @@ -1359,7 +1359,7 @@ class B:

@pydantic.dataclasses.dataclass
class Top:
sub: Union[A, B] = dataclasses.field(metadata=dict(discriminator='l'))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we want to change most tests, just add some new tests specifically for the new behaviour.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've reverted the random test changes, updated existing tests more strategically where it seemed natural (i.e. they already used from __future__ import annotations or they were skipped for Python <= 3.9), and added new tests in various files (all called test_eval_type_backport) to test the different places where eval_type_backport is called.

sub: 'A | B' = dataclasses.field(metadata=dict(discriminator='l'))

t = Top(sub=A(l='a'))
assert isinstance(t, Top)
Expand Down
32 changes: 16 additions & 16 deletions tests/test_discriminated_union.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ def test_discriminated_union_invalid_type():
):

class Model(BaseModel):
x: Union[str, int] = Field(..., discriminator='qwe')
x: 'str | int' = Field(..., discriminator='qwe')


def test_discriminated_union_defined_discriminator():
Expand All @@ -80,7 +80,7 @@ class Dog(BaseModel):
with pytest.raises(PydanticUserError, match="Model 'Cat' needs a discriminator field for key 'pet_type'"):

class Model(BaseModel):
pet: Union[Cat, Dog] = Field(..., discriminator='pet_type')
pet: 'Cat | Dog' = Field(..., discriminator='pet_type')
number: int


Expand All @@ -96,7 +96,7 @@ class Dog(BaseModel):
with pytest.raises(PydanticUserError, match="Model 'Cat' needs field 'pet_type' to be of type `Literal`"):

class Model(BaseModel):
pet: Union[Cat, Dog] = Field(..., discriminator='pet_type')
pet: 'Cat | Dog' = Field(..., discriminator='pet_type')
number: int


Expand Down Expand Up @@ -331,7 +331,7 @@ class B(BaseModel):
foo: Literal['b']

class Top(BaseModel):
sub: Union[A, B] = Field(..., discriminator='foo')
sub: 'A | B' = Field(..., discriminator='foo')

t = Top(sub=A(foo='a'))
assert isinstance(t, Top)
Expand All @@ -346,7 +346,7 @@ class B(BaseModel):
literal: Literal['b'] = Field(alias='lit')

class Top(BaseModel):
sub: Union[A, B] = Field(..., discriminator='literal')
sub: 'A | B' = Field(..., discriminator='literal')

with pytest.raises(ValidationError) as exc_info:
Top(sub=A(literal='a'))
Expand All @@ -367,7 +367,7 @@ class B(BaseModel):
m: Literal[2]

class Top(BaseModel):
sub: Union[A, B] = Field(..., discriminator='m')
sub: 'A | B' = Field(..., discriminator='m')

assert isinstance(Top.model_validate({'sub': {'m': 2}}).sub, B)
with pytest.raises(ValidationError) as exc_info:
Expand Down Expand Up @@ -416,7 +416,7 @@ class B(BaseModel):
m: Literal[EnumValue.b]

class Top(BaseModel):
sub: Union[A, B] = Field(..., discriminator='m')
sub: 'A | B' = Field(..., discriminator='m')

assert isinstance(Top.model_validate({'sub': {'m': EnumValue.b}}).sub, B)
if isinstance(EnumValue.b, (int, str)):
Expand Down Expand Up @@ -449,7 +449,7 @@ class Dog(BaseModel):
with pytest.raises(TypeError, match=re.escape("Aliases for discriminator 'pet_type' must be the same (got T, U)")):

class Model(BaseModel):
pet: Union[Cat, Dog] = Field(discriminator='pet_type')
pet: 'Cat | Dog' = Field(discriminator='pet_type')


def test_alias_same():
Expand All @@ -462,7 +462,7 @@ class Dog(BaseModel):
d: str

class Model(BaseModel):
pet: Union[Cat, Dog] = Field(discriminator='pet_type')
pet: 'Cat | Dog' = Field(discriminator='pet_type')

assert Model(**{'pet': {'typeOfPet': 'dog', 'd': 'milou'}}).pet.pet_type == 'dog'

Expand All @@ -483,7 +483,7 @@ class Lizard(BaseModel):
name: str

class Model(BaseModel):
pet: Union[CommonPet, Lizard] = Field(..., discriminator='pet_type')
pet: 'CommonPet | Lizard' = Field(..., discriminator='pet_type')
n: int

assert isinstance(Model(**{'pet': {'pet_type': 'dog', 'name': 'Milou'}, 'n': 5}).pet, Dog)
Expand All @@ -501,7 +501,7 @@ class Failure(BaseModel):
error_message: str

class Container(BaseModel, Generic[T]):
result: Union[Success[T], Failure] = Field(discriminator='type')
result: 'Success[T] | Failure' = Field(discriminator='type')

with pytest.raises(ValidationError, match="Unable to extract tag using discriminator 'type'"):
Container[str].model_validate({'result': {}})
Expand Down Expand Up @@ -545,7 +545,7 @@ class Dog(BaseModel):
name: str

class Pet(BaseModel):
pet: Optional[Union[Cat, Dog]] = Field(discriminator='pet_type')
pet: 'Cat | Dog | None' = Field(discriminator='pet_type')

assert Pet(pet={'pet_type': 'cat', 'name': 'Milo'}).model_dump() == {'pet': {'name': 'Milo', 'pet_type': 'cat'}}
assert Pet(pet={'pet_type': 'dog', 'name': 'Otis'}).model_dump() == {'pet': {'name': 'Otis', 'pet_type': 'dog'}}
Expand Down Expand Up @@ -592,7 +592,7 @@ class Dog(BaseModel):
name: str

class Pet(BaseModel):
pet: Optional[Union[Cat, Dog]] = Field(default=None, discriminator='pet_type')
pet: 'Cat | Dog | None' = Field(default=None, discriminator='pet_type')

assert Pet(pet={'pet_type': 'cat', 'name': 'Milo'}).model_dump() == {'pet': {'name': 'Milo', 'pet_type': 'cat'}}
assert Pet(pet={'pet_type': 'dog', 'name': 'Otis'}).model_dump() == {'pet': {'name': 'Otis', 'pet_type': 'dog'}}
Expand Down Expand Up @@ -634,7 +634,7 @@ class Case2(BaseModel):
with pytest.raises(PydanticUserError, match="Model 'Case1' needs a discriminator field for key 'kind'"):

class TaggedParent(BaseModel):
tagged: Union[Case1, Case2] = Field(discriminator='kind')
tagged: 'Case1 | Case2' = Field(discriminator='kind')


def test_nested_optional_unions() -> None:
Expand All @@ -651,7 +651,7 @@ class Lizard(BaseModel):
MaybeDogLizard = Annotated[Union[Dog, Lizard, None], Field(discriminator='pet_type')]

class Pet(BaseModel):
pet: Union[MaybeCatDog, MaybeDogLizard] = Field(discriminator='pet_type')
pet: 'MaybeCatDog | MaybeDogLizard' = Field(discriminator='pet_type')

Pet.model_validate({'pet': {'pet_type': 'dog'}})
Pet.model_validate({'pet': {'pet_type': 'cat'}})
Expand Down Expand Up @@ -740,7 +740,7 @@ class Lizard(BaseModel):
MaybeDogLizard = Annotated[Optional[Union[Dog, Lizard]], 'some other annotation']

class Model(BaseModel):
maybe_pet: Union[MaybeCat, MaybeDogLizard] = Field(discriminator='pet_type')
maybe_pet: 'MaybeCat | MaybeDogLizard' = Field(discriminator='pet_type')

assert Model(**{'maybe_pet': None}).maybe_pet is None
assert Model(**{'maybe_pet': {'typeOfPet': 'dog', 'd': 'milou'}}).maybe_pet.pet_type == 'dog'
Expand Down
18 changes: 9 additions & 9 deletions tests/test_edge_cases.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@

def test_str_bytes():
class Model(BaseModel):
v: Union[str, bytes]
v: 'str | bytes'

m = Model(v='s')
assert m.v == 's'
Expand All @@ -72,7 +72,7 @@ class Model(BaseModel):

def test_str_bytes_none():
class Model(BaseModel):
v: Union[None, str, bytes] = ...
v: 'None | str | bytes' = ...

m = Model(v='s')
assert m.v == 's'
Expand Down Expand Up @@ -496,7 +496,7 @@ class Model(BaseModel):

def test_list_unions():
class Model(BaseModel):
v: List[Union[int, str]] = ...
v: 'List[int | str]' = ...

assert Model(v=[123, '456', 'foobar']).v == [123, '456', 'foobar']

Expand All @@ -511,7 +511,7 @@ class Model(BaseModel):

def test_recursive_lists():
class Model(BaseModel):
v: List[List[Union[int, float]]] = ...
v: 'List[List[int | float]]' = ...

assert Model(v=[[1, 2], [3, '4', '4.1']]).v == [[1, 2], [3, 4, 4.1]]
assert Model.model_fields['v'].annotation == List[List[Union[int, float]]]
Expand Down Expand Up @@ -1209,7 +1209,7 @@ class InvalidDefinitionModel(BaseModel):

def test_multiple_errors():
class Model(BaseModel):
a: Union[None, int, float, Decimal]
a: 'None | int | float | Decimal'

with pytest.raises(ValidationError) as exc_info:
Model(a='foobar')
Expand Down Expand Up @@ -1373,8 +1373,8 @@ class Model(BaseModel):
e: Type[FooBar]
f: Type[FooBar] = FooBar
g: Sequence[Type[FooBar]] = [FooBar]
h: Union[Type[FooBar], Sequence[Type[FooBar]]] = FooBar
i: Union[Type[FooBar], Sequence[Type[FooBar]]] = [FooBar]
h: 'Type[FooBar] | Sequence[Type[FooBar]]' = FooBar
i: 'Type[FooBar] | Sequence[Type[FooBar]]' = [FooBar]

model_config = dict(arbitrary_types_allowed=True)

Expand Down Expand Up @@ -2555,8 +2555,8 @@ class Model(BaseModel):

def test_type_union():
class Model(BaseModel):
a: Type[Union[str, bytes]]
b: Type[Union[Any, str]]
a: 'Type[str | bytes]'
b: 'Type[Any | str]'

m = Model(a=bytes, b=int)
assert m.model_dump() == {'a': bytes, 'b': int}
Expand Down
4 changes: 2 additions & 2 deletions tests/test_generics.py
Original file line number Diff line number Diff line change
Expand Up @@ -595,7 +595,7 @@ def test_complex_nesting():
T = TypeVar('T')

class MyModel(BaseModel, Generic[T]):
item: List[Dict[Union[int, T], str]]
item: 'List[Dict[int | T, str]]'

item = [{1: 'a', 'a': 'a'}]
model = MyModel[str](item=item)
Expand Down Expand Up @@ -1869,7 +1869,7 @@ class M1(BaseModel, Generic[V1, V2]):
m: 'M2[V1]'

class M2(BaseModel, Generic[V3]):
m: Union[M1[V3, int], V3]
m: 'M1[V3, int] | V3'

M1.model_rebuild()

Expand Down
4 changes: 2 additions & 2 deletions tests/test_json_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -1278,7 +1278,7 @@ class ModelWithOverride(BaseModel):
)
def test_callable_type_with_fallback(default_value, properties):
class Model(BaseModel):
callback: Union[int, Callable[[int], int]] = default_value
callback: 'int | Callable[[int], int]' = default_value

class MyGenerator(GenerateJsonSchema):
ignored_warning_kinds = ()
Expand Down Expand Up @@ -1331,7 +1331,7 @@ class Model(BaseModel):
)
def test_callable_fallback_with_non_serializable_default(warning_match):
class Model(BaseModel):
callback: Union[int, Callable[[int], int]] = lambda x: x # noqa E731
callback: 'int | Callable[[int], int]' = lambda x: x # noqa E731

class MyGenerator(GenerateJsonSchema):
ignored_warning_kinds = ()
Expand Down
6 changes: 3 additions & 3 deletions tests/test_root_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -474,12 +474,12 @@ class RModel(RootModel):
if order == 'BR':

class Model(RootModel):
root: List[Union[BModel, RModel]]
root: 'List[BModel | RModel]'

elif order == 'RB':

class Model(RootModel):
root: List[Union[RModel, BModel]]
root: 'List[RModel | BModel]'

m = Model([1, 2, {'value': 'abc'}])

Expand Down Expand Up @@ -508,7 +508,7 @@ class SModel(BaseModel):
str_value: str

class Model(RootModel):
root: Union[SModel, RModel] = Field(discriminator='kind')
root: 'SModel | RModel' = Field(discriminator='kind')

assert Model(data).model_dump() == data
assert Model(**data).model_dump() == data
Expand Down
12 changes: 6 additions & 6 deletions tests/test_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -4803,7 +4803,7 @@ class Model(BaseModel):

def test_default_union_types():
class DefaultModel(BaseModel):
v: Union[int, bool, str]
v: 'int | bool | str'

# do it this way since `1 == True`
assert repr(DefaultModel(v=True).v) == 'True'
Expand Down Expand Up @@ -4884,7 +4884,7 @@ class B(BaseModel):
x: str

class Model(BaseModel):
y: Union[A, B]
y: 'A | B'

assert isinstance(Model(y=A(x='a')).y, A)
assert isinstance(Model(y=B(x='b')).y, B)
Expand All @@ -4896,7 +4896,7 @@ class MyStr(str):
...

class Model(BaseModel):
x: Union[int, Annotated[str, Field(max_length=max_length)]]
x: 'int | Annotated[str, Field(max_length=max_length)]'

v = Model(x=MyStr('1')).x
assert type(v) is str
Expand All @@ -4905,7 +4905,7 @@ class Model(BaseModel):

def test_union_compound_types():
class Model(BaseModel):
values: Union[Dict[str, str], List[str], Dict[str, List[str]]]
values: 'Dict[str, str] | List[str] | Dict[str, List[str]]'

assert Model(values={'L': '1'}).model_dump() == {'values': {'L': '1'}}
assert Model(values=['L1']).model_dump() == {'values': ['L1']}
Expand Down Expand Up @@ -4939,7 +4939,7 @@ class Model(BaseModel):

def test_smart_union_compounded_types_edge_case():
class Model(BaseModel):
x: Union[List[str], List[int]]
x: 'List[str] | List[int]'

assert Model(x=[1, 2]).x == [1, 2]
assert Model(x=['1', '2']).x == ['1', '2']
Expand All @@ -4954,7 +4954,7 @@ class Dict2(TypedDict):
bar: str

class M(BaseModel):
d: Union[Dict2, Dict1]
d: 'Dict2 | Dict1'

assert M(d=dict(foo='baz')).d == {'foo': 'baz'}

Expand Down
5 changes: 4 additions & 1 deletion tests/test_typing.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from pydantic import Field # noqa: F401
from pydantic._internal._typing_extra import (
NoneType,
eval_type_backport,
get_function_type_hints,
is_classvar,
is_literal_type,
Expand Down Expand Up @@ -65,7 +66,9 @@ def test_is_none_type():
assert is_none_type(Callable) is False


@pytest.mark.parametrize('union_gen', [lambda: typing.Union[int, str], lambda: int | str])
@pytest.mark.parametrize(
'union_gen', [lambda: typing.Union[int, str], lambda: int | str, eval_type_backport('int | str')]
)
def test_is_union(union_gen):
try:
union = union_gen()
Expand Down
4 changes: 2 additions & 2 deletions tests/test_validators.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from enum import Enum
from functools import partial, partialmethod
from itertools import product
from typing import Any, Callable, Deque, Dict, FrozenSet, List, NamedTuple, Optional, Tuple, Union
from typing import Any, Callable, Deque, Dict, FrozenSet, List, NamedTuple, Optional, Tuple
from unittest.mock import MagicMock

import pytest
Expand Down Expand Up @@ -1670,7 +1670,7 @@ class Model(BaseModel):

def test_union_literal_with_constraints():
class Model(BaseModel, validate_assignment=True):
x: Union[Literal[42], Literal['pika']] = Field(frozen=True)
x: "Literal[42] | Literal['pika']" = Field(frozen=True)

m = Model(x=42)
with pytest.raises(ValidationError) as exc_info:
Expand Down
0