8000 Improve `__setattr__` performance of Pydantic models by caching sette… · pydantic/pydantic@addf1f9 · GitHub
[go: up one dir, main page]
Skip to content

Commit addf1f9

Browse files
MarkusSintonenMarkusSintonen
authored
Improve __setattr__ performance of Pydantic models by caching setter functions (#10868)
1 parent 30ee4f4 commit addf1f9

File tree

5 files changed

+221
-37
lines changed

5 files changed

+221
-37
lines changed

‎pydantic/_internal/_model_construction.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,8 @@ def wrapped_model_post_init(self: BaseModel, context: Any, /) -> None:
152152
None if cls.model_post_init is BaseModel_.model_post_init else 'model_post_init'
153153
)
154154

155+
cls.__pydantic_setattr_handlers__ = {}
156+
155157
cls.__pydantic_decorators__ = DecoratorInfos.build(cls)
156158

157159
# Use the getattr below to grab the __parameters__ from the `typing.Generic` parent class

‎pydantic/main.py

Lines changed: 64 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,20 @@
8181
_object_setattr = _model_construction.object_setattr
8282

8383

84+
def _model_field_setattr_handler(model: BaseModel, name: str, val: Any) -> None:
85+
model.__dict__[name] = val
86+
model.__pydantic_fields_set__.add(name)
87+
88+
89+
_SIMPLE_SETATTR_HANDLERS: Mapping[str, Callable[[BaseModel, str, Any], None]] = {
90+
'model_field': _model_field_setattr_handler,
91+
'validate_assignment': lambda model, name, val: model.__pydantic_validator__.validate_assignment(model, name, val), # pyright: ignore[reportAssignmentType]
92+
'private': lambda model, name, val: model.__pydantic_private__.__setitem__(name, val), # pyright: ignore[reportOptionalMemberAccess]
93+
'cached_property': lambda model, name, val: model.__dict__.__setitem__(name, val),
94+
'extra_known': lambda model, name, val: _object_setattr(model, name, val),
95+
}
96+
97+
8498
class BaseModel(metaclass=_model_construction.ModelMetaclass):
8599
"""Usage docs: https://docs.pydantic.dev/2.10/concepts/models/
86100
@@ -169,6 +183,9 @@ class BaseModel(metaclass=_model_construction.ModelMetaclass):
169183
This replaces `Model.__fields__` from Pydantic V1.
170184
"""
171185

186+
__pydantic_setattr_handlers__: ClassVar[Dict[str, Callable[[BaseModel, str, Any], None]]] # noqa: UP006
187+
"""`__setattr__` handlers. Memoizing the handlers leads to a dramatic performance improvement in `__setattr__`"""
188+
172189
__pydantic_computed_fields__: ClassVar[Dict[str, ComputedFieldInfo]] # noqa: UP006
173190
"""A dictionary of computed field names and their corresponding [`ComputedFieldInfo`][pydantic.fields.ComputedFieldInfo] objects."""
174191

@@ -890,53 +907,63 @@ def __getattr__(self, item: str) -> Any:
890907
raise AttributeError(f'{type(self).__name__!r} object has no attribute {item!r}')
891908

892909
def __setattr__(self, name: str, value: Any) -> None:
893-
if name in self.__class_vars__:
910+
if (setattr_handler := self.__pydantic_setattr_handlers__.get(name)) is not None:
911+
setattr_handler(self, name, value)
912+
# if None is returned from _setattr_handler, the attribute was set directly
913+
elif (setattr_handler := self._setattr_handler(name, value)) is not None:
914+
setattr_handler(self, name, value) # call here to not memo on possibly unknown fields
915+
self.__pydantic_setattr_handlers__[name] = setattr_handler # memoize the handler for faster access
916+
917+
def _setattr_handler(self, name: str, value: Any) -> Callable[[BaseModel, str, Any], None] | None:
918+
"""Get a handler for setting an attribute on the model instance.
919+
920+
Returns:
921+
A handler for setting an attribute on the model instance. Used for memoization of the handler.
922+
Memoizing the handlers leads to a dramatic performance improvement in `__setattr__`
923+
Returns `None` when memoization is not safe, then the attribute is set directly.
924+
"""
925+
cls = self.__class__
926+
if name in cls.__class_vars__:
894927
raise AttributeError(
895-
f'{name!r} is a ClassVar of `{self.__class__.__name__}` and cannot be set on an instance. '
896-
f'If you want to set a value on the class, use `{self.__class__.__name__}.{name} = value`.'
928+
f'{name!r} is a ClassVar of `{cls.__name__}` and cannot be set on an instance. '
929+
f'If you want to set a value on the class, use `{cls.__name__}.{name} = value`.'
897930
)
898931
elif not _fields.is_valid_field_name(name):
899-
if self.__pydantic_private__ is None or name not in self.__private_attributes__:
900-
_object_setattr(self, name, value)
901-
else:
902-
attribute = self.__private_attributes__[name]
932+
if (attribute := cls.__private_attributes__.get(name)) is not None:
903933
if hasattr(attribute, '__set__'):
904-
attribute.__set__(self, value) # type: ignore
934+
return lambda model, _name, val: attribute.__set__(model, val)
905935
else:
906-
self.__pydantic_private__[name] = value
907-
return
936+
return _SIMPLE_SETATTR_HANDLERS['private']
937+
else:
938+
_object_setattr(self, name, value)
939+
return None # Can not return memoized handler with possibly freeform attr names
908940

909-
self._check_frozen(name, value)
941+
cls._check_frozen(name, value)
910942

911-
attr = getattr(self.__class__, name, None)
943+
attr = getattr(cls, name, None)
912944
# NOTE: We currently special case properties and `cached_property`, but we might need
913945
# to generalize this to all data/non-data descriptors at some point. For non-data descriptors
914946
# (such as `cached_property`), it isn't obvious though. `cached_property` caches the value
915947
# to the instance's `__dict__`, but other non-data descriptors might do things differently.
916948
if isinstance(attr, property):
917-
attr.__set__(self, value)
949+
return lambda model, _name, val: attr.__set__(model, val)
918950
elif isinstance(attr, cached_property):
919-
self.__dict__[name] = value
920-
elif self.model_config.get('validate_assignment', None):
921-
self.__pydantic_validator__.validate_assignment(self, name, value)
922-
elif self.model_config.get('extra') != 'allow' and name not in self.__pydantic_fields__:
923-
# TODO - matching error
924-
raise ValueError(f'"{self.__class__.__name__}" object has no field "{name}"')
925-
elif self.model_config.get('extra') == 'allow' and name not in self.__pydantic_fields__:
926-
if self.model_extra and name in self.model_extra:
927-
self.__pydantic_extra__[name] = value # type: ignore
951+
return _SIMPLE_SETATTR_HANDLERS['cached_property']
952+
elif cls.model_config.get('validate_assignment'):
953+
return _SIMPLE_SETATTR_HANDLERS['validate_assignment']
954+
elif name not in cls.__pydantic_fields__:
955+
if cls.model_config.get('extra') != 'allow':
956+
# TODO - matching error
957+
raise ValueError(f'"{cls.__name__}" object has no field "{name}"')
958+
elif attr is None:
959+
# attribute does not exist, so put it in extra
960+
self.__pydantic_extra__[name] = value
961+
return None # Can not return memoized handler with possibly freeform attr names
928962
else:
929-
try:
930-
getattr(self, name)
931-
except AttributeError:
932-
# attribute does not already exist on instance, so put it in extra
933-
self.__pydantic_extra__[name] = value # type: ignore
934-
else:
935-
# attribute _does_ already exist on instance, and was not in extra, so update it
936-
_object_setattr(self, name, value)
963+
# attribute _does_ exist, and was not in extra, so update it
964+
return _SIMPLE_SETATTR_HANDLERS['extra_known']
937965
else:
938-
self.__dict__[name] = value
939-
self.__pydantic_fields_set__.add(name)
966+
return _SIMPLE_SETATTR_HANDLERS['model_field']
940967

941968
def __delattr__(self, item: str) -> Any:
942969
if item in self.__private_attributes__:
@@ -964,10 +991,11 @@ def __delattr__(self, item: str) -> Any:
964991
except AttributeError:
965992
raise AttributeError(f'{type(self).__name__!r} object has no attribute {item!r}')
966993

967-
def _check_frozen(self, name: str, value: Any) -> None:
968-
if self.model_config.get('frozen', None):
994+
@classmethod
995+
def _check_frozen(cls, name: str, value: Any) -> None:
996+
if cls.model_config.get('frozen', None):
969997
typ = 'frozen_instance'
970-
elif getattr(self.__pydantic_fields__.get(name), 'frozen', False):
998+
elif getattr(cls.__pydantic_fields__.get(name), 'frozen', False):
971999
typ = 'frozen_field'
9721000
else:
9731001
return
@@ -976,7 +1004,7 @@ def _check_frozen(self, name: str, value: Any) -> None:
9761004
'loc': (name,),
9771005
'input': value,
9781006
}
979-
raise pydantic_core.ValidationError.from_exception_data(self.__class__.__name__, [error])
1007+
raise pydantic_core.ValidationError.from_exception_data(cls.__name__, [error])
9801008

9811009
def __getstate__(self) -> dict[Any, Any]:
9821010
private = self.__pydantic_private__

‎tests/benchmarks/test_attribute_access.py

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
from functools import cached_property
2+
3+
import pytest
4+
5+
from pydantic import BaseModel, ConfigDict, ValidationError
6+
7+
8+
class InnerValidateAssignment(BaseModel):
9+
model_config = ConfigDict(validate_assignment=True)
10+
inner_field1: str
11+
inner_field2: int
12+
13+
14+
class Model(BaseModel):
15+
field1: str
16+
field2: int
17+
field3: float
18+
inner1: InnerValidateAssignment
19+
inner2: InnerValidateAssignment
20+
21+
_private_field1: str
22+
_private_field2: int
23+
_private_field3: float
24+
25+
@cached_property
26+
def prop_cached1(self) -> str:
27+
return self.field1 + self._private_field1
28+
29+
@cached_property
30+
def prop_cached2(self) -> int:
31+
return self.field2 + self._private_field2
32+
33+
@cached_property
34+
def prop_cached3(self) -> float:
35+
return self.field3 + self._private_field3
36+
37+
38+
def test_setattr(benchmark):
39+
def set_attrs(m):
40+
m.field1 = 'test1'
41+
m.field2 = 43
42+
m.field3 = 4.0
43+
m.inner1.inner_field1 = 'test inner1'
44+
m.inner1.inner_field2 = 421
45+
m.inner2.inner_field1 = 'test inner2'
46+
m.inner2.inner_field2 = 422
47+
m._private_field1 = 'test2'
48+
m._private_field2 = 44
49+
m._private_field3 = 5.1
50+
m.prop_cached1 = 'cache override'
51+
m.prop_cached2 = 10
52+
m.prop_cached3 = 10.1
53+
54+
inner = {'inner_field1': 'test inner', 'inner_field2': 420}
55+
model = Model(field1='test', field2=42, field3=3.14, inner1=inner, inner2=inner)
56+
benchmark(set_attrs, model)
57+
58+
model.field2 = 'bad' # check benchmark setup
59+
with pytest.raises(ValidationError):
60+
model.inner1.field2 = 'bad'
61+
62+
63+
def test_getattr(benchmark):
64+
def get_attrs(m):
65+
_ = m.field1
66+
_ = m.field2
67+
_ = m.field3
68+
_ = m.inner1.inner_field1
69+
_ = m.inner1.inner_field2
70+
_ = m.inner2.inner_field1
71+
_ = m.inner2.inner_field2
72+
_ = m._private_field1
73+
_ = m._private_field2
74+
_ = m._private_field3
75+
_ = m.prop_cached1
76+
_ = m.prop_cached2
77+
_ = m.prop_cached3
78+
79+
inner = {'inner_field1': 'test inner', 'inner_field2': 420}
80+
model = Model(field1='test1', field2=42, field3=3.14, inner1=inner, inner2=inner)
81+
model._private_field1 = 'test2'
82+
model._private_field2 = 43
83+
model._private_field3 = 4.14
84+
benchmark(get_attrs, model)

‎tests/test_edge_cases.py

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2900,3 +2900,69 @@ def default_int_factory() -> int: ...
29002900
int_factory: Callable[[], int] = Field(default=default_int_factory)
29012901

29022902
assert SomeModel.model_fields['int_factory'].default is SomeModel.default_int_factory
2903+
2904+
2905+
def test_setattr_handler_memo_does_not_inherit() -> None:
2906+
class Model1(BaseModel):
2907+
a: int
2908+
2909+
class Model2(Model1):
2910+
a: int
2911+
2912+
m1 = Model1(a=1)
2913+
m2 = Model2(a=10)
2914+
2915+
assert not Model1.__pydantic_setattr_handlers__
2916+
assert not Model2.__pydantic_setattr_handlers__
2917+
2918+
m2.a = 11
2919+
assert not Model1.__pydantic_setattr_handlers__
2920+
assert 'a' in Model2.__pydantic_setattr_handlers__
2921+
handler2 = Model2.__pydantic_setattr_handlers__['a']
2922+
2923+
m1.a = 2
2924+
assert 'a' in Model1.__pydantic_setattr_handlers__
2925+
assert Model1.__pydantic_setattr_handlers__['a'] is handler2
2926+
assert Model2.__pydantic_setattr_handlers__['a'] is handler2
2927+
assert m1.a == 2 and m2.a == 11
2928+
2929+
2930+
def test_setattr_handler_does_not_memoize_unknown_field() -> None:
2931+
class Model(BaseModel):
2932+
a: int
2933+
2934+
m = Model(a=1)
2935+
with pytest.raises(ValueError, match='object has no field "unknown"'):
2936+
m.unknown = 'x'
2937+
assert not Model.__pydantic_setattr_handlers__
2938+
m.a = 2
2939+
assert 'a' in Model.__pydantic_setattr_handlers__
2940+
2941+
2942+
def test_setattr_handler_does_not_memoize_unknown_private_field() -> None:
2943+
class Model(BaseModel):
2944+
a: int
2945+
_p: str
2946+
2947+
m = Model(a=1)
2948+
assert not Model.__pydantic_setattr_handlers__
2949+
m.a = 2
2950+
assert len(Model.__pydantic_setattr_handlers__) == 1
2951+
m._unknown = 'x'
2952+
assert len(Model.__pydantic_setattr_handlers__) == 1
2953+
m._p = 'y'
2954+
assert len(Model.__pydantic_setattr_handlers__) == 2
2955+
2956+
2957+
def test_setattr_handler_does_not_memoize_on_validate_assignment_field_failure() -> None:
2958+
class Model(BaseModel, validate_assignment=True):
2959+
a: int
2960+
2961+
m = Model(a=1)
2962+
with pytest.raises(ValidationError):
2963+
m.unknown = 'x'
2964+
with pytest.raises(ValidationError):
2965+
m.a = 'y'
2966+
assert not Model.__pydantic_setattr_handlers__
2967+
m.a = 2
2968+
assert 'a' in Model.__pydantic_setattr_handlers__

‎tests/test_main.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -735,8 +735,12 @@ def test_validating_assignment_pass(ValidateAssignmentModel):
735735
assert p.model_dump() == {'a': 2, 'b': 'hi'}
736736

737737

738-
def test_validating_assignment_fail(ValidateAssignmentModel):
738+
@pytest.mark.parametrize('init_valid', [False, True])
739+
def test_validating_assignment_fail(ValidateAssignmentModel, init_valid: bool):
739740
p = ValidateAssignmentModel(a=5, b='hello')
741+
if init_valid:
742+
p.a = 5
743+
p.b = 'hello'
740744

741745
with pytest.raises(ValidationError) as exc_info:
742746
p.a = 'b'

0 commit comments

Comments
 (0)
0