8000 Add pipeline API by adriangb · Pull Request #9459 · pydantic/pydantic · GitHub
[go: up one dir, main page]

Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
62 commits
Select commit Hold shift + click to select a range
e7eb43c
Add pipeline API
adriangb May 20, 2024
7d78407
port to python 3.10
adriangb May 20, 2024
6a1622d
port to python 3.10
adriangb May 20, 2024
947e920
fix syntax
adriangb May 20, 2024
b3fe1c4
handle slots
adriangb May 20, 2024
a6d56cc
Remove match
adriangb May 21, 2024
42505df
Remove match
adriangb May 21, 2024
b2855d0
ignore warning
adriangb May 21, 2024
aafb856
fix import
adriangb May 21, 2024
0bd8b39
fix union
adriangb May 21, 2024
697833f
fix union
adriangb May 21, 2024
d767731
sort imports
adriangb May 21, 2024
8abb6e4
move
adriangb May 21, 2024
217b11d
move
adriangb May 21, 2024
89c46a1
add missing file
adriangb May 21, 2024
6e91a32
namespace
adriangb May 23, 2024
8e4d535
initial tests
sydney-runkle May 29, 2024
ada5853
add more operators
adriangb May 30, 2024
8742e9e
Add json schema tests, add section mapping existing validators
adriangb May 31, 2024
f55b6e1
move things around for expeirmental pattern
sydney-runkle May 31, 2024
7132bae
fix docs tests
sydney-runkle May 31, 2024
0444fc9
maybe fix 3.9 test
sydney-runkle May 31, 2024
1a8e505
use typing Pattern
sydney-runkle May 31, 2024
d979841
add PydanticExperimentalWarning
sydney-runkle May 31, 2024
fadf3bb
ignore warnings, for some reason pytestmark wasn't working
sydney-runkle May 31, 2024
1699f35
3.8 friendly removesuffix
sydney-runkle May 31, 2024
d0a9372
Apply docs suggestions from code review
sydney-runkle Jun 4, 2024
bed0752
add __all__
adriangb Jun 4, 2024
eb61549
rename class to pipeline
adriangb Jun 4, 2024
a18a4df
get rid of on_lambda_err
adriangb Jun 4, 2024
34663fe
pr feedback
adriangb Jun 4, 2024
dff9ad9
make transform use the field type instead of any
adriangb Jun 4, 2024
479ab3c
add import
adriangb Jun 4, 2024
7b49219
rename parse() -> validate_as()
adriangb Jun 4, 2024
51bcad6
rename internal classes
adriangb Jun 4, 2024
13b1721
make Pipeline _Pipeline
adriangb Jun 4, 2024
b8573b5
Remove namespaces
adriangb Jun 4, 2024
888c4ed
more test
adriangb Jun 4, 2024
141c8b6
use ellipsis
sydney-runkle Jun 4, 2024
9d4194b
updating imports from internal test
sydney-runkle Jun 4, 2024
128d4ea
maybe fixing zoneinfo tests, switching up validate_as annotation again
sydney-runkle Jun 4, 2024
1c7302d
docs and linting
sydney-runkle Jun 4, 2024
88dcb75
removing tzinfo stuff :(
sydney-runkle Jun 5, 2024
19a3ee6
a bit more explanation
sydney-runkle Jun 5, 2024
0652472
api docs update
sydney-runkle Jun 5, 2024
4ccf4e5
Additional Test Cases for Experimental Pipeline API (#9566)
dAIsySHEng1 Jun 5, 2024
bad0a1a
fix common predicates + add tests
sydney-runkle Jun 5, 2024
a9d1099
remove unneeded line
sydney-runkle Jun 5, 2024
14e9944
update to version policy docs
sydney-runkle Jun 5, 2024
42a2708
skip linting
sydney-runkle Jun 5, 2024
021604f
fix type hint for _Pipeline.then
adriangb Jun 5, 2024
38a2730
Apply suggestions from code review
sydney-runkle Jun 5, 2024
0c36b7c
Update pydantic/experimental/pipeline.py
sydney-runkle Jun 5, 2024
8d46b21
add public todo
sydney-runkle Jun 5, 2024
a46c2e3
move predicate up
sydney-runkle Jun 5, 2024
7386d69
new idea for overload
sydney-runkle Jun 5, 2024
dc07b50
test fixes
sydney-runkle Jun 5, 2024
cbb216b
update test cases with comments
sydney-runkle Jun 5, 2024
581cbe8
no freeze notes
sydney-runkle Jun 5, 2024
c3a008f
suggested frozen change
sydney-runkle Jun 5, 2024
26c5325
add test
adriangb Jun 5, 2024
166df3d
add more assertions
adriangb Jun 5, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Remove match
  • Loading branch information
adriangb committed May 21, 2024
commit a6d56cce1549561c422723d27848150463543b1f
233 changes: 123 additions & 110 deletions pydantic/_internal/_transform.py
629A
Original file line number Diff line number Diff line change
Expand Up @@ -285,21 +285,19 @@ def handler(v: Any) -> Any:
def _apply_step(step: _Step, s: cs.CoreSchema | None, handler: GetCoreSchemaHandler, source_type: Any) -> cs.CoreSchema:
from pydantic_core import core_schema as cs

match step:
case _Parse():
s = _apply_parse(s, step.tp, step.strict, handler, source_type)
case _ParseDefer():
s = _apply_parse(s, step.tp, False, handler, source_type)
case _Transform():
s = _apply_transform(s, step.func)
case _Constraint():
s = _apply_constraint(s, step.constraint)
case _ValidateOr():
assert isinstance(step, _ValidateOr)
s = cs.union_schema([handler(step.left), handler(step.right)])
case _ValidateAnd():
assert isinstance(step, _ValidateAnd)
s = cs.chain_schema([handler(step.left), handler(step.right)])
if i 10000 sinstance(step, _Parse):
s = _apply_parse(s, step.tp, step.strict, handler, source_type)
elif isinstance(step, _ParseDefer):
s = _apply_parse(s, step.tp, False, handler, source_type)
elif isinstance(step, _Transform):
s = _apply_transform(s, step.func)
elif isinstance(step, _Constraint):
s = _apply_constraint(s, step.constraint)
elif isinstance(step, _ValidateOr):
s = cs.union_schema([handler(step.left), handler(step.right)])
else:
assert isinstance(step, _ValidateAnd)
s = cs.chain_schema([handler(step.left), handler(step.right)])
return s


Expand Down Expand Up @@ -351,120 +349,135 @@ def _apply_transform(s: cs.CoreSchema | None, func: Callable[[Any], Any]) -> cs.
def _apply_constraint( # noqa: C901
s: cs.CoreSchema | None, constraint: _ConstraintAnnotation
) -> cs.CoreSchema:
match constraint:
case annotated_types.Gt(gt):
if s and s['type'] in ('int', 'float', 'decimal'):
s = s.copy()
if s['type'] == 'int' and isinstance(gt, int):
s['gt'] = gt
elif s['type'] == 'float' and isinstance(gt, float):
s['gt'] = gt
elif s['type'] == 'decimal' and isinstance(gt, Decimal):
s['gt'] = gt
return s

def check_gt(v: Any) -> bool:
return v > gt
if isinstance(constraint, annotated_types.Gt):
gt = constraint.gt
if s and s['type'] in ('int', 'float', 'decimal'):
s = s.copy()
if s['type'] == 'int' and isinstance(gt, int):
s['gt'] = gt
elif s['type'] == 'float' and isinstance(gt, float):
s['gt'] = gt
elif s['type'] == 'decimal' and isinstance(gt, Decimal):
s['gt'] = gt
return s

s = _check_func(check_gt, f'> {gt}', s)
case annotated_types.Ge(ge):
def check_gt(v: Any) -> bool:
return v > gt

def check_ge(v: Any) -> bool:
return v >= ge
s = _check_func(check_gt, f'> {gt}', s)
if isinstance(constraint, annotated_types.Ge):
ge = constraint.ge

s = _check_func(check_ge, f'>= {ge}', s)
case annotated_types.Lt(lt):
def check_ge(v: Any) -> bool:
return v >= ge

def check_lt(v: Any) -> bool:
return v < lt
s = _check_func(check_ge, f'>= {ge}', s)
if isinstance(constraint, annotated_types.Lt):
lt = constraint.lt

s = _check_func(check_lt, f'< {lt}', s)
case annotated_types.Le(le):
def check_lt(v: Any) -> bool:
return v < lt

def check_le(v: Any) -> bool:
return v <= le
s = _check_func(check_lt, f'< {lt}', s)
if isinstance(constraint, annotated_types.Le):
le = constraint.le

s = _check_func(check_le, f'<= {le}', s)
case annotated_types.Len(min_len, max_len):
def check_le(v: Any) -> bool:
return v <= le

def check_len(v: Any) -> bool:
if max_len is not None:
return (min_len <= len(v)) and (len(v) <= max_len)
return min_len <= len(v)
s = _check_func(check_le, f'<= {le}', s)
if isinstance(constraint, annotated_types.Len):
min_len = constraint.min_length
max_len = constraint.max_length

s = _check_func(check_len, f'length >= {min_len} and length <= {max_len}', s)
case annotated_types.MultipleOf(multiple_of):
def check_len(v: Any) -> bool:
if max_len is not None:
return (min_len <= len(v)) and (len(v) <= max_len)
return min_len <= len(v)

def check_multiple_of(v: Any) -> bool:
return v % multiple_of == 0
s = _check_func(check_len, f'length >= {min_len} and length <= {max_len}', s)
if isinstance(constraint, annotated_types.MultipleOf):
multiple_of = constraint.multiple_of

s = _check_func(check_multiple_of, f'% {multiple_of} == 0', s)
case annotated_types.Timezone(tz):
if tz is ...:
if s and s['type'] == 'datetime':
s = s.copy()
s['tz_constraint'] = 'aware'
else:
def check_multiple_of(v: Any) -> bool:
return v % multiple_of == 0

def check_tz_aware(v: object) -> bool:
assert isinstance(v, datetime.datetime)
return v.tzinfo is not None
s = _check_func(check_multiple_of, f'% {multiple_of} == 0', s)
if isinstance(constraint, annotated_types.Timezone):
tz = constraint.tz

s = _check_func(check_tz_aware, 'timezone aware', s)
elif tz is None:
if s and s['type'] == 'datetime':
s = s.copy()
s['tz_constraint'] = 'naive'
else:
if tz is ...:
if s and s['type'] == 'datetime':
s = s.copy()
s['tz_constraint'] = 'aware'
else:

def check_tz_naive(v: object) -> bool:
assert isinstance(v, datetime.datetime)
return v.tzinfo is None
def check_tz_aware(v: object) -> bool:
assert isinstance(v, datetime.datetime)
return v.tzinfo is not None

s = _check_func(check_tz_naive, 'timezone naive', s)
else:
raise NotImplementedError('Constraining to a specific timezone is not yet supported')
case annotated_types.Interval(min_val, max_val):

def check_interval(v: Any) -> bool:
return min_val <= v <= max_val

s = _check_func(check_interval, f'>= {min_val} and <= {max_val}', s)
case annotated_types.Predicate(func):
if func.__name__ == '<lambda>':

def on_lambda_err() -> str:
# TODO: is there a better way?
import inspect

try:
return (
'`'
+ ''.join(
''.join(inspect.getsource(func).strip().removesuffix(')').split('lambda ')[1:]).split(
':'
)[1:]
).strip()
+ '`'
)
except OSError:
# stringified annotations
return 'lambda'

s = _check_func(func, on_lambda_err, s)
else:
s = _check_func(func, func.__name__, s)
case re.Pattern():
if s and s['type'] == 'str':
s = _check_func(check_tz_aware, 'timezone aware', s)
elif tz is None:
if s and s['type'] == 'datetime':
s = s.copy()
s['pattern'] = constraint.pattern
s['tz_constraint'] = 'naive'
else:

def check_pattern(v: object) -> bool:
assert isinstance(v, str)
return constraint.match(v) is not None
def check_tz_naive(v: object) -> bool:
assert isinstance(v, datetime.datetime)
return v.tzinfo is None

s = _check_func(check_tz_naive, 'timezone naive', s)
else:
raise NotImplementedError('Constraining to a specific timezone is not yet supported')
if isinstance(constraint, annotated_types.Interval):
if constraint.ge:
s = _apply_constraint(s, annotated_types.Ge(constraint.ge))
if constraint.gt:
s = _apply_constraint(s, annotated_types.Gt(constraint.gt))
if constraint.le:
s = _apply_constraint(s, annotated_types.Le(constraint.le))
if constraint.lt:
s = _apply_constraint(s, annotated_types.Lt(constraint.lt))
if isinstance(constraint, annotated_types.Predicate):
func = constraint.func

if func.__name__ == '<lambda>':

def on_lambda_err() -> str:
# TODO: is there a better way?
import inspect

try:
return (
'`'
+ ''.join(
''.join(inspect.getsource(func).strip().removesuffix(')').split('lambda ')[1:]).split(':')[
1:
]
).strip()
+ '`'
)
except OSError:
# stringified annotations
return 'lambda'

s = _check_func(func, on_lambda_err, s)
else:
s = _check_func(func, func.__name__, s)
if isinstance(constraint, re.Pattern):
if s and s['type'] == 'str':
s = s.copy()
s['pattern'] = constraint.pattern
else:

def check_pattern(v: object) -> bool:
assert isinstance(v, str)
return constraint.match(v) is not None

s = _check_func(check_pattern, f'~ {constraint.pattern}', s)
s = _check_func(check_pattern, f'~ {constraint.pattern}', s)
else:
raise NotImplementedError(f'Constraint {constraint} is not yet supported')
return s


Expand Down
0