8000 gh-122163: Add notes for JSON serialization errors (GH-122165) · python/cpython@e6b25e9 · GitHub
[go: up one dir, main page]
Skip to content

Commit e6b25e9

Browse files
serhiy-storchakaserhiy-storchaka
authored
gh-122163: Add notes for JSON serialization errors (GH-122165)
This allows to identify the source of the error.
1 parent c908d1f commit e6b25e9

File tree

8 files changed

+135
-66
lines changed

8 files changed

+135
-66
lines changed

‎Doc/whatsnew/3.14.rst

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,13 @@ Added support for converting any objects that have the
112112
:meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`.
113113
(Contributed by Serhiy Storchaka in :gh:`82017`.)
114114

115+
json
116+
----
117+
118+
Add notes for JSON serialization errors that allow to identify the source
119+
of the error.
120+
(Contributed by Serhiy Storchaka in :gh:`122163`.)
121+
115122
os
116123
--
117124

‎Include/internal/pycore_pyerrors.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,8 @@ extern PyObject* _Py_Offer_Suggestions(PyObject* exception);
161161
PyAPI_FUNC(Py_ssize_t) _Py_UTF8_Edit_Cost(PyObject *str_a, PyObject *str_b,
162162
Py_ssize_t max_cost);
163163

164-
void _PyErr_FormatNote(const char *format, ...);
164+
// Export for '_json' shared extension
165+
PyAPI_FUNC(void) _PyErr_FormatNote(const char *format, ...);
165166

166167
/* Context manipulation (PEP 3134) */
167168

‎Lib/json/encoder.py

Lines changed: 67 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -293,37 +293,40 @@ def _iterencode_list(lst, _current_indent_level):
293293
else:
294294
newline_indent = None
295295
separator = _item_separator
296-
first = True
297-
for value in lst:
298-
if first:
299-
first = False
300-
else:
296+
for i, value in enumerate(lst):
297+
if i:
301298
buf = separator
302-
if isinstance(value, str):
303-
yield buf + _encoder(value)
304-
elif value < F438 span class=pl-c1>is None:
305-
yield buf + 'null'
306-
elif value is True:
307-
yield buf + 'true'
308-
elif value is False:
309-
yield buf + 'false'
310-
elif isinstance(value, int):
311-
# Subclasses of int/float may override __repr__, but we still
312-
# want to encode them as integers/floats in JSON. One example
313-
# within the standard library is IntEnum.
314-
yield buf + _intstr(value)
315-
elif isinstance(value, float):
316-
# see comment above for int
317-
yield buf + _floatstr(value)
318-
else:
319-
yield buf
320-
if isinstance(value, (list, tuple)):
321-
chunks = _iterencode_list(value, _current_indent_level)
322-
elif isinstance(value, dict):
323-
chunks = _iterencode_dict(value, _current_indent_level)
299+
try:
300+
if isinstance(value, str):
301+
yield buf + _encoder(value)
302+
elif value is None:
303+
yield buf + 'null'
304+
elif value is True:
305+
yield buf + 'true'
306+
elif value is False:
307+
yield buf + 'false'
308+
elif isinstance(value, int):
309+
# Subclasses of int/float may override __repr__, but we still
310+
# want to encode them as integers/floats in JSON. One example
311+
# within the standard library is IntEnum.
312+
yield buf + _intstr(value)
313+
elif isinstance(value, float):
314+
# see comment above for int
315+
yield buf + _floatstr(value)
324316
else:
325-
chunks = _iterencode(value, _current_indent_level)
326-
yield from chunks
317+
yield buf
318+
if isinstance(value, (list, tuple)):
319+
chunks = _iterencode_list(value, _current_indent_level)
320+
elif isinstance(value, dict):
321+
chunks = _iterencode_dict(value, _current_indent_level)
322+
else:
323+
chunks = _iterencode(value, _current_indent_level)
324+
yield from chunks
325+
except GeneratorExit:
326+
raise
327+
except BaseException as exc:
328+
exc.add_note(f'when serializing {type(lst).__name__} item {i}')
329+
raise
327330
if newline_indent is not None:
328331
_current_indent_level -= 1
329332
yield '\n' + _indent * _current_indent_level
@@ -382,28 +385,34 @@ def _iterencode_dict(dct, _current_indent_level):
382385
yield item_separator
383386
yield _encoder(key)
384387
yield _key_separator
385-
if isinstance(value, str):
386-
yield _encoder(value)
387-
elif value is None:
388-
yield 'null'
389-
elif value is True:
390-
yield 'true'
391-
elif value is False:
392-
yield 'false'
393-
elif isinstance(value, int):
394-
# see comment for int/float in _make_iterencode
395-
yield _intstr(value)
396-
elif isinstance(value, float):
397-
# see comment for int/float in _make_iterencode
398-
yield _floatstr(value)
399-
else:
400-
if isinstance(value, (list, tuple)):
401-
chunks = _iterencode_list(value, _current_indent_level)
402-
elif isinstance(value, dict):
403-
chunks = _iterencode_dict(value, _current_indent_level)
388+
try:
389+
if isinstance(value, str):
390+
yield _encoder(value)
391+
elif value is None:
392+
yield 'null'
393+
elif value is True:
394+
yield 'true'
395+
elif value is False:
396+
yield 'false'
397+
elif isinstance(value, int):
398+
# see comment for int/float in _make_iterencode
399+
yield _intstr(value)
400+
elif isinstance(value, float):
401+
# see comment for int/float in _make_iterencode
402+
yield _floatstr(value)
404403
else:
405-
chunks = _iterencode(value, _current_indent_level)
406-
yield from chunks
404+
if isinstance(value, (list, tuple)):
405+
chunks = _iterencode_list(value, _current_indent_level)
406+
elif isinstance(value, dict):
407+
chunks = _iterencode_dict(value, _current_indent_level)
408+
else:
409+
chunks = _iterencode(value, _current_indent_level)
410+
yield from chunks
411+
except GeneratorExit:
412+
raise
413+
except BaseException as exc:
414+
exc.add_note(f'when serializing {type(dct).__name__} item {key!r}')
415+
raise
407416
if newline_indent is not None:
408417
_current_indent_level -= 1
409418
yield '\n' + _indent * _current_indent_level
@@ -436,8 +445,14 @@ def _iterencode(o, _current_indent_level):
436445
if markerid in markers:
437446
raise ValueError("Circular reference detected")
438447
markers[markerid] = o
439-
o = _default(o)
440-
yield from _iterencode(o, _current_indent_level)
448+
newobj = _default(o)
449+
try:
450+
yield from _iterencode(newobj, _current_indent_level)
451+
except GeneratorExit:
452+
raise
453+
except BaseException as exc:
454+
exc.add_note(f'when serializing {type(o).__name__} object')
455+
raise
441456
if markers is not None:
442457
del markers[markerid]
443458
return _iterencode

‎Lib/test/test_json/test_default.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,24 @@ def test_default(self):
88
self.dumps(type, default=repr),
99
self.dumps(repr(type)))
1010

11+
def test_bad_default(self):
12+
def default(obj):
13+
if obj is NotImplemented:
14+
raise ValueError
15+
if obj is ...:
16+
return NotImplemented
17+
if obj is type:
18+
return collections
19+
return [...]
20+
21+
with self.assertRaises(ValueError) as cm:
22+
self.dumps(type, default=default)
23+
self.assertEqual(cm.exception.__notes__,
24+
['when serializing ellipsis object',
25+
'when serializing list item 0',
26+
'when serializing module object',
27+
'when serializing type object'])
28+
1129
def test_ordereddict(self):
1230
od = collections.OrderedDict(a=1, b=2, c=3, d=4)
1331
od.move_to_end('b')

‎Lib/test/test_json/test_fail.py

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,8 +100,27 @@ def test_non_string_keys_dict(self):
100100
def test_not_serializable(self):
101101
import sys
102102
with self.assertRaisesRegex(TypeError,
103-
'Object of type module is not JSON serializable'):
103+
'Object of type module is not JSON serializable') as cm:
104104
self.dumps(sys)
105+
self.assertFalse(hasattr(cm.exception, '__notes__'))
106+
107+
with self.assertRaises(TypeError) as cm:
108+
self.dumps([1, [2, 3, sys]])
109+
self.assertEqual(cm.exception.__notes__,
110+
['when serializing list item 2',
111+
'when serializing list item 1'])
112+
113+
with self.assertRaises(TypeError) as cm:
114+
self.dumps((1, (2, 3, sys)))
115+
self.assertEqual(cm.exception.__notes__,
116+
['when serializing tuple item 2',
117+
'when serializing tuple item 1'])
118+
119+
with self.assertRaises(TypeError) as cm:
120+
self.dumps({'a': {'b': sys}})
121+
self.assertEqual(cm.exception.__notes__,
122+
["when serializing dict item 'b'",
123+
"when serializing dict item 'a'"])
105124

106125
def test_truncated_input(self):
107126
test_cases = [

‎Lib/test/test_json/test_recursion.py

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,17 @@ def test_listrecursion(self):
1212
x.append(x)
1313
try:
1414
self.dumps(x)
15-
except ValueError:
16-
pass
15+
except ValueError as exc:
16+
self.assertEqual(exc.__notes__, ["when serializing list item 0"])
1717
else:
1818
self.fail("didn't raise ValueError on list recursion")
1919
x = []
2020
y = [x]
2121
x.append(y)
2222
try:
2323
self.dumps(x)
24-
except ValueError:
25-
pass
24+
except ValueError as exc:
25+
self.assertEqual(exc.__notes__, ["when serializing list item 0"]*2)
2626
else:
2727
self.fail("didn't raise ValueError on alternating list recursion")
2828
y = []
@@ -35,8 +35,8 @@ def test_dictrecursion(self):
3535
x["test"] = x
3636
try:
3737
self.dumps(x)
38-
except ValueError:
39-
pass
38+
except ValueError as exc:
39+
self.assertEqual(exc.__notes__, ["when serializing dict item 'test'"])
4040
else:
4141
self.fail("didn't raise ValueError on dict recursion")
4242
x = {}
@@ -60,8 +60,10 @@ def default(self, o):
6060
enc.recurse = True
6161
try:
6262
enc.encode(JSONTestObject)
63-
except ValueError:
64-
pass
63+
except ValueError as exc:
64+
self.assertEqual(exc.__notes__,
65+
["when serializing list item 0",
66+
"when serializing type object"])
6567
else:
6668
self.fail("didn't raise ValueError on default recursion")
6769

‎Misc/NEWS.d/next/Library/2024-07-23-15-11-13.gh-issue-122163.4wRUuM.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Add notes for JSON serialization errors that allow to identify the source of
2+
the error.

‎Modules/_json.c

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "Python.h"
1212
#include "pycore_ceval.h" // _Py_EnterRecursiveCall()
1313
#include "pycore_runtime.h" // _PyRuntime
14+
#include "pycore_pyerrors.h" // _PyErr_FormatNote
1415

1516
#include "pycore_global_strings.h" // _Py_ID()
1617
#include <stdbool.h> // bool
@@ -1461,6 +1462,7 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer,
14611462

14621463
Py_DECREF(newobj);
14631464
if (rv) {
1465+
_PyErr_FormatNote("when serializing %T object", obj);
14641466
Py_XDECREF(ident);
14651467
return -1;
14661468
}
@@ -1477,7 +1479,7 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer,
14771479

14781480
static int
14791481
encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *first,
1480-
PyObject *key, PyObject *value,
1482+
PyObject *dct, PyObject *key, PyObject *value,
14811483
PyObject *newline_indent,
14821484
PyObject *item_separator)
14831485
{
@@ -1535,6 +1537,7 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir
15351537
return -1;
15361538
}
15371539
if (encoder_listencode_obj(s, writer, value, newline_indent) < 0) {
1540+
_PyErr_FormatNote("when serializing %T item %R", dct, key);
15381541
return -1;
15391542
}
15401543
return 0;
@@ -1606,7 +1609,7 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer,
16061609

16071610
key = PyTuple_GET_ITEM(item, 0);
16081611
value = PyTuple_GET_ITEM(item, 1);
1609-
if (encoder_encode_key_value(s, writer, &first, key, value,
1612+
if (encoder_encode_key_value(s, writer, &first, dct, key, value,
16101613
new_newline_indent,
16111614
current_item_separator) < 0)
16121615
goto bail;
@@ -1616,7 +1619,7 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer,
16161619
} else {
16171620
Py_ssize_t pos = 0;
16181621
while (PyDict_Next(dct, &pos, &key, &value)) {
1619-
if (encoder_encode_key_value(s, writer, &first, key, value,
1622+
if (encoder_encode_key_value(s, writer, &first, dct, key, value,
16201623
new_newline_indent,
16211624
current_item_separator) < 0)
16221625
goto bail;
@@ -1710,8 +1713,10 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer,
17101713
if (_PyUnicodeWriter_WriteStr(writer, separator) < 0)
17111714
goto bail;
17121715
}
1713-
if (encoder_listencode_obj(s, writer, obj, new_newline_indent))
1716+
if (encoder_listencode_obj(s, writer, obj, new_newline_indent)) {
1717+
_PyErr_FormatNote("when serializing %T item %zd", seq, i);
17141718
goto bail;
1719+
}
17151720
}
17161721
if (ident != NULL) {
17171722
if (PyDict_DelItem(s->markers, ident))

0 commit comments

Comments
 (0)
0