8000 [3.13] gh-82951: Fix serializing by name in pickle protocols < 4 (GH-… · python/cpython@984f8aa · GitHub
[go: up one dir, main page]
Skip to content

Commit 984f8aa

Browse files
miss-islingtonserhiy-storchaka
miss-islington
and
serhiy-storchaka
authored
[3.13] gh-82951: Fix serializing by name in pickle protocols < 4 (GH-122149) (GH-122264)
Serializing objects with complex __qualname__ (such as unbound methods and nested classes) by name no longer involves serializing parent objects by value in pickle protocols < 4. (cherry picked from commit dc07f65) Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
1 parent 977c799 commit 984f8aa

File tree

4 files changed

+82
-26
lines changed

4 files changed

+82
-26
lines changed

‎Lib/pickle.py

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1110,11 +1110,35 @@ def save_global(self, obj, name=None):
11101110
self.save(module_name)
11111111
self.save(name)
11121112
write(STACK_GLOBAL)
1113-
elif parent is not module:
1114-
self.save_reduce(getattr, (parent, lastname))
1115-
elif self.proto >= 3:
1116-
write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
1117-
bytes(name, "utf-8") + b'\n')
1113+
elif '.' in name:
1114+
# In protocol < 4, objects with multi-part __qualname__
1115+
# are represented as
1116+
# getattr(getattr(..., attrname1), attrname2).
1117+
dotted_path = name.split('.')
1118+
name = dotted_path.pop(0)
1119+
save = self.save
1120+
for attrname in dotted_path:
1121+
save(getattr)
1122+
if self.proto < 2:
1123+
write(MARK)
1124+
self._save_toplevel_by_name(module_name, name)
1125+
for attrname in dotted_path:
1126+
save(attrname)
1127+
if self.proto < 2:
1128+
write(TUPLE)
1129+
else:
1130+
write(TUPLE2)
1131+
write(REDUCE)
1132+
else:
1133+
self._save_toplevel_by_name(module_name, name)
1134+
1135+
self.memoize(obj)
1136+
1137+
def _save_toplevel_by_name(self, module_name, name):
1138+
if self.proto >= 3:
1139+
# Non-ASCII identifiers are supported only with protocols >= 3.
1140+
self.write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
1141+
bytes(name, "utf-8") + b'\n')
11181142
else:
11191143
if self.fix_imports:
11201144
r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
@@ -1124,15 +1148,13 @@ def save_global(self, obj, name=None):
11241148
elif module_name in r_import_mapping:
11251149
module_name = r_import_mapping[module_name]
11261150
try:
1127-
write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
1128-
bytes(name, "ascii") + b'\n')
1151+
self.write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
1152+
bytes(name, "ascii") + b'\n')
11291153
except UnicodeEncodeError:
11301154
raise PicklingError(
11311155
"can't pickle global identifier '%s.%s' using "
11321156
"pickle protocol %i" % (module, name, self.proto)) from None
11331157

1134-
self.memoize(obj)
1135-
11361158
def save_type(self, obj):
11371159
if obj is type(None):
11381160
return self.save_reduce(type, (None,), obj=obj)

‎Lib/test/pickletester.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2818,6 +2818,18 @@ class Recursive:
28182818
self.assertIs(unpickled, Recursive)
28192819
del Recursive.mod # break reference loop
28202820

2821+
def test_recursive_nested_names2(self):
2822+
global Recursive
2823+
class Recursive:
2824+
pass
2825+
Recursive.ref = Recursive
2826+
Recursive.__qualname__ = 'Recursive.ref'
2827+
for proto in range(pickle.HIGHEST_PROTOCOL + 1):
2828+
with self.subTest(proto=proto):
2829+
unpickled = self.loads(self.dumps(Recursive, proto))
2830+
self.assertIs(unpickled, Recursive)
2831+
del Recursive.ref # break reference loop
2832+
28212833
def test_py_methods(self):
28222834
global PyMethodsTest
28232835
class PyMethodsTest:

‎Misc/NEWS.d/next/Library/2024-07-23-09-14-44.gh-issue-82951.-F5p5A.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Serializing objects with complex ``__qualname__`` (such as unbound methods
2+
and nested classes) by name no longer involves serializing parent objects by
3+
value in pickle protocols < 4.

‎Modules/_pickle.c

Lines changed: 36 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3592,7 +3592,6 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj,
35923592
PyObject *module = NULL;
35933593
PyObject *parent = NULL;
35943594
PyObject *dotted_path = NULL;
3595-
PyObject *lastname = NULL;
35963595
PyObject *cls;
35973596
int status = 0;
35983597

@@ -3633,10 +3632,7 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj,
36333632
obj, module_name);
36343633
goto error;
36353634
}
3636-
lastname = Py_NewRef(PyList_GET_ITEM(dotted_path,
3637-
PyList_GET_SIZE(dotted_path) - 1));
36383635
cls = get_deep_attribute(module, dotted_path, &parent);
3639-
Py_CLEAR(dotted_path);
36403636
if (cls == NULL) {
36413637
PyErr_Format(st->PicklingError,
36423638
"Can't pickle %R: attribute lookup %S on %S failed",
@@ -3724,7 +3720,10 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj,
37243720
else {
37253721
gen_global:
37263722
if (parent == module) {
3727-
Py_SETREF(global_name, Py_NewRef(lastname));
3723+
Py_SETREF(global_name,
3724+
Py_NewRef(PyList_GET_ITEM(dotted_path,
3725+
PyList_GET_SIZE(dotted_path) - 1)));
3726+
Py_CLEAR(dotted_path);
37283727
}
37293728
if (self->proto >= 4) {
37303729
const char stack_global_op = STACK_GLOBAL;
@@ -3737,20 +3736,30 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj,
37373736
if (_Pickler_Write(self, &stack_global_op, 1) < 0)
37383737
goto error;
37393738
}
3740-
else if (parent != module) {
3741-
PyObject *reduce_value = Py_BuildValue("(O(OO))",
3742-
st->getattr, parent, lastname);
3743-
if (reduce_value == NULL)
3744-
goto error;
3745-
status = save_reduce(st, self, reduce_value, NULL);
3746-
Py_DECREF(reduce_value);
3747-
if (status < 0)
3748-
goto error;
3749-
}
37503739
else {
37513740
/* Generate a normal global opcode if we are using a pickle
37523741
protocol < 4, or if the object is not registered in the
3753-
extension registry. */
3742+
extension registry.
3743+
3744+
Objects with multi-part __qualname__ are represented as
3745+
getattr(getattr(..., attrname1), attrname2). */
3746+
const char mark_op = MARK;
3747+
const char tupletwo_op = (self->proto < 2) ? TUPLE : TUPLE2;
3748+
const char reduce_op = REDUCE;
3749+
Py_ssize_t i;
3750+
if (dotted_path) {
3751+
if (PyList_GET_SIZE(dotted_path) > 1) {
3752+
Py_SETREF(global_name, Py_NewRef(PyList_GET_ITEM(dotted_path, 0)));
3753+
}
3754+
for (i = 1; i < PyList_GET_SIZE(dotted_path); i++) {
3755+
if (save(st, self, st->getattr, 0) < 0 ||
3756+
(self->proto < 2 && _Pickler_Write(self, &mark_op, 1) < 0))
3757+
{
3758+
goto error;
3759+
}
3760+
}
3761+
}
3762+
37543763
PyObject *encoded;
37553764
PyObject *(*unicode_encoder)(PyObject *);
37563765

@@ -3812,6 +3821,17 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj,
38123821
Py_DECREF(encoded);
38133822
if (_Pickler_Write(self, "\n", 1) < 0)
38143823
goto error;
3824+
3825+
if (dotted_path) {
3826+
for (i = 1; i < PyList_GET_SIZE(dotted_path); i++) {
3827+
if (save(st, self, PyList_GET_ITEM(dotted_path, i), 0) < 0 ||
3828+
_Pickler_Write(self, &tupletwo_op, 1) < 0 ||
3829+
_Pickler_Write(self, &reduce_op, 1) < 0)
3830+
{
3831+
goto error;
3832+
}
3833+
}
3834+
}
38153835
}
38163836
/* Memoize the object. */
38173837
if (memo_put(st, self, obj) < 0)
@@ -3827,7 +3847,6 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj,
38273847
Py_XDECREF(module);
38283848
Py_XDECREF(parent);
38293849
Py_XDECREF(dotted_path);
3830-
Py_XDECREF(lastname);
38313850

38323851
return status;
38333852
}

0 commit comments

Comments
 (0)
0