8000 Merge branch 'main' into fetch-restore · python/cpython@df86e20 · GitHub
[go: up one dir, main page]
Skip to content

Commit df86e20

Browse files
iritkatrieliritkatriel
authored
Merge branch 'main' into fetch-restore
2 parents 603922d + b7c1126 commit df86e20

15 files changed

+253
-117
lines changed

‎Doc/library/decimal.rst

Lines changed: 84 additions & 84 deletions
Large diffs are not rendered by default.

‎Lib/test/test_iter.py

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
from test.support import check_free_after_iterating, ALWAYS_EQ, NEVER_EQ
88
import pickle
99
import collections.abc
10+
import functools
11+
import contextlib
12+
import builtins
1013

1114
# Test result of triple loop (too big to inline)
1215
TRIPLETS = [(0, 0, 0), (0, 0, 1), (0, 0, 2),
@@ -91,6 +94,12 @@ def __call__(self):
9194
raise IndexError # Emergency stop
9295
return i
9396

97+
class EmptyIterClass:
98+
def __len__(self):
99+
return 0
100+
def __getitem__(self, i):
101+
raise StopIteration
102+
94103
# Main test suite
95104

96105
class TestCase(unittest.TestCase):
@@ -238,6 +247,78 @@ def test_mutating_seq_class_exhausted_iter(self):
238247
self.assertEqual(list(empit), [5, 6])
239248
self.assertEqual(list(a), [0, 1, 2, 3, 4, 5, 6])
240249

250+
def test_reduce_mutating_builtins_iter(self):
251+
# This is a reproducer of issue #101765
252+
# where iter `__reduce__` calls could lead to a segfault or SystemError
253+
# depending on the order of C argument evaluation, which is undefined
254+
255+
# Backup builtins
256+
builtins_dict = builtins.__dict__
257+
orig = {"iter": iter, "reversed": reversed}
258+
259+
def run(builtin_name, item, sentinel=None):
260+
it = iter(item) if sentinel is None else iter(item, sentinel)
261+
262+
class CustomStr:
263+
def __init__(self, name, iterator):
264+
self.name = name
265+
self.iterator = iterator
266+
def __hash__(self):
267+
return hash(self.name)
268+
def __eq__(self, other):
269+
# Here we exhaust our iterator, possibly changing
270+
# its `it_seq` pointer to NULL
271+
# The `__reduce__` call should correctly get
272+
# the pointers after this call
273+
list(self.iterator)
274+
return other == self.name
275+
276+
# del is required here
277+
# to not prematurely call __eq__ from
278+
# the hash collision with the old key
279+
del builtins_dict[builtin_name]
280+
builtins_dict[CustomStr(builtin_name, it)] = orig[builtin_name]
281+
282+
return it.__reduce__()
283+
284+
types = [
285+
(EmptyIterClass(),),
286+
(bytes(8),),
287+
(bytearray(8),),
288+
((1, 2, 3),),
289+
(lambda: 0, 0),
290+
(tuple[int],) # GenericAlias
291+
]
292+
293+
try:
294+
run_iter = functools.partial(run, "iter")
295+
# The returned value of `__reduce__` should not only be valid
296+
# but also *empty*, as `it` was exhausted during `__eq__`
297+
# i.e "xyz" returns (iter, ("",))
298+
self.assertEqual(run_iter("xyz"), (orig["iter"], ("",)))
299+
self.assertEqual(run_iter([1, 2, 3]), (orig["iter"], ([],)))
300+
301+
# _PyEval_GetBuiltin is also called for `reversed` in a branch of
302+
# listiter_reduce_general
303+
self.assertEqual(
304+
run("reversed", orig["reversed"](list(range(8)))),
305+
(iter, ([],))
306+
)
307+
308+
for case in types:
309+
self.assertEqual(run_iter(*case), (orig["iter"], ((),)))
310+
finally:
311+
# Restore original builtins
312+
for key, func in orig.items():
313+
# need to suppress KeyErrors in case
314+
# a failed test deletes the key without setting anything
315+
with contextlib.suppress(KeyError):
316+
# del is required here
317+
# to not invoke our custom __eq__ from
318+
# the hash collision with the old key
319+
del builtins_dict[key]
320+
builtins_dict[key] = func
321+
241322
# Test a new_style class with __iter__ but no next() method
242323
def test_new_style_iter_class(self):
243324
class IterClass(object):

‎Lib/test/test_tarfile.py

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -225,18 +225,19 @@ def test_add_dir_getmember(self):
225225
self.add_dir_and_getmember('bar')
226226
self.add_dir_and_getmember('a'*101)
227227

228-
@unittest.skipIf(
229-
(hasattr(os, 'getuid') and os.getuid() > 0o777_7777) or
230-
(hasattr(os, 'getgid') and os.getgid() > 0o777_7777),
231-
"uid or gid too high for USTAR format."
232-
)
228+
@unittest.skipUnless(hasattr(os, "getuid") and hasattr(os, "getgid"),
229+
"Missing getuid or getgid implementation")
233230
def add_dir_and_getmember(self, name):
231+
def filter(tarinfo):
232+
tarinfo.uid = tarinfo.gid = 100
233+
return tarinfo
234+
234235
with os_helper.temp_cwd():
235236
with tarfile.open(tmpname, 'w') as tar:
236237
tar.format = tarfile.USTAR_FORMAT
237238
try:
238239
os.mkdir(name)
239-
tar.add(name)
240+
tar.add(name, filter=filter)
240241
finally:
241242
os.rmdir(name)
242243
with tarfile.open(tmpname) as tar:

‎Lib/test/test_zipfile/test_path.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,8 @@ def test_joinpath_constant_time(self):
330330
# Check the file iterated all items
331331
assert entries.count == self.HUGE_ZIPFILE_NUM_ENTRIES
332332

333-
@set_timeout(3)
333+
# timeout disabled due to #102209
334+
# @set_timeout(3)
334335
def test_implied_dirs_performance(self):
335336
data = ['/'.join(string.ascii_lowercase + str(n)) for n in range(10000)]
336337
zipfile.CompleteDirs._implied_dirs(data)

‎Misc/ACKS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1308,6 +1308,7 @@ Jon Oberheide
13081308
Milan Oberkirch
13091309
Pascal Oberndoerfer
13101310
Géry Ogam
1311+
Seonkyo Ok
13111312
Jeffrey Ollie
13121313
Adam Olsen
13131314
Bryan Olson

‎Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix SystemError / segmentation fault in iter ``__reduce__`` when internal access of ``builtins.__dict__`` keys mutates the iter object.

‎Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix deadlock at shutdown when clearing thread states if any finalizer tries to acquire the runtime head lock. Patch by Kumar Aditya.

‎Objects/bytearrayobject.c

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2391,11 +2391,16 @@ PyDoc_STRVAR(length_hint_doc,
23912391
static PyObject *
23922392
bytearrayiter_reduce(bytesiterobject *it, PyObject *Py_UNUSED(ignored))
23932393
{
2394+
PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter));
2395+
2396+
/* _PyEval_GetBuiltin can invoke arbitrary code,
2397+
* call must be before access of iterator pointers.
2398+
* see issue #101765 */
2399+
23942400
if (it->it_seq != NULL) {
2395-
return Py_BuildValue("N(O)n", _PyEval_GetBuiltin(&_Py_ID(iter)),
2396-
it->it_seq, it->it_index);
2401+
return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index);
23972402
} else {
2398-
return Py_BuildValue("N(())", _PyEval_GetBuiltin(&_Py_ID(iter)));
2403+
return Py_BuildValue("N(())", iter);
23992404
}
24002405
}
24012406

‎Objects/bytesobject.c

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3169,11 +3169,16 @@ PyDoc_STRVAR(length_hint_doc,
31693169
static PyObject *
31703170
striter_reduce(striterobject *it, PyObject *Py_UNUSED(ignored))
31713171
{
3172+
PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter));
3173+
3174+
/* _PyEval_GetBuiltin can invoke arbitrary code,
3175+
* call must be before access of iterator pointers.
3176+
* see issue #101765 */
3177+
31723178
if (it->it_seq != NULL) {
3173-
return Py_BuildValue("N(O)n", _PyEval_GetBuiltin(&_Py_ID(iter)),
3174-
it->it_seq, it->it_index);
3179+
return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index);
31753180
} else {
3176-
return Py_BuildValue("N(())", _PyEval_GetBuiltin(&_Py_ID(iter)));
3181+
return Py_BuildValue("N(())", iter);
31773182
}
31783183
}
31793184

‎Objects/genericaliasobject.c

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -877,8 +877,17 @@ ga_iter_clear(PyObject *self) {
877877
static PyObject *
878878
ga_iter_reduce(PyObject *self, PyObject *Py_UNUSED(ignored))
879879
{
880+
PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter));
880881
gaiterobject *gi = (gaiterobject *)self;
881-
return Py_BuildValue("N(O)", _PyEval_GetBuiltin(&_Py_ID(iter)), gi->obj);
882+
883+
/* _PyEval_GetBuiltin can invoke arbitrary code,
884+
* call must be before access of iterator pointers.
885+
* see issue #101765 */
886+
887+
if (gi->obj)
888+
return Py_BuildValue("N(O)", iter, gi->obj);
889+
else
890+
return Py_BuildValue("N(())", iter);
882891
}
883892

884893
static PyMethodDef ga_iter_methods[] = {

‎Objects/iterobject.c

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -102,11 +102,16 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(
102102
static PyObject *
103103
iter_reduce(seqiterobject *it, PyObject *Py_UNUSED(ignored))
104104
{
105+
PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter));
106+
107+
/* _PyEval_GetBuiltin can invoke arbitrary code,
108+
* call must be before access of iterator pointers.
109+
* see issue #101765 */
110+
105111
if (it->it_seq != NULL)
106-
return Py_BuildValue("N(O)n", _PyEval_GetBuiltin(&_Py_ID(iter)),
107-
it->it_seq, it->it_index);
112+
return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index);
108113
else
109-
return Py_BuildValue("N(())", _PyEval_GetBuiltin(&_Py_ID(iter)));
114+
return Py_BuildValue("N(())", iter);
110115
}
111116

112117
PyDoc_STRVAR(reduce_doc, "Return state information for pickling.");
@@ -239,11 +244,16 @@ calliter_iternext(calliterobject *it)
239244
static PyObject *
240245
calliter_reduce(calliterobject *it, PyObject *Py_UNUSED(ignored))
241246
{
247+
PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter));
248+
249+
/* _PyEval_GetBuiltin can invoke arbitrary code,
250+
* call must be before access of iterator pointers.
251+
* see issue #101765 */
252+
242253
if (it->it_callable != NULL && it->it_sentinel != NULL)
243-
return Py_BuildValue("N(OO)", _PyEval_GetBuiltin(&_Py_ID(iter)),
244-
it->it_callable, it->it_sentinel);
254+
return Py_BuildValue("N(OO)", iter, it->it_callable, it->it_sentinel);
245255
else
246-
return Py_BuildValue("N(())", _PyEval_GetBuiltin(&_Py_ID(iter)));
256+
return Py_BuildValue("N(())", iter);
247257
}
248258

249259
static PyMethodDef calliter_methods[] = {

‎Objects/listobject.c

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3444,18 +3444,22 @@ listiter_reduce_general(void *_it, int forward)
34443444
{
34453445
PyObject *list;
34463446

3447+
/* _PyEval_GetBuiltin can invoke arbitrary code,
3448+
* call must be before access of iterator pointers.
3449+
* see issue #101765 */
3450+
34473451
/* the objects are not the same, index is of different types! */
34483452
if (forward) {
3453+
PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter));
34493454
_PyListIterObject *it = (_PyListIterObject *)_it;
34503455
if (it->it_seq) {
3451-
return Py_BuildValue("N(O)n", _PyEval_GetBuiltin(&_Py_ID(iter)),
3452-
it->it_seq, it->it_index);
3456+
return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index);
34533457
}
34543458
} else {
3459+
PyObject *reversed = _PyEval_GetBuiltin(&_Py_ID(reversed));
34553460
listreviterobject *it = (listreviterobject *)_it;
34563461
if (it->it_seq) {
3457-
return Py_BuildValue("N(O)n", _PyEval_GetBuiltin(&_Py_ID(reversed)),
3458-
it->it_seq, it->it_index);
3462+
return Py_BuildValue("N(O)n", reversed, it->it_seq, it->it_index);
34593463
}
34603464
}
34613465
/* empty iterator, create an empty list */

‎Objects/tupleobject.c

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1048,11 +1048,16 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(
10481048
static PyObject *
10491049
tupleiter_reduce(_PyTupleIterObject *it, PyObject *Py_UNUSED(ignored))
10501050
{
1051+
PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter));
1052+
1053+
/* _PyEval_GetBuiltin can invoke arbitrary code,
1054+
* call must be before access of iterator pointers.
1055+
* see issue #101765 */
1056+
10511057
if (it->it_seq)
1052-
return Py_BuildValue("N(O)n", _PyEval_GetBuiltin(&_Py_ID(iter)),
1053-
it->it_seq, it->it_index);
1058+
return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index);
10541059
else
1055-
return Py_BuildValue("N(())", _PyEval_GetBuiltin(&_Py_ID(iter)));
1060+
return Py_BuildValue("N(())", iter);
10561061
}
10571062

10581063
static PyObject *

‎Objects/unicodeobject.c

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14784,14 +14784,19 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(
1478414784
static PyObject *
1478514785
unicodeiter_reduce(unicodeiterobject *it, PyObject *Py_UNUSED(ignored))
1478614786
{
14787+
PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter));
14788+
14789+
/* _PyEval_GetBuiltin can invoke arbitrary code,
14790+
* call must be before access of iterator pointers.
14791+
* see issue #101765 */
14792+
1478714793
if (it->it_seq != NULL) {
14788-
return Py_BuildValue("N(O)n", _PyEval_GetBuiltin(&_Py_ID(iter)),
14789-
it->it_seq, it->it_index);
14794+
return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index);
1479014795
} else {
1479114796
PyObject *u = unicode_new_empty();
1479214797
if (u == NULL)
1479314798
return NULL;
14794-
return Py_BuildValue("N(N)", _PyEval_GetBuiltin(&_Py_ID(iter)), u);
14799+
return Py_BuildValue("N(N)", iter, u);
1479514800
}
1479614801
}
1479714802

‎Python/pystate.c

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -754,12 +754,19 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)
754754
_PyErr_Clear(tstate);
755755
}
756756

757+
// Clear the current/main thread state last.
757758
HEAD_LOCK(runtime);
758-
// XXX Clear the current/main thread state last.
759-
for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) {
759+
PyThreadState *p = interp->threads.head;
760+
HEAD_UNLOCK(runtime);
761+
while (p != NULL) {
762+
// See https://github.com/python/cpython/issues/102126
763+
// Must be called without HEAD_LOCK held as it can deadlock
764+
// if any finalizer tries to acquire that lock.
760765
PyThreadState_Clear(p);
766+
HEAD_LOCK(runtime);
767+
p = p->next;
768+
HEAD_UNLOCK(runtime);
761769
}
762-
HEAD_UNLOCK(runtime);
763770

764771
/* It is possible that any of the objects below have a finalizer
765772
that runs Python code or otherwise relies on a thread state

0 commit comments

Comments
 (0)
0