8000 [3.14] GH-135171: Fix generator expressions one last time (hopefully)… · python/cpython@a5274cd · GitHub
[go: up one dir, main page]
Skip to content

Commit a5274cd

Browse files
markshannonmarkshannon
authored
[3.14] GH-135171: Fix generator expressions one last time (hopefully) (GH-135225)
* Add NULL check to FOR_ITER * Move GET_ITER back to genexpr creation
1 parent 1497866 commit a5274cd

File tree

8 files changed

+92
-19
lines changed

8 files changed

+92
-19
lines changed

‎Lib/test/test_dis.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,7 @@ def bug1333982(x=[]):
204204
LOAD_CONST 1 (<code object <genexpr> at 0x..., file "%s", line %d>)
205205
MAKE_FUNCTION
206206
LOAD_FAST_BORROW 0 (x)
207+
GET_ITER
207208
CALL 0
208209
209210
%3d LOAD_SMALL_INT 1
@@ -832,6 +833,7 @@ def foo(x):
832833
MAKE_FUNCTION
833834
SET_FUNCTION_ATTRIBUTE 8 (closure)
834835
LOAD_DEREF 1 (y)
836+
GET_ITER
835837
CALL 0
836838
CALL 1
837839
RETURN_VALUE
@@ -851,8 +853,7 @@ def foo(x):
851853
%4d RETURN_GENERATOR
852854
POP_TOP
853855
L1: RESUME 0
854-
LOAD_FAST_BORROW 0 (.0)
855-
GET_ITER
856+
LOAD_FAST 0 (.0)
856857
L2: FOR_ITER 14 (to L3)
857858
STORE_FAST 1 (z)
858859
LOAD_DEREF 2 (x)

‎Lib/test/test_generators.py

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -318,21 +318,26 @@ def gen(it):
318318
yield x
319319
return gen(range(10))
320320

321-
def process_tests(self, get_generator):
321+
def process_tests(self, get_generator, is_expr):
322+
err_iterator = "'.*' object is not an iterator"
323+
err_iterable = "'.*' object is not iterable"
322324
for obj in self.iterables:
323325
g_obj = get_generator(obj)
324326
with self.subTest(g_obj=g_obj, obj=obj):
325-
self.assertListEqual(list(g_obj), list(obj))
327+
if is_expr:
328+
self.assertRaisesRegex(TypeError, err_iterator, list, g_obj)
329+
else:
330+
self.assertListEqual(list(g_obj), list(obj))
326331

327332
g_iter = get_generator(iter(obj))
328333
with self.subTest(g_iter=g_iter, obj=obj):
329334
self.assertListEqual(list(g_iter), list(obj))
330335

331-
err_regex = "'.*' object is not iterable"
332336
for obj in self.non_iterables:
333337
g_obj = get_generator(obj)
334338
with self.subTest(g_obj=g_obj):
335-
self.assertRaisesRegex(TypeError, err_regex, list, g_obj)
339+
err = err_iterator if is_expr else err_iterable
340+
self.assertRaisesRegex(TypeError, err, list, g_obj)
336341

337342
def test_modify_f_locals(self):
338343
def modify_f_locals(g, local, obj):
@@ -345,8 +350,8 @@ def get_generator_genexpr(obj):
345350
def get_generator_genfunc(obj):
346351
return modify_f_locals(self.genfunc(), 'it', obj)
347352

348-
self.process_tests(get_generator_genexpr)
349-
self.process_tests(get_generator_genfunc)
353+
self.process_tests(get_generator_genexpr, True)
354+
self.process_tests(get_generator_genfunc, False)
350355

351356
def test_new_gen_from_gi_code(self):
352357
def new_gen_from_gi_code(g, obj):
@@ -359,8 +364,8 @@ def get_generator_genexpr(obj):
359364
def get_generator_genfunc(obj):
360365
return new_gen_from_gi_code(self.genfunc(), obj)
361366

362-
self.process_tests(get_generator_genexpr)
363-
self.process_tests(get_generator_genfunc)
367+
self.process_tests(get_generator_genexpr, True)
368+
self.process_tests(get_generator_genfunc, False)
364369

365370

366371
class ExceptionTest(unittest.TestCase):

‎Lib/test/test_genexps.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,14 @@
131131
>>> list(g)
132132
[1, 9, 25, 49, 81]
133133
134+
Verify that the outermost for-expression makes an immediate check
135+
for iterability
136+
>>> (i for i in 6)
137+
Traceback (most recent call last):
138+
File "<pyshell#4>", line 1, in -toplevel-
139+
(i for i in 6)
140+
TypeError: 'int' object is not iterable
141+
134142
Verify late binding for the innermost for-expression
135143
136144
>>> g = ((i,j) for i in range(3) for j in range(x))

‎Misc/NEWS.d/next/Core_and_Builtins/2025-06-06-18-57-30.gh-issue-135171.0YtLq6.rst

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
Reverts the behavior of generator expressions when created with a
2+
non-iterable to the pre-3.13 behavior of raising a TypeError. It is no
3+
longer possible to cause a crash in the debugger by altering the generator
4+
expression's local variables. This is achieved by moving the ``GET_ITER``
5+
instruction back to the creation of the generator expression and adding an
6+
additional check to ``FOR_ITER``.

‎Python/bytecodes.c

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3155,7 +3155,14 @@ dummy_func(
31553155
replaced op(_FOR_ITER, (iter -- iter, next)) {
31563156
/* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
31573157
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
3158-
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
3158+
iternextfunc func = Py_TYPE(iter_o)->tp_iternext;
3159+
if (func == NULL) {
3160+
_PyErr_Format(tstate, PyExc_TypeError,
3161+
"'%.100s' object is not an iterator",
3162+
Py_TYPE(iter_o)->tp_name);
3163+
ERROR_NO_POP();
3164+
}
3165+
PyObject *next_o = func(iter_o);
31593166
if (next_o == NULL) {
31603167
if (_PyErr_Occurred(tstate)) {
31613168
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
@@ -3179,7 +3186,14 @@ dummy_func(
31793186
op(_FOR_ITER_TIER_TWO, (iter -- iter, next)) {
31803187
/* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
31813188
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
3182-
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
3189+
iternextfunc func = Py_TYPE(iter_o)->tp_iternext;
3190+
if (func == NULL) {
3191+
_PyErr_Format(tstate, PyExc_TypeError,
3192+
"'%.100s' object is not an iterator",
3193+
Py_TYPE(iter_o)->tp_name);
3194+
ERROR_NO_POP();
3195+
}
3196+
PyObject *next_o = func(iter_o);
31833197
if (next_o == NULL) {
31843198
if (_PyErr_Occurred(tstate)) {
31853199
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
@@ -3202,7 +3216,14 @@ dummy_func(
32023216

32033217
inst(INSTRUMENTED_FOR_ITER, (unused/1, iter -- iter, next)) {
32043218
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
3205-
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
3219+
iternextfunc func = Py_TYPE(iter_o)->tp_iternext;
3220+
if (func == NULL) {
3221+
_PyErr_Format(tstate, PyExc_TypeError,
3222+
"'%.100s' object is not an iterator",
3223+
Py_TYPE(iter_o)->tp_name);
3224+
ERROR_NO_POP();
3225+
}
3226+
PyObject *next_o = func(iter_o);
32063227
if (next_o != NULL) {
32073228
next = PyStackRef_FromPyObjectSteal(next_o);
32083229
INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);

‎Python/codegen.c

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4411,7 +4411,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc,
44114411

44124412
comprehension_ty gen = (comprehension_ty)asdl_seq_GET(generators,
44134413
gen_index);
4414-
4414+
int is_outer_genexpr = gen_index == 0 && type == COMP_GENEXP;
44154415
if (!iter_on_stack) {
44164416
if (gen_index == 0) {
44174417
assert(METADATA(c)->u_argcount == 1);
@@ -4442,14 +4442,15 @@ codegen_sync_comprehension_generator(compiler *c, location loc,
44424442
}
44434443
if (IS_JUMP_TARGET_LABEL(start)) {
44444444
VISIT(c, expr, gen->iter);
4445-
ADDOP(c, LOC(gen->iter), GET_ITER);
44464445
}
44474446
}
44484447
}
44494448

44504449
if (IS_JUMP_TARGET_LABEL(start)) {
44514450
depth++;
4452-
ADDOP(c, LOC(gen->iter), GET_ITER);
4451+
if (!is_outer_genexpr) {
4452+
ADDOP(c, LOC(gen->iter), GET_ITER);
4453+
}
44534454
USE_LABEL(c, start);
44544455
ADDOP_JUMP(c, LOC(gen->iter), FOR_ITER, anchor);
44554456
}
@@ -4775,6 +4776,7 @@ codegen_comprehension(compiler *c, expr_ty e, int type,
47754776
location loc = LOC(e);
47764777

47774778
outermost = (comprehension_ty) asdl_seq_GET(generators, 0);
4779+
int is_sync_genexpr = type == COMP_GENEXP && !outermost->is_async;
47784780
if (is_inlined) {
47794781
VISIT(c, expr, outermost->iter);
47804782
if (push_inlined_comprehension_state(c, loc, entry, &inline_state)) {
@@ -4851,6 +4853,9 @@ codegen_comprehension(compiler *c, expr_ty e, int type,
48514853
Py_CLEAR(co);
48524854

48534855
VISIT(c, expr, outermost->iter);
4856+
if (is_sync_genexpr) {
4857+
ADDOP(c, loc, GET_ITER);
4858+
}
48544859
ADDOP_I(c, loc, CALL, 0);
48554860

48564861
if (is_async_comprehension && type != COMP_GENEXP) {

‎Python/executor_cases.c.h

Lines changed: 10 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/generated_cases.c.h

Lines changed: 20 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)
0