8000 gh-97933: (PEP 709) inline list/dict/set comprehensions (#101441) · python/cpython@c3b595e · GitHub
[go: up one dir, main page]
Skip to content

Commit c3b595e

Browse files
carljmiritkatrielerlend-aasland
authored
gh-97933: (PEP 709) inline list/dict/set comprehensions (#101441)
Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Co-authored-by: Erlend E. Aasland <erlend.aasland@protonmail.com>
1 parent 0aeda29 commit c3b595e

27 files changed

+1243
-695
lines changed

‎Doc/library/dis.rst

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1196,6 +1196,14 @@ iterations of the loop.
11961196

11971197
.. versionadded:: 3.12
11981198

1199+
.. opcode:: LOAD_FAST_AND_CLEAR (var_num)
1200+
1201+
Pushes a reference to the local ``co_varnames[var_num]`` onto the stack (or
1202+
pushes ``NULL`` onto the stack if the local variable has not been
1203+
initialized) and sets ``co_varnames[var_num]`` to ``NULL``.
1204+
1205+
.. versionadded:: 3.12
1206+
11991207
.. opcode:: STORE_FAST (var_num)
12001208

12011209
Stores ``STACK.pop()`` into the local ``co_varnames[var_num]``.

‎Doc/whatsnew/3.12.rst

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,30 @@ New Features
153153
In Python 3.14, the default will switch to ``'data'``.
154154
(Contributed by Petr Viktorin in :pep:`706`.)
155155

156+
.. _whatsnew312-pep709:
157+
158+
PEP 709: Comprehension inlining
159+
-------------------------------
160+
161+
Dictionary, list, and set comprehensions are now inlined, rather than creating a
162+
new single-use function object for each execution of the comprehension. This
163+
speeds up execution of a comprehension by up to 2x.
164+
165+
Comprehension iteration variables remain isolated; they don't overwrite a
166+
variable of the same name in the outer scope, nor are they visible after the
167+
comprehension. This isolation is now maintained via stack/locals manipulation,
168+
not via separate function scope.
169+
170+
Inlining does result in a few visible behavior changes:
171+
172+
* There is no longer a separate frame for the comprehension in tracebacks,
173+
and tracing/profiling no longer shows the comprehension as a function call.
174+
* Calling :func:`locals` inside a comprehension now includes variables
175+
from outside the comprehension, and no longer includes the synthetic ``.0``
176+
variable for the comprehension "argument".
177+
178+
Contributed by Carl Meyer and Vladimir Matveev in :pep:`709`.
179+
156180
PEP 688: Making the buffer protocol accessible in Python
157181
--------------------------------------------------------
158182

‎Include/internal/pycore_code.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ struct callable_cache {
131131
// Note that these all fit within a byte, as do combinations.
132132
// Later, we will use the smaller numbers to differentiate the different
133133
// kinds of locals (e.g. pos-only arg, varkwargs, local-only).
134+
#define CO_FAST_HIDDEN 0x10
134135
#define CO_FAST_LOCAL 0x20
135136
#define CO_FAST_CELL 0x40
136137
#define CO_FAST_FREE 0x80

‎Include/internal/pycore_compile.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,9 @@ typedef struct {
7070
PyObject *u_varnames; /* local variables */
7171
PyObject *u_cellvars; /* cell variables */
7272
PyObject *u_freevars; /* free variables */
73+
PyObject *u_fasthidden; /* dict; keys are names that are fast-locals only
74+
temporarily within an inlined comprehension. When
75+
value is True, treat as fast-local. */
7376

7477
Py_ssize_t u_argcount; /* number of arguments for block */
7578
Py_ssize_t u_posonlyargcount; /* number of positional only arguments for block */

‎Include/internal/pycore_flowgraph.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ _PyCfgInstruction* _PyCfg_BasicblockLastInstr(const _PyCfgBasicblock *b);
9494
int _PyCfg_OptimizeCodeUnit(_PyCfgBuilder *g, PyObject *consts, PyObject *const_cache,
9595
int code_flags, int nlocals, int nparams, int firstlineno);
9696
int _PyCfg_Stackdepth(_PyCfgBasicblock *entryblock, int code_flags);
97-
void _PyCfg_ConvertExceptionHa 10000 ndlersToNops(_PyCfgBasicblock *entryblock);
97+
void _PyCfg_ConvertPseudoOps(_PyCfgBasicblock *entryblock);
9898
int _PyCfg_ResolveJumps(_PyCfgBuilder *g);
9999

100100

‎Include/internal/pycore_opcode.h

Lines changed: 7 additions & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_symtable.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ typedef struct _symtable_entry {
6464
unsigned ste_needs_class_closure : 1; /* for class scopes, true if a
6565
closure over __class__
6666
should be created */
67+
unsigned ste_comp_inlined : 1; /* true if this comprehension is inlined */
6768
unsigned ste_comp_iter_target : 1; /* true if visiting comprehension target */
6869
int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */
6970
int ste_lineno; /* first line of block */

‎Include/opcode.h

Lines changed: 12 additions & 9 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Lib/importlib/_bootstrap_external.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -442,6 +442,7 @@ def _write_atomic(path, data, mode=0o666):
442442
# Python 3.12b1 3526 (Add instrumentation support)
443443
# Python 3.12b1 3527 (Add LOAD_SUPER_ATTR)
444444
# Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization)
445+
# Python 3.12b1 3529 (Inline list/dict/set comprehensions)
445446

446447
# Python 3.13 will start with 3550
447448

@@ -458,7 +459,7 @@ def _write_atomic(path, data, mode=0o666):
458459
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
459460
# in PC/launcher.c must also be updated.
460461

461-
MAGIC_NUMBER = (3528).to_bytes(2, 'little') + b'\r\n'
462+
MAGIC_NUMBER = (3529).to_bytes(2, 'little') + b'\r\n'
462463

463464
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
464465

‎Lib/opcode.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,8 @@ def pseudo_op(name, op, real_ops):
198198
jrel_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards)
199199
name_op('LOAD_SUPER_ATTR', 141)
200200
def_op('CALL_FUNCTION_EX', 142) # Flags
201+
def_op('LOAD_FAST_AND_CLEAR', 143) # Local variable number
202+
haslocal.append(143)
201203

202204
def_op('EXTENDED_ARG', 144)
203205
EXTENDED_ARG = 144
@@ -268,6 +270,8 @@ def pseudo_op(name, op, real_ops):
268270
pseudo_op('LOAD_ZERO_SUPER_METHOD', 264, ['LOAD_SUPER_ATTR'])
269271
pseudo_op('LOAD_ZERO_SUPER_ATTR', 265, ['LOAD_SUPER_ATTR'])
270272

273+
pseudo_op('STORE_FAST_MAYBE_NULL', 266, ['STORE_FAST'])
274+
271275
MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1
272276

273277
del def_op, name_op, jrel_op, jabs_op, pseudo_op

‎Lib/test/test_compile.py

Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1352,14 +1352,11 @@ def test_multiline_list_comprehension(self):
13521352
and x != 50)]
13531353
""")
13541354
compiled_code, _ = self.check_positions_against_ast(snippet)
1355-
compiled_code = compiled_code.co_consts[0]
13561355
self.assertIsInstance(compiled_code, types.CodeType)
13571356
self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND',
13581357
line=1, end_line=2, column=1, end_column=8, occurrence=1)
13591358
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
13601359
line=1, end_line=2, column=1, end_column=8, occurrence=1)
1361-
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
1362-
line=1, end_line=6, column=0, end_column=32, occurrence=1)
13631360

13641361
def test_multiline_async_list_comprehension(self):
13651362
snippet = textwrap.dedent("""\
@@ -1374,13 +1371,13 @@ async def f():
13741371
compiled_code, _ = self.check_positions_against_ast(snippet)
13751372
g = {}
13761373
eval(compiled_code, g)
1377-
compiled_code = g['f'].__code__.co_consts[1]
1374+
compiled_code = g['f'].__code__
13781375
self.assertIsInstance(compiled_code, types.CodeType)
13791376
self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND',
13801377
line=2, end_line=3, column=5, end_column=12, occurrence=1)
13811378
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
13821379
line=2, end_line=3, column=5, end_column=12, occurrence=1)
1383-
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
1380+
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST',
13841381
line=2, end_line=7, column=4, end_column=36, occurrence=1)
13851382

13861383
def test_multiline_set_comprehension(self):
@@ -1393,14 +1390,11 @@ def test_multiline_set_comprehension(self):
13931390
and x != 50)}
13941391
""")
13951392
compiled_code, _ = self.check_positions_against_ast(snippet)
1396-
compiled_code = compiled_code.co_consts[0]
13971393
self.assertIsInstance(compiled_code, types.CodeType)
13981394
self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD',
13991395
line=1, end_line=2, column=1, end_column=8, occurrence=1)
14001396
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
14011397
line=1, en 10000 d_line=2, column=1, end_column=8, occurrence=1)
1402-
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
1403-
line=1, end_line=6, column=0, end_column=32, occurrence=1)
14041398

14051399
def test_multiline_async_set_comprehension(self):
14061400
snippet = textwrap.dedent("""\
@@ -1415,13 +1409,13 @@ async def f():
14151409
compiled_code, _ = self.check_positions_against_ast(snippet)
14161410
g = {}
14171411
eval(compiled_code, g)
1418-
compiled_code = g['f'].__code__.co_consts[1]
1412+
compiled_code = g['f'].__code__
14191413
self.assertIsInstance(compiled_code, types.CodeType)
14201414
self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD',
14211415
line=2, end_line=3, column=5, end_column=12, occurrence=1)
14221416
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
14231417
line=2, end_line=3, column=5, end_column=12, occurrence=1)
1424-
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
1418+
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST',
14251419
line=2, end_line=7, column=4, end_column=36, occurrence=1)
14261420

14271421
def test_multiline_dict_comprehension(self):
@@ -1434,14 +1428,11 @@ def test_multiline_dict_comprehension(self):
14341428
and x != 50)}
14351429
""")
14361430
compiled_code, _ = self.check_positions_against_ast(snippet)
1437-
compiled_code = compiled_code.co_consts[0]
14381431
self.assertIsInstance(compiled_code, types.CodeType)
14391432
self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD',
14401433
line=1, end_line=2, column=1, end_column=7, occurrence=1)
14411434
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
14421435
line=1, end_line=2, column=1, end_column=7, occurrence=1)
1443-
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
1444-
line=1, end_line=6, column=0, end_column=32, occurrence=1)
14451436

14461437
def test_multiline_async_dict_comprehension(self):
14471438
snippet = textwrap.dedent("""\
@@ -1456,13 +1447,13 @@ async def f():
14561447
compiled_code, _ = self.check_positions_against_ast(snippet)
14571448
g = {}
14581449
eval(compiled_code, g)
1459-
compiled_code = g['f'].__code__.co_consts[1]
1450+
compiled_code = g['f'].__code__
14601451
self.assertIsInstance(compiled_code, types.CodeType)
14611452
self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD',
14621453
line=2, end_line=3, column=5, end_column=11, occurrence=1)
14631454
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
14641455
line=2, end_line=3, column=5, end_column=11, occurrence=1)
1465-
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
1456+
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST',
14661457
line=2, end_line=7, column=4, end_column=36, occurrence=1)
14671458

14681459
def test_matchcase_sequence(self):
@@ -1711,9 +1702,6 @@ def test_column_offset_deduplication(self):
17111702
for source in [
17121703
"lambda: a",
17131704
"(a for b in c)",
1714-
"[a for b in c]",
1715-
"{a for b in c}",
1716-
"{a: b for c in d}",
17171705
]:
17181706
with self.subTest(source):
17191707
code = compile(f"{source}, {source}", "<test>", "eval")

‎Lib/test/test_compiler_assemble.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ def complete_metadata(self, metadata, filename="myfile.py"):
1616
metadata.setdefault(key, key)
1717
for key in ['consts']:
1818
metadata.setdefault(key, [])
19-
for key in ['names', 'varnames', 'cellvars', 'freevars']:
19+
for key in ['names', 'varnames', 'cellvars', 'freevars', 'fasthidden']:
2020
metadata.setdefault(key, {})
2121
for key in ['argcount', 'posonlyargcount', 'kwonlyargcount']:
2222
metadata.setdefault(key, 0)
@@ -33,6 +33,9 @@ def assemble_test(self, insts, metadata, expected):
3333

3434
expected_metadata = {}
3535
for key, value in metadata.items():
36+
if key == "fasthidden":
37+
# not exposed on code object
38+
continue
3639
if isinstance(value, list):
3740
expected_metadata[key] = tuple(value)
3841
elif isinstance(value, dict):

0 commit comments

Comments
 (0)
0