8000 gh-131738: optimize builtin any/all/tuple calls with a generator expr… · python/cpython@2c8f329 · GitHub
[go: up one dir, main page]
Skip to content

Commit 2c8f329

Browse files
iritkatrieliritkatriel
authored
gh-131738: optimize builtin any/all/tuple calls with a generator expression arg (#131737)
1 parent 674dbf3 commit 2c8f329

16 files changed

+199
-38
lines changed

‎Include/internal/pycore_global_objects_fini_generated.h

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_global_strings.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,9 @@ struct _Py_global_strings {
283283
STRUCT_FOR_ID(aggregate_class)
284284
STRUCT_FOR_ID(alias)
285285
STRUCT_FOR_ID(align)
286+
STRUCT_FOR_ID(all)
286287
STRUCT_FOR_ID(allow_code)
288+
STRUCT_FOR_ID(any)
287289
STRUCT_FOR_ID(append)
288290
STRUCT_FOR_ID(arg)
289291
STRUCT_FOR_ID(argdefs)

‎Include/internal/pycore_interp_structs.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ extern "C" {
99

1010
#include "pycore_ast_state.h" // struct ast_state
1111
#include "pycore_llist.h" // struct llist_node
12+
#include "pycore_opcode_utils.h" // NUM_COMMON_CONSTANTS
1213
#include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR
1314
#include "pycore_structs.h" // PyHamtObject
1415
#include "pycore_tstate.h" // _PyThreadStateImpl
@@ -912,6 +913,7 @@ struct _is {
912913
struct ast_state ast;
913914
struct types_state types;
914915
struct callable_cache callable_cache;
916+
PyObject *common_consts[NUM_COMMON_CONSTANTS];
915917
bool jit;
916918
struct _PyExecutorObject *executor_list_head;
917919
size_t trace_run_counter;

‎Include/internal/pycore_magic_number.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,7 @@ Known values:
272272
Python 3.14a6 3617 (Branch monitoring for async for loops)
273273
Python 3.14a6 3618 (Add oparg to END_ASYNC_FOR)
274274
Python 3.14a6 3619 (Renumber RESUME opcode from 149 to 128)
275+
Python 3.14a6 3620 (Optimize bytecode for all/any/tuple called on a genexp)
275276
276277
Python 3.15 will start with 3650
277278
@@ -284,7 +285,7 @@ PC/launcher.c must also be updated.
284285
285286
*/
286287

287-
#define PYC_MAGIC_NUMBER 3619
288+
#define PYC_MAGIC_NUMBER 3620
288289
/* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes
289290
(little-endian) and then appending b'\r\n'. */
290291
#define PYC_MAGIC_NUMBER_TOKEN \

‎Include/internal/pycore_opcode_utils.h

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11-
#include "opcode_ids.h"
12-
1311
#define MAX_REAL_OPCODE 254
1412

1513
#define IS_WITHIN_OPCODE_RANGE(opcode) \
@@ -67,7 +65,10 @@ extern "C" {
6765
/* Values used as the oparg for LOAD_COMMON_CONSTANT */
6866
#define CONSTANT_ASSERTIONERROR 0
6967
#define CONSTANT_NOTIMPLEMENTEDERROR 1
70-
#define NUM_COMMON_CONSTANTS 2
68+
#define CONSTANT_BUILTIN_TUPLE 2
69+
#define CONSTANT_BUILTIN_ALL 3
70+
#define CONSTANT_BUILTIN_ANY 4
71+
#define NUM_COMMON_CONSTANTS 5
7172

7273
/* Values used in the oparg for RESUME */
7374
#define RESUME_AT_FUNC_START 0

‎Include/internal/pycore_runtime_init_generated.h

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_unicodeobject_generated.h

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Lib/opcode.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"HAVE_ARGUMENT", "EXTENDED_ARG", "hasarg", "hasconst", "hasname",
1010
"hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc"]
1111

12+
import builtins
1213
import _opcode
1314
from _opcode import stack_effect
1415

@@ -38,7 +39,8 @@
3839
_intrinsic_1_descs = _opcode.get_intrinsic1_descs()
3940
_intrinsic_2_descs = _opcode.get_intrinsic2_descs()
4041
_special_method_names = _opcode.get_special_method_names()
41-
_common_constants = [AssertionError, NotImplementedError]
42+
_common_constants = [builtins.AssertionError, builtins.NotImplementedError,
43+
builtins.tuple, builtins.all, builtins.any]
4244
_nb_ops = _opcode.get_nb_ops()
4345

4446
hascompare = [opmap["COMPARE_OP"]]

‎Lib/test/test_builtin.py

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,8 @@ def test_all(self):
225225
self.assertEqual(all(x > 42 for x in S), True)
226226
S = [50, 40, 60]
227227
self.assertEqual(all(x > 42 for x in S), False)
228+
S = [50, 40, 60, TestFailingBool()]
229+
self.assertEqual(all(x > 42 for x in S), False)
228230

229231
def test_any(self):
230232
self.assertEqual(any([None, None, None]), False)
@@ -238,9 +240,59 @@ def test_any(self):
238240
self.assertEqual(any([1, TestFailingBool()]), True) # Short-circuit
239241
S = [40, 60, 30]
240242
self.assertEqual(any(x > 42 for x in S), True)
243+
S = [40, 60, 30, TestFailingBool()]
244+
self.assertEqual(any(x > 42 for x in S), True)
241245
S = [10, 20, 30]
242246
self.assertEqual(any(x > 42 for x in S), False)
243247

248+
def test_all_any_tuple_optimization(self):
249+
def f_all():
250+
return all(x-2 for x in [1,2,3])
251+
252+
def f_any():
253+
return any(x-1 for x in [1,2,3])
254+
255+
def f_tuple():
256+
return tuple(2*x for x in [1,2,3])
257+
258+
funcs = [f_all, f_any, f_tuple]
259+
260+
for f in funcs:
261+
# check that generator code object is not duplicated
262+
code_objs = [c for c in f.__code__.co_consts if isinstance(c, type(f.__code__))]
263+
self.assertEqual(len(code_objs), 1)
264+
265+
266+
# check the overriding the builtins works
267+
268+
global all, any, tuple
269+
saved = all, any, tuple
270+
try:
271+
all = lambda x : "all"
272+
any = lambda x : "any"
273+
tuple = lambda x : "tuple"
274+
275+
overridden_outputs = [f() for f in funcs]
276+
finally:
277+
all, any, tuple = saved
278+
279+
self.assertEqual(overridden_outputs, ['all', 'any', 'tuple'])
280+
281+
# Now repeat, overriding the builtins module as well
282+
saved = all, any, tuple
283+
try:
284+
builtins.all = all = lambda x : "all"
285+
builtins.any = any = lambda x : "any"
286+
builtins.tuple = tuple = lambda x : "tuple"
287+
288+
overridden_outputs = [f() for f in funcs]
289+
finally:
290+
all, any, tuple = saved
291+
builtins.all, builtins.any, builtins.tuple = saved
292+
293+
self.assertEqual(overridden_outputs, ['all', 'any', 'tuple'])
294+
295+
244296
def test_ascii(self):
245297
self.assertEqual(ascii(''), '\'\'')
246298
self.assertEqual(ascii(0), '0')

‎Misc/NEWS.d/next/Core_and_Builtins/2025-03-25-20-38-06.gh-issue-131738.eCb0OQ.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Compiler emits optimized code for builtin any/all/tuple calls over a generator expression.

‎Objects/genobject.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
#include "pycore_warnings.h" // _PyErr_WarnUnawaitedCoroutine()
2020

2121

22+
#include "opcode_ids.h" // RESUME, etc
23+
2224
// Forward declarations
2325
static PyObject* gen_close(PyObject *, PyObject *);
2426
static PyObject* async_gen_asend_new(PyAsyncGenObject *, PyObject *);

‎Python/bytecodes.c

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1402,16 +1402,8 @@ dummy_func(
14021402

14031403
inst(LOAD_COMMON_CONSTANT, ( -- value)) {
14041404
// Keep in sync with _common_constants in opcode.py
1405-
// If we ever have more than two constants, use a lookup table
1406-
PyObject *val;
1407-
if (oparg == CONSTANT_ASSERTIONERROR) {
1408-
val = PyExc_AssertionError;
1409-
}
1410-
else {
1411-
assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR);
1412-
val = PyExc_NotImplementedError;
1413-
}
1414-
value = PyStackRef_FromPyObjectImmortal(val);
1405+
assert(oparg < NUM_COMMON_CONSTANTS);
1406+
value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]);
14151407
}
14161408

14171409
inst(LOAD_BUILD_CLASS, ( -- bc)) {

‎Python/codegen.c

Lines changed: 93 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3820,6 +3820,92 @@ update_start_location_to_match_attr(compiler *c, location loc,
38203820
return loc;
38213821
}
38223822

3823+
static int
3824+
maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end)
3825+
{
3826+
asdl_expr_seq *args = e->v.Call.args;
3827+
asdl_keyword_seq *kwds = e->v.Call.keywords;
3828+
expr_ty func = e->v.Call.func;
3829+
3830+
if (! (func->kind == Name_kind &&
3831+
asdl_seq_LEN(args) == 1 &&
3832+
asdl_seq_LEN(kwds) == 0 &&
3833+
asdl_seq_GET(args, 0)->kind == GeneratorExp_kind))
3834+
{
3835+
return 0;
3836+
}
3837+
3838+
location loc = LOC(func);
3839+
3840+
int optimized = 0;
3841+
NEW_JUMP_TARGET_LABEL(c, skip_optimization);
3842+
3843+
int const_oparg = -1;
3844+
PyObject *initial_res = NULL;
3845+
int continue_jump_opcode = -1;
3846+
if (_PyUnicode_EqualToASCIIString(func->v.Name.id, "all")) {
3847+
const_oparg = CONSTANT_BUILTIN_ALL;
3848+
initial_res = Py_True;
3849+
continue_jump_opcode = POP_JUMP_IF_TRUE;
3850+
}
3851+
else if (_PyUnicode_EqualToASCIIString(func->v.Name.id, "any")) {
3852+
const_oparg = CONSTANT_BUILTIN_ANY;
3853+
initial_res = Py_False;
3854+
continue_jump_opcode = POP_JUMP_IF_FALSE;
3855+
}
3856+
else if (_PyUnicode_EqualToASCIIString(func->v.Name.id, "tuple")) {
3857+
const_oparg = CONSTANT_BUILTIN_TUPLE;
3858+
}
3859+
if (const_oparg != -1) {
3860+
ADDOP_I(c, loc, COPY, 1); // the function
3861+
ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, const_oparg);
3862+
ADDOP_COMPARE(c, loc, Is);
3863+
ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, skip_optimization);
3864+
ADDOP(c, loc, POP_TOP);
3865+
3866+
if (const_oparg == CONSTANT_BUILTIN_TUPLE) {
3867+
ADDOP_I(c, loc, BUILD_LIST, 0);
3868+
}
3869+
expr_ty generator_exp = asdl_seq_GET(args, 0);
3870+
VISIT(c, expr, generator_exp);
3871+
3872+
NEW_JUMP_TARGET_LABEL(c, loop);
3873+
NEW_JUMP_TARGET_LABEL(c, cleanup);
3874+
3875+
USE_LABEL(c, loop);
3876+
ADDOP_JUMP(c, loc, FOR_ITER, cleanup);
3877+
if (const_oparg == CONSTANT_BUILTIN_TUPLE) {
3878+
ADDOP_I(c, loc, LIST_APPEND, 2);
3879+
ADDOP_JUMP(c, loc, JUMP, loop);
3880+
}
3881+
else {
3882+
ADDOP(c, loc, TO_BOOL);
3883+
ADDOP_JUMP(c, loc, continue_jump_opcode, loop);
3884+
}
3885+
3886+
ADDOP(c, NO_LOCATION, POP_ITER);
3887+
if (const_oparg != CONSTANT_BUILTIN_TUPLE) {
3888+
ADDOP_LOAD_CONST(c, loc, initial_res == Py_True ? Py_False : Py_True);
3889+
}
3890+
ADDOP_JUMP(c, loc, JUMP, end);
3891+
3892+
USE_LABEL(c, cleanup);
3893+
ADDOP(c, NO_LOCATION, END_FOR);
3894+
ADDOP(c, NO_LOCATION, POP_ITER);
3895+
if (const_oparg == CONSTANT_BUILTIN_TUPLE) {
3896+
ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_LIST_TO_TUPLE);
3897+
}
3898+
else {
3899+
ADDOP_LOAD_CONST(c, loc, initial_res);
3900+
}
3901+
3902+
optimized = 1;
3903+
ADDOP_JUMP(c, loc, JUMP, end);
3904+
}
3905+
USE_LABEL(c, skip_optimization);
3906+
return optimized;
3907+
}
3908+
38233909
// Return 1 if the method call was optimized, 0 if not, and -1 on error.
38243910
static int
38253911
maybe_optimize_method_call(compiler *c, expr_ty e)
@@ -3926,14 +4012,18 @@ codegen_call(compiler *c, expr_ty e)
39264012
if (ret == 1) {
39274013
return SUCCESS;
39284014
}
4015+
NEW_JUMP_TARGET_LABEL(c, skip_normal_call);
39294016
RETURN_IF_ERROR(check_caller(c, e->v.Call.func));
39304017
VISIT(c, expr, e->v.Call.func);
4018+
RETURN_IF_ERROR(maybe_optimize_function_call(c, e, skip_normal_call));
39314019
location loc = LOC(e->v.Call.func);
39324020
ADDOP(c, loc, PUSH_NULL);
39334021
loc = LOC(e);
3934-
return codegen_call_helper(c, loc, 0,
3935-
e->v.Call.args,
3936-
e->v.Call.keywords);
4022+
ret = codegen_call_helper(c, loc, 0,
4023+
e->v.Call.args,
4024+
e->v.Call.keywords);
4025+
USE_LABEL(c, skip_normal_call);
4026+
return ret;
39374027
}
39384028

39394029
static int

‎Python/executor_cases.c.h

Lines changed: 2 additions & 10 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/generated_cases.c.h

Lines changed: 2 additions & 10 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/pylifecycle.c

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -790,6 +790,26 @@ pycore_init_builtins(PyThreadState *tstate)
790790
}
791791
interp->callable_cache.len = len;
792792

793+
PyObject *all = PyDict_GetItemWithError(builtins_dict, &_Py_ID(all));
794+
if (!all) {
795+
goto error;
796+
}
797+
798+
PyObject *any = PyDict_GetItemWithError(builtins_dict, &_Py_ID(any));
799+
if (!any) {
800+
goto error;
801+
}
802+
803+
interp->common_consts[CONSTANT_ASSERTIONERROR] = PyExc_AssertionError;
804+
interp->common_consts[CONSTANT_NOTIMPLEMENTEDERROR] = PyExc_NotImplementedError;
805+
interp->common_consts[CONSTANT_BUILTIN_TUPLE] = (PyObject*)&PyTuple_Type;
806+
interp->common_consts[CONSTANT_BUILTIN_ALL] = all;
807+
interp->common_consts[CONSTANT_BUILTIN_ANY] = any;
808+
809+
for (int i=0; i < NUM_COMMON_CONSTANTS; i++) {
810+
assert(interp->common_consts[i] != NULL);
811+
}
812+
793813
PyObject *list_append = _PyType_Lookup(&PyList_Type, &_Py_ID(append));
794814
if (list_append == NULL) {
795815
goto error;

0 commit comments

Comments
 (0)
0