diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 622829fccdd0dd..e525d2cecefe32 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -268,6 +268,7 @@ int _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNI int _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache); int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr); int _Py_Specialize_BinaryAdd(PyObject *left, PyObject *right, _Py_CODEUNIT *instr); +int _Py_Specialize_InplaceAdd(PyObject *left, PyObject *right, _Py_CODEUNIT *instr); int _Py_Specialize_BinaryMultiply(PyObject *left, PyObject *right, _Py_CODEUNIT *instr); int _Py_Specialize_CallFunction(PyObject *callable, _Py_CODEUNIT *instr, int nargs, SpecializedCacheEntry *cache, PyObject *builtins); diff --git a/Include/opcode.h b/Include/opcode.h index f8c02b840e0521..c4c4934191fa07 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -141,42 +141,47 @@ extern "C" { #define BINARY_ADD_FLOAT 13 #define BINARY_ADD_UNICODE 14 #define BINARY_ADD_UNICODE_INPLACE_FAST 18 -#define BINARY_MULTIPLY_ADAPTIVE 21 -#define BINARY_MULTIPLY_INT 36 -#define BINARY_MULTIPLY_FLOAT 38 -#define BINARY_SUBSCR_ADAPTIVE 39 -#define BINARY_SUBSCR_LIST_INT 40 -#define BINARY_SUBSCR_TUPLE_INT 41 -#define BINARY_SUBSCR_DICT 42 -#define CALL_FUNCTION_ADAPTIVE 43 -#define CALL_FUNCTION_BUILTIN_O 44 -#define CALL_FUNCTION_BUILTIN_FAST 45 -#define CALL_FUNCTION_LEN 46 -#define CALL_FUNCTION_ISINSTANCE 47 -#define CALL_FUNCTION_PY_SIMPLE 48 -#define JUMP_ABSOLUTE_QUICK 58 -#define LOAD_ATTR_ADAPTIVE 80 -#define LOAD_ATTR_INSTANCE_VALUE 81 -#define LOAD_ATTR_WITH_HINT 87 -#define LOAD_ATTR_SLOT 88 -#define LOAD_ATTR_MODULE 120 -#define LOAD_GLOBAL_ADAPTIVE 122 -#define LOAD_GLOBAL_MODULE 123 -#define LOAD_GLOBAL_BUILTIN 127 -#define LOAD_METHOD_ADAPTIVE 128 -#define LOAD_METHOD_CACHED 134 -#define LOAD_METHOD_CLASS 140 -#define LOAD_METHOD_MODULE 143 -#define LOAD_METHOD_NO_DICT 149 -#define STORE_ATTR_ADAPTIVE 150 -#define STORE_ATTR_INSTANCE_VALUE 151 -#define STORE_ATTR_SLOT 153 -#define STORE_ATTR_WITH_HINT 154 -#define LOAD_FAST__LOAD_FAST 158 -#define STORE_FAST__LOAD_FAST 159 -#define LOAD_FAST__LOAD_CONST 167 -#define LOAD_CONST__LOAD_FAST 168 -#define STORE_FAST__STORE_FAST 169 +#define INPLACE_ADD_ADAPTIVE 21 +#define INPLACE_ADD_INT 36 +#define INPLACE_ADD_FLOAT 38 +#define INPLACE_ADD_UNICODE 39 +#define INPLACE_ADD_UNICODE_FAST 40 +#define BINARY_MULTIPLY_ADAPTIVE 41 +#define BINARY_MULTIPLY_INT 42 +#define BINARY_MULTIPLY_FLOAT 43 +#define BINARY_SUBSCR_ADAPTIVE 44 +#define BINARY_SUBSCR_LIST_INT 45 +#define BINARY_SUBSCR_TUPLE_INT 46 +#define BINARY_SUBSCR_DICT 47 +#define CALL_FUNCTION_ADAPTIVE 48 +#define CALL_FUNCTION_BUILTIN_O 58 +#define CALL_FUNCTION_BUILTIN_FAST 80 +#define CALL_FUNCTION_LEN 81 +#define CALL_FUNCTION_ISINSTANCE 87 +#define CALL_FUNCTION_PY_SIMPLE 88 +#define JUMP_ABSOLUTE_QUICK 120 +#define LOAD_ATTR_ADAPTIVE 122 +#define LOAD_ATTR_INSTANCE_VALUE 123 +#define LOAD_ATTR_WITH_HINT 127 +#define LOAD_ATTR_SLOT 128 +#define LOAD_ATTR_MODULE 134 +#define LOAD_GLOBAL_ADAPTIVE 140 +#define LOAD_GLOBAL_MODULE 143 +#define LOAD_GLOBAL_BUILTIN 149 +#define LOAD_METHOD_ADAPTIVE 150 +#define LOAD_METHOD_CACHED 151 +#define LOAD_METHOD_CLASS 153 +#define LOAD_METHOD_MODULE 154 +#define LOAD_METHOD_NO_DICT 158 +#define STORE_ATTR_ADAPTIVE 159 +#define STORE_ATTR_INSTANCE_VALUE 167 +#define STORE_ATTR_SLOT 168 +#define STORE_ATTR_WITH_HINT 169 +#define LOAD_FAST__LOAD_FAST 170 +#define STORE_FAST__LOAD_FAST 171 +#define LOAD_FAST__LOAD_CONST 172 +#define LOAD_CONST__LOAD_FAST 173 +#define STORE_FAST__STORE_FAST 174 #define DO_TRACING 255 #ifdef NEED_OPCODE_JUMP_TABLES static uint32_t _PyOpcode_RelativeJump[8] = { diff --git a/Lib/opcode.py b/Lib/opcode.py index 5377ec32bf1535..49fd35887268db 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -225,6 +225,11 @@ def jabs_op(name, op): "BINARY_ADD_FLOAT", "BINARY_ADD_UNICODE", "BINARY_ADD_UNICODE_INPLACE_FAST", + "INPLACE_ADD_ADAPTIVE", + "INPLACE_ADD_INT", + "INPLACE_ADD_FLOAT", + "INPLACE_ADD_UNICODE", + "INPLACE_ADD_UNICODE_FAST", "BINARY_MULTIPLY_ADAPTIVE", "BINARY_MULTIPLY_INT", "BINARY_MULTIPLY_FLOAT", diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-18-09-54-55.bpo-45508.6sPZsS.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-18-09-54-55.bpo-45508.6sPZsS.rst new file mode 100644 index 00000000000000..abbc7f6007b054 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-18-09-54-55.bpo-45508.6sPZsS.rst @@ -0,0 +1 @@ +Specialized the ``INPLACE_ADD`` opcode to match the specializations of ``BINARY_ADD`` (see PEP 659 for details). \ No newline at end of file diff --git a/Python/ceval.c b/Python/ceval.c index adc7b536247b2e..57319c94297ab8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -92,8 +92,6 @@ static PyObject * import_from(PyThreadState *, PyObject *, PyObject *); static int import_all_from(PyThreadState *, PyObject *, PyObject *); static void format_exc_check_arg(PyThreadState *, PyObject *, const char *, PyObject *); static void format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg); -static PyObject * unicode_concatenate(PyThreadState *, PyObject *, PyObject *, - InterpreterFrame *, const _Py_CODEUNIT *); static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg); static void format_kwargs_error(PyThreadState *, PyObject *func, PyObject *kwargs); static void format_awaitable_error(PyThreadState *, PyTypeObject *, int, int); @@ -2437,21 +2435,109 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr } TARGET(INPLACE_ADD) { + PREDICTED(INPLACE_ADD); + STAT_INC(INPLACE_ADD, unquickened); PyObject *right = POP(); PyObject *left = TOP(); - PyObject *sum; - if (PyUnicode_CheckExact(left) && PyUnicode_CheckExact(right)) { - sum = unicode_concatenate(tstate, left, right, frame, next_instr); - /* unicode_concatenate consumed the ref to left */ + PyObject *sum = PyNumber_InPlaceAdd(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(sum); + if (sum == NULL) + goto error; + DISPATCH(); + } + + TARGET(INPLACE_ADD_ADAPTIVE) { + if (oparg == 0) { + PyObject *left = SECOND(); + PyObject *right = TOP(); + next_instr--; + if (_Py_Specialize_InplaceAdd(left, right, next_instr) < 0) { + goto error; + } + DISPATCH(); } else { - sum = PyNumber_InPlaceAdd(left, right); - Py_DECREF(left); + STAT_INC(INPLACE_ADD, deferred); + UPDATE_PREV_INSTR_OPARG(next_instr, oparg - 1); + STAT_DEC(INPLACE_ADD, unquickened); + JUMP_TO_INSTRUCTION(INPLACE_ADD); } + } + + TARGET(INPLACE_ADD_INT) { + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyLong_CheckExact(left), INPLACE_ADD); + DEOPT_IF(!PyLong_CheckExact(right), INPLACE_ADD); + STAT_INC(INPLACE_ADD, hit); + PyObject *sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); + SET_SECOND(sum); + Py_DECREF(left); Py_DECREF(right); - SET_TOP(sum); - if (sum == NULL) + STACK_SHRINK(1); + if (sum == NULL) { goto error; + } + DISPATCH(); + } + + TARGET(INPLACE_ADD_FLOAT) { + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyFloat_CheckExact(left), INPLACE_ADD); + DEOPT_IF(!PyFloat_CheckExact(right), INPLACE_ADD); + STAT_INC(INPLACE_ADD, hit); + double dsum = ((PyFloatObject *)left)->ob_fval + + ((PyFloatObject *)right)->ob_fval; + PyObject *sum = PyFloat_FromDouble(dsum); + SET_SECOND(sum); + Py_DECREF(left); + Py_DECREF(right); + STACK_SHRINK(1); + if (sum == NULL) { + goto error; + } + DISPATCH(); + } + + TARGET(INPLACE_ADD_UNICODE) { + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyUnicode_CheckExact(left), INPLACE_ADD); + DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), INPLACE_ADD); + STAT_INC(BINARY_ADD, hit); + PyObject *res = PyUnicode_Concat(left, right); + STACK_SHRINK(1); + SET_TOP(res); + Py_DECREF(left); + Py_DECREF(right); + if (TOP() == NULL) { + goto error; + } + DISPATCH(); + } + + TARGET(INPLACE_ADD_UNICODE_FAST) { + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyUnicode_CheckExact(left), INPLACE_ADD); + DEOPT_IF(!PyUnicode_CheckExact(right), INPLACE_ADD); + DEOPT_IF(Py_REFCNT(left) != 2, INPLACE_ADD); + int next_oparg = _Py_OPARG(*next_instr); + assert(_Py_OPCODE(*next_instr) == STORE_FAST); + PyObject *var = GETLOCAL(next_oparg); + DEOPT_IF(var != left, INPLACE_ADD); + STAT_INC(INPLACE_ADD, hit); + GETLOCAL(next_oparg) = NULL; + Py_DECREF(left); + STACK_SHRINK(1); + PyUnicode_Append(&TOP(), right); + Py_DECREF(right); + if (TOP() == NULL) { + goto error; + } DISPATCH(); } @@ -5134,6 +5220,7 @@ MISS_WITH_CACHE(LOAD_METHOD) MISS_WITH_CACHE(CALL_FUNCTION) MISS_WITH_OPARG_COUNTER(BINARY_SUBSCR) MISS_WITH_OPARG_COUNTER(BINARY_ADD) +MISS_WITH_OPARG_COUNTER(INPLACE_ADD) MISS_WITH_OPARG_COUNTER(BINARY_MULTIPLY) binary_subscr_dict_error: @@ -7149,60 +7236,6 @@ format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int prevprevop } } -static PyObject * -unicode_concatenate(PyThreadState *tstate, PyObject *v, PyObject *w, - InterpreterFrame *frame, const _Py_CODEUNIT *next_instr) -{ - PyObject *res; - if (Py_REFCNT(v) == 2) { - /* In the common case, there are 2 references to the value - * stored in 'variable' when the += is performed: one on the - * value stack (in 'v') and one still stored in the - * 'variable'. We try to delete the variable now to reduce - * the refcnt to 1. - */ - int opcode, oparg; - NEXTOPARG(); - next_instr++; - switch (opcode) { - case STORE_FAST: - { - if (GETLOCAL(oparg) == v) - SETLOCAL(oparg, NULL); - break; - } - case STORE_DEREF: - { - PyObject *c = _PyFrame_GetLocalsArray(frame)[oparg]; - if (PyCell_GET(c) == v) { - PyCell_SET(c, NULL); - Py_DECREF(v); - } - break; - } - case STORE_NAME: - { - PyObject *names = frame->f_code->co_names; - PyObject *name = GETITEM(names, oparg); - PyObject *locals = frame->f_locals; - if (locals && PyDict_CheckExact(locals)) { - PyObject *w = PyDict_GetItemWithError(locals, name); - if ((w == v && PyDict_DelItem(locals, name) != 0) || - (w == NULL && _PyErr_Occurred(tstate))) - { - Py_DECREF(v); - return NULL; - } - } - break; - } - } - } - res = v; - PyUnicode_Append(&res, w); - return res; -} - #ifdef DYNAMIC_EXECUTION_PROFILE static PyObject * diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 5c7d3ad544e563..78698efa11a644 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -20,7 +20,7 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_ADD_UNICODE_INPLACE_FAST, &&TARGET_BINARY_POWER, &&TARGET_BINARY_MULTIPLY, - &&TARGET_BINARY_MULTIPLY_ADAPTIVE, + &&TARGET_INPLACE_ADD_ADAPTIVE, &&TARGET_BINARY_MODULO, &&TARGET_BINARY_ADD, &&TARGET_BINARY_SUBTRACT, @@ -35,19 +35,19 @@ static void *opcode_targets[256] = { &&TARGET_MATCH_KEYS, &&TARGET_COPY_DICT_WITHOUT_KEYS, &&TARGET_PUSH_EXC_INFO, - &&TARGET_BINARY_MULTIPLY_INT, + &&TARGET_INPLACE_ADD_INT, &&TARGET_POP_EXCEPT_AND_RERAISE, + &&TARGET_INPLACE_ADD_FLOAT, + &&TARGET_INPLACE_ADD_UNICODE, + &&TARGET_INPLACE_ADD_UNICODE_FAST, + &&TARGET_BINARY_MULTIPLY_ADAPTIVE, + &&TARGET_BINARY_MULTIPLY_INT, &&TARGET_BINARY_MULTIPLY_FLOAT, &&TARGET_BINARY_SUBSCR_ADAPTIVE, &&TARGET_BINARY_SUBSCR_LIST_INT, &&TARGET_BINARY_SUBSCR_TUPLE_INT, &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_CALL_FUNCTION_ADAPTIVE, - &&TARGET_CALL_FUNCTION_BUILTIN_O, - &&TARGET_CALL_FUNCTION_BUILTIN_FAST, - &&TARGET_CALL_FUNCTION_LEN, - &&TARGET_CALL_FUNCTION_ISINSTANCE, - &&TARGET_CALL_FUNCTION_PY_SIMPLE, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -57,7 +57,7 @@ static void *opcode_targets[256] = { &&TARGET_INPLACE_ADD, &&TARGET_INPLACE_SUBTRACT, &&TARGET_INPLACE_MULTIPLY, - &&TARGET_JUMP_ABSOLUTE_QUICK, + &&TARGET_CALL_FUNCTION_BUILTIN_O, &&TARGET_INPLACE_MODULO, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, @@ -79,15 +79,15 @@ static void *opcode_targets[256] = { &&TARGET_INPLACE_AND, &&TARGET_INPLACE_XOR, &&TARGET_INPLACE_OR, - &&TARGET_LOAD_ATTR_ADAPTIVE, - &&TARGET_LOAD_ATTR_INSTANCE_VALUE, + &&TARGET_CALL_FUNCTION_BUILTIN_FAST, + &&TARGET_CALL_FUNCTION_LEN, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, &&TARGET_YIELD_VALUE, - &&TARGET_LOAD_ATTR_WITH_HINT, - &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_CALL_FUNCTION_ISINSTANCE, + &&TARGET_CALL_FUNCTION_PY_SIMPLE, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, &&TARGET_DELETE_NAME, @@ -119,46 +119,46 @@ static void *opcode_targets[256] = { &&TARGET_IS_OP, &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, - &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_JUMP_ABSOLUTE_QUICK, &&TARGET_JUMP_IF_NOT_EXC_MATCH, - &&TARGET_LOAD_GLOBAL_ADAPTIVE, - &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_LOAD_ATTR_ADAPTIVE, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_FAST, &&TARGET_STORE_FAST, &&TARGET_DELETE_FAST, - &&TARGET_LOAD_GLOBAL_BUILTIN, - &&TARGET_LOAD_METHOD_ADAPTIVE, + &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_ATTR_SLOT, &&TARGET_GEN_START, &&TARGET_RAISE_VARARGS, &&TARGET_CALL_FUNCTION, &&TARGET_MAKE_FUNCTION, &&TARGET_BUILD_SLICE, - &&TARGET_LOAD_METHOD_CACHED, + &&TARGET_LOAD_ATTR_MODULE, &&TARGET_MAKE_CELL, &&TARGET_LOAD_CLOSURE, &&TARGET_LOAD_DEREF, &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, - &&TARGET_LOAD_METHOD_CLASS, + &&TARGET_LOAD_GLOBAL_ADAPTIVE, &&TARGET_CALL_FUNCTION_KW, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_METHOD_MODULE, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, &&TARGET_MAP_ADD, &&TARGET_LOAD_CLASSDEREF, - &&TARGET_LOAD_METHOD_NO_DICT, - &&TARGET_STORE_ATTR_ADAPTIVE, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_METHOD_ADAPTIVE, + &&TARGET_LOAD_METHOD_CACHED, &&TARGET_MATCH_CLASS, - &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_LOAD_METHOD_CLASS, + &&TARGET_LOAD_METHOD_MODULE, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_LOAD_FAST__LOAD_FAST, - &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_LOAD_METHOD_NO_DICT, + &&TARGET_STORE_ATTR_ADAPTIVE, &&TARGET_LOAD_METHOD, &&TARGET_CALL_METHOD, &&TARGET_LIST_EXTEND, @@ -166,6 +166,11 @@ static void *opcode_targets[256] = { &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, &&TARGET_CALL_METHOD_KW, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_STORE_ATTR_SLOT, + &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_STORE_FAST__STORE_FAST, @@ -249,10 +254,5 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_DO_TRACING }; diff --git a/Python/specialize.c b/Python/specialize.c index 162728314e1003..63658d720b2558 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -125,6 +125,7 @@ _Py_GetSpecializationStats(void) { err += add_stat_dict(stats, LOAD_GLOBAL, "load_global"); err += add_stat_dict(stats, LOAD_METHOD, "load_method"); err += add_stat_dict(stats, BINARY_ADD, "binary_add"); + err += add_stat_dict(stats, INPLACE_ADD, "inplace_add"); err += add_stat_dict(stats, BINARY_MULTIPLY, "binary_multiply"); err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr"); err += add_stat_dict(stats, STORE_ATTR, "store_attr"); @@ -183,6 +184,7 @@ _Py_PrintSpecializationStats(void) print_stats(out, &_specialization_stats[LOAD_GLOBAL], "load_global"); print_stats(out, &_specialization_stats[LOAD_METHOD], "load_method"); print_stats(out, &_specialization_stats[BINARY_ADD], "binary_add"); + print_stats(out, &_specialization_stats[INPLACE_ADD], "inplace_add"); print_stats(out, &_specialization_stats[BINARY_MULTIPLY], "binary_multiply"); print_stats(out, &_specialization_stats[BINARY_SUBSCR], "binary_subscr"); print_stats(out, &_specialization_stats[STORE_ATTR], "store_attr"); @@ -235,6 +237,7 @@ static uint8_t adaptive_opcodes[256] = { [LOAD_GLOBAL] = LOAD_GLOBAL_ADAPTIVE, [LOAD_METHOD] = LOAD_METHOD_ADAPTIVE, [BINARY_ADD] = BINARY_ADD_ADAPTIVE, + [INPLACE_ADD] = INPLACE_ADD_ADAPTIVE, [BINARY_MULTIPLY] = BINARY_MULTIPLY_ADAPTIVE, [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE, [CALL_FUNCTION] = CALL_FUNCTION_ADAPTIVE, @@ -247,6 +250,7 @@ static uint8_t cache_requirements[256] = { [LOAD_GLOBAL] = 2, /* _PyAdaptiveEntry and _PyLoadGlobalCache */ [LOAD_METHOD] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache */ [BINARY_ADD] = 0, + [INPLACE_ADD] = 0, [BINARY_MULTIPLY] = 0, [BINARY_SUBSCR] = 0, [CALL_FUNCTION] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */ @@ -1224,6 +1228,47 @@ _Py_Specialize_BinaryAdd(PyObject *left, PyObject *right, _Py_CODEUNIT *instr) return 0; } +int +_Py_Specialize_InplaceAdd(PyObject *left, PyObject *right, _Py_CODEUNIT *instr) +{ + PyTypeObject *left_type = Py_TYPE(left); + if (left_type != Py_TYPE(right)) { + SPECIALIZATION_FAIL(INPLACE_ADD, SPEC_FAIL_DIFFERENT_TYPES); + goto fail; + } + if (left_type == &PyUnicode_Type) { + int next_opcode = _Py_OPCODE(instr[1]); + if (next_opcode == STORE_FAST) { + *instr = _Py_MAKECODEUNIT(INPLACE_ADD_UNICODE_FAST, initial_counter_value()); + } + else { + *instr = _Py_MAKECODEUNIT(INPLACE_ADD_UNICODE, initial_counter_value()); + } + goto success; + } + else if (left_type == &PyLong_Type) { + *instr = _Py_MAKECODEUNIT(INPLACE_ADD_INT, initial_counter_value()); + goto success; + } + else if (left_type == &PyFloat_Type) { + *instr = _Py_MAKECODEUNIT(INPLACE_ADD_FLOAT, initial_counter_value()); + goto success; + + } + else { + SPECIALIZATION_FAIL(INPLACE_ADD, SPEC_FAIL_OTHER); + } +fail: + STAT_INC(INPLACE_ADD, specialization_failure); + assert(!PyErr_Occurred()); + *instr = _Py_MAKECODEUNIT(_Py_OPCODE(*instr), ADAPTIVE_CACHE_BACKOFF); + return 0; +success: + STAT_INC(INPLACE_ADD, specialization_success); + assert(!PyErr_Occurred()); + return 0; +} + int _Py_Specialize_BinaryMultiply(PyObject *left, PyObject *right, _Py_CODEUNIT *instr) {