diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 0ed4c7c3a35436..2f957afb259b0f 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -287,7 +287,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_FATAL_ERROR] = 0, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, [_DEOPT] = 0, - [_ERROR_POP_N] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_ERROR_POP_N] = HAS_ARG_FLAG, [_TIER2_RESUME_CHECK] = HAS_DEOPT_FLAG, }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 07022e9932d203..9fb58b9cdcc022 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3526,6 +3526,7 @@ dummy_func( } op(_MAYBE_EXPAND_METHOD, (callable[1], self_or_null[1], args[oparg] -- func[1], maybe_self[1], args[oparg])) { + (void)args; if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -3892,6 +3893,7 @@ dummy_func( _CHECK_PERIODIC; op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable[1], null[1], args[oparg] -- init[1], self[1], args[oparg])) { + (void)args; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); DEOPT_IF(!PyStackRef_IsNull(null[0])); DEOPT_IF(!PyType_Check(callable_o)); @@ -4119,7 +4121,7 @@ dummy_func( PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { - GOTO_ERROR(error); + ERROR_NO_POP(); } PyStackRef_CLOSE(arg_stackref); DEAD(args); @@ -4364,6 +4366,7 @@ dummy_func( } op(_MAYBE_EXPAND_METHOD_KW, (callable[1], self_or_null[1], args[oparg], kwnames_in -- func[1], maybe_self[1], args[oparg], kwnames_out)) { + (void)args; if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -5022,7 +5025,7 @@ dummy_func( if (frame->lltrace >= 2) { printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %u, temp %d, target %d -> %s]\n", + printf(", exit %lu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); @@ -5032,11 +5035,11 @@ dummy_func( exit->temperature = initial_temperature_backoff_counter(); Py_CLEAR(exit->executor); } + tstate->previous_executor = (PyObject *)current_executor; if (exit->executor == NULL) { _Py_BackoffCounter temperature = exit->temperature; if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); - tstate->previous_executor = (PyObject *)current_executor; GOTO_TIER_ONE(target); } _PyExecutorObject *executor; @@ -5049,20 +5052,13 @@ dummy_func( int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); if (optimized <= 0) { exit->temperature = restart_backoff_counter(temperature); - if (optimized < 0) { - GOTO_UNWIND(); - } - tstate->previous_executor = (PyObject *)current_executor; - GOTO_TIER_ONE(target); - } - else { - exit->temperature = initial_temperature_backoff_counter(); + GOTO_TIER_ONE(optimized < 0 ? NULL : target); } + exit->temperature = initial_temperature_backoff_counter(); } exit->executor = executor; } Py_INCREF(exit->executor); - tstate->previous_executor = (PyObject *)current_executor; GOTO_TIER_TWO(exit->executor); } @@ -5130,7 +5126,7 @@ dummy_func( if (frame->lltrace >= 2) { printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %u, temp %d, target %d -> %s]\n", + printf(", exit %lu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); @@ -5150,21 +5146,15 @@ dummy_func( int optimized = _PyOptimizer_Optimize(frame, target, &executor, 0); if (optimized <= 0) { exit->temperature = restart_backoff_counter(exit->temperature); - if (optimized < 0) { - GOTO_UNWIND(); - } - GOTO_TIER_ONE(target); - } - else { - exit->temperature = initial_temperature_backoff_counter(); + GOTO_TIER_ONE(optimized < 0 ? NULL : target); } + exit->temperature = initial_temperature_backoff_counter(); } GOTO_TIER_TWO(executor); } tier2 op(_START_EXECUTOR, (executor/4 --)) { - Py_DECREF(tstate->previous_executor); - tstate->previous_executor = NULL; + Py_CLEAR(tstate->previous_executor); #ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; #endif @@ -5190,14 +5180,16 @@ dummy_func( } tier2 op(_DEOPT, (--)) { - EXIT_TO_TIER1(); + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } tier2 op(_ERROR_POP_N, (target/2 --)) { + tstate->previous_executor = (PyObject *)current_executor; assert(oparg == 0); frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; SYNC_SP(); - GOTO_UNWIND(); + GOTO_TIER_ONE(NULL); } /* Progress is guaranteed if we DEOPT on the eval breaker, because diff --git a/Python/ceval.c b/Python/ceval.c index b2e2b540783880..6c8e39a09c255b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -879,9 +879,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #undef LOAD_IP #define LOAD_IP(UNUSED) (void)0 -#undef GOTO_ERROR -#define GOTO_ERROR(LABEL) goto LABEL ## _tier_two - #ifdef Py_STATS // Disable these macros that apply to Tier 1 stats when we are in Tier 2 #undef STAT_INC @@ -957,46 +954,17 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _PyOpcode_OpName[frame->instr_ptr->op.code]); } #endif - assert (next_uop[-1].format == UOP_FORMAT_JUMP); + assert(next_uop[-1].format == UOP_FORMAT_JUMP); uint16_t target = uop_get_error_target(&next_uop[-1]); next_uop = current_executor->trace + target; goto tier2_dispatch; -error_tier_two: - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - assert(next_uop[-1].format == UOP_FORMAT_TARGET); - frame->return_offset = 0; // Don't leave this random - Py_DECREF(current_executor); - tstate->previous_executor = NULL; - next_instr = frame->instr_ptr; - goto error; - jump_to_jump_target: assert(next_uop[-1].format == UOP_FORMAT_JUMP); target = uop_get_jump_target(&next_uop[-1]); next_uop = current_executor->trace + target; goto tier2_dispatch; -exit_to_tier1_dynamic: - next_instr = frame->instr_ptr; - goto goto_to_tier1; -exit_to_tier1: - assert(next_uop[-1].format == UOP_FORMAT_TARGET); - next_instr = next_uop[-1].target + _PyFrame_GetBytecode(frame); -goto_to_tier1: -#ifdef Py_DEBUG - if (frame->lltrace >= 2) { - printf("DEOPT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(" -> %s]\n", - _PyOpcode_OpName[next_instr->op.code]); - } -#endif - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - Py_DECREF(current_executor); - tstate->previous_executor = NULL; - DISPATCH(); - #endif // _Py_JIT #endif // _Py_TIER2 diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index fde8e5efa983b2..0a4f65feb3b512 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -161,9 +161,6 @@ do { \ JUMP_TO_LABEL(start_frame); \ } while (0) -// Use this instead of 'goto error' so Tier 2 can go to a different label -#define GOTO_ERROR(LABEL) JUMP_TO_LABEL(LABEL) - /* Tuple access macros */ #ifndef Py_DEBUG @@ -387,17 +384,19 @@ _PyFrame_SetStackPointer(frame, stack_pointer) #define GOTO_TIER_TWO(EXECUTOR) \ do { \ OPT_STAT_INC(traces_executed); \ - jit_func jitted = (EXECUTOR)->jit_code; \ + _PyExecutorObject *_executor = (EXECUTOR); \ + jit_func jitted = _executor->jit_code; \ + /* Keep the shim frame alive via the executor: */ \ + Py_INCREF(_executor); \ next_instr = jitted(frame, stack_pointer, tstate); \ - Py_DECREF(tstate->previous_executor); \ - tstate->previous_executor = NULL; \ + Py_DECREF(_executor); \ + Py_CLEAR(tstate->previous_executor); \ frame = tstate->current_frame; \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ if (next_instr == NULL) { \ next_instr = frame->instr_ptr; \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ goto error; \ } \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ DISPATCH(); \ } while (0) #else @@ -410,24 +409,25 @@ do { \ } while (0) #endif -#define GOTO_TIER_ONE(TARGET) \ -do { \ - Py_DECREF(tstate->previous_executor); \ - tstate->previous_executor = NULL; \ - next_instr = target; \ - DISPATCH(); \ +#define GOTO_TIER_ONE(TARGET) \ +do { \ + next_instr = (TARGET); \ + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ + Py_CLEAR(tstate->previous_executor); \ + if (next_instr == NULL) { \ + next_instr = frame->instr_ptr; \ + goto error; \ + } \ + DISPATCH(); \ } while (0) -#define CURRENT_OPARG() (next_uop[-1].oparg) - +#define CURRENT_OPARG() (next_uop[-1].oparg) #define CURRENT_OPERAND0() (next_uop[-1].operand0) #define CURRENT_OPERAND1() (next_uop[-1].operand1) +#define CURRENT_TARGET() (next_uop[-1].target) #define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target #define JUMP_TO_ERROR() goto jump_to_error_target -#define GOTO_UNWIND() goto error_tier_two -#define EXIT_TO_TIER1() goto exit_to_tier1 -#define EXIT_TO_TIER1_DYNAMIC() goto exit_to_tier1_dynamic; /* Stackref macros */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 22d11059fcadb8..2e7f5ecc7aa4da 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4369,6 +4369,8 @@ callable = &stack_pointer[-2 - oparg]; func = &stack_pointer[-2 - oparg]; maybe_self = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-oparg]; + (void)args; if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -4936,7 +4938,9 @@ callable = &stack_pointer[-2 - oparg]; init = &stack_pointer[-2 - oparg]; self = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-oparg]; uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + (void)args; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (!PyStackRef_IsNull(null[0])) { UOP_STAT_INC(uopcode, miss); @@ -4989,9 +4993,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); - stack_pointer = _PyFrame_GetStackPointer(frame); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self[0]); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5317,7 +5321,7 @@ PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { - GOTO_ERROR(error); + JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg_stackref); @@ -5723,6 +5727,8 @@ callable = &stack_pointer[-3 - oparg]; func = &stack_pointer[-3 - oparg]; maybe_self = &stack_pointer[-2 - oparg]; + args = &stack_pointer[-1 - oparg]; + (void)args; if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -6339,16 +6345,14 @@ PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _PyExitData *exit = (_PyExitData *)exit_p; PyCodeObject *code = _PyFrame_GetCode(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; - stack_pointer = _PyFrame_GetStackPointer(frame); #if defined(Py_DEBUG) && !defined(_Py_JIT) OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { _PyFrame_SetStackPointer(frame, stack_pointer); printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %u, temp %d, target %d -> %s]\n", + printf(", exit %lu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); @@ -6361,11 +6365,11 @@ Py_CLEAR(exit->executor); stack_pointer = _PyFrame_GetStackPointer(frame); } + tstate->previous_executor = (PyObject *)current_executor; if (exit->executor == NULL) { _Py_BackoffCounter temperature = exit->temperature; if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); - tstate->previous_executor = (PyObject *)current_executor; GOTO_TIER_ONE(target); } _PyExecutorObject *executor; @@ -6380,20 +6384,13 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (optimized <= 0) { exit->temperature = restart_backoff_counter(temperature); - if (optimized < 0) { - GOTO_UNWIND(); - } - tstate->previous_executor = (PyObject *)current_executor; - GOTO_TIER_ONE(target); - } - else { - exit->temperature = initial_temperature_backoff_counter(); + GOTO_TIER_ONE(optimized < 0 ? NULL : target); } + exit->temperature = initial_temperature_backoff_counter(); } exit->executor = executor; } Py_INCREF(exit->executor); - tstate->previous_executor = (PyObject *)current_executor; GOTO_TIER_TWO(exit->executor); break; } @@ -6524,7 +6521,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %u, temp %d, target %d -> %s]\n", + printf(", exit %lu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); @@ -6547,14 +6544,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (optimized <= 0) { exit->temperature = restart_backoff_counter(exit->temperature); - if (optimized < 0) { - GOTO_UNWIND(); - } - GOTO_TIER_ONE(target); - } - else { - exit->temperature = initial_temperature_backoff_counter(); + GOTO_TIER_ONE(optimized < 0 ? NULL : target); } + exit->temperature = initial_temperature_backoff_counter(); } GOTO_TIER_TWO(executor); break; @@ -6563,9 +6555,8 @@ case _START_EXECUTOR: { PyObject *executor = (PyObject *)CURRENT_OPERAND0(); _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(tstate->previous_executor); + Py_CLEAR(tstate->previous_executor); stack_pointer = _PyFrame_GetStackPointer(frame); - tstate->previous_executor = NULL; #ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; #endif @@ -6599,18 +6590,18 @@ } case _DEOPT: { - EXIT_TO_TIER1(); + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); break; } case _ERROR_POP_N: { oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND0(); + tstate->previous_executor = (PyObject *)current_executor; assert(oparg == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; - stack_pointer = _PyFrame_GetStackPointer(frame); - GOTO_UNWIND(); + GOTO_TIER_ONE(NULL); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ec9b287541f016..914b06987ed64e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1278,6 +1278,8 @@ args = &stack_pointer[-oparg]; func = &stack_pointer[-2 - oparg]; maybe_self = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-oparg]; + (void)args; if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -1440,7 +1442,9 @@ callable = &stack_pointer[-2 - oparg]; init = &stack_pointer[-2 - oparg]; self = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); + (void)args; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (!PyStackRef_IsNull(null[0])) { UPDATE_MISS_STATS(CALL); @@ -1491,9 +1495,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); - stack_pointer = _PyFrame_GetStackPointer(frame); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self[0]); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2532,6 +2536,8 @@ args = &stack_pointer[-1 - oparg]; func = &stack_pointer[-3 - oparg]; maybe_self = &stack_pointer[-2 - oparg]; + args = &stack_pointer[-1 - oparg]; + (void)args; if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -3058,7 +3064,7 @@ PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { - GOTO_ERROR(error); + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg_stackref); @@ -5679,6 +5685,8 @@ callable = &stack_pointer[-2 - oparg]; func = &stack_pointer[-2 - oparg]; maybe_self = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-oparg]; + (void)args; if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -6082,6 +6090,8 @@ kwnames_in = stack_pointer[-1]; func = &stack_pointer[-3 - oparg]; maybe_self = &stack_pointer[-2 - oparg]; + args = &stack_pointer[-1 - oparg]; + (void)args; if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -6397,9 +6407,7 @@ int original_opcode = 0; if (tstate->tracing) { PyCodeObject *code = _PyFrame_GetCode(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; next_instr = this_instr; } else { @@ -6672,9 +6680,7 @@ if (bytecode == NULL) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; frame->instr_ptr = bytecode + off; // Make sure this_instr gets reset correctley for any uops that @@ -9492,11 +9498,7 @@ if (oparg) { PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); if (PyLong_Check(lasti)) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti); - stack_pointer = _PyFrame_GetStackPointer(frame); assert(!_PyErr_Occurred(tstate)); } else { @@ -9508,8 +9510,6 @@ stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); } assert(exc && PyExceptionInstance_Check(exc)); stack_pointer += -1; @@ -9558,9 +9558,7 @@ if (bytecode == NULL) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; frame->instr_ptr = bytecode + off; // Make sure this_instr gets reset correctley for any uops that diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 91573e82841cc9..09f1915bb3a5e0 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -385,9 +385,6 @@ dummy_func(void) { } op(_BINARY_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame: _Py_UOpsAbstractFrame *)) { - (void)container; - (void)sub; - (void)getitem; new_frame = NULL; ctx->done = true; } @@ -434,8 +431,6 @@ dummy_func(void) { } op(_COMPARE_OP, (left, right -- res)) { - (void)left; - (void)right; if (oparg & 16) { res = sym_new_type(ctx, &PyBool_Type); } @@ -445,32 +440,22 @@ dummy_func(void) { } op(_COMPARE_OP_INT, (left, right -- res)) { - (void)left; - (void)right; res = sym_new_type(ctx, &PyBool_Type); } op(_COMPARE_OP_FLOAT, (left, right -- res)) { - (void)left; - (void)right; res = sym_new_type(ctx, &PyBool_Type); } op(_COMPARE_OP_STR, (left, right -- res)) { - (void)left; - (void)right; res = sym_new_type(ctx, &PyBool_Type); } op(_IS_OP, (left, right -- res)) { - (void)left; - (void)right; res = sym_new_type(ctx, &PyBool_Type); } op(_CONTAINS_OP, (left, right -- res)) { - (void)left; - (void)right; res = sym_new_type(ctx, &PyBool_Type); } @@ -522,7 +507,6 @@ dummy_func(void) { op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr)) { attr = sym_new_not_null(ctx); (void)offset; - (void)owner; } op(_CHECK_ATTR_MODULE_PUSH_KEYS, (dict_version/2, owner -- owner, mod_keys)) { @@ -583,26 +567,21 @@ dummy_func(void) { op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict)) { dict = sym_new_not_null(ctx); - (void)owner; } op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict -- attr)) { attr = sym_new_not_null(ctx); (void)hint; - (void)owner; - (void)dict; } op(_LOAD_ATTR_SLOT, (index/1, owner -- attr)) { attr = sym_new_not_null(ctx); (void)index; - (void)owner; } op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) { attr = sym_new_not_null(ctx); (void)descr; - (void)owner; } op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) { @@ -625,19 +604,16 @@ dummy_func(void) { op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame: _Py_UOpsAbstractFrame *)) { (void)fget; - (void)owner; new_frame = NULL; ctx->done = true; } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable[1], self_or_null[1], unused[oparg] -- callable[1], self_or_null[1], unused[oparg])) { - (void)callable; callable[0] = sym_new_not_null(ctx); self_or_null[0] = sym_new_not_null(ctx); } op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - (void)self_or_null; if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(callable))); REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); @@ -657,7 +633,6 @@ dummy_func(void) { } } } - (void)self_or_null; } op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { @@ -667,7 +642,6 @@ dummy_func(void) { op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { int argcount = oparg; - (void)callable; PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); @@ -695,16 +669,12 @@ dummy_func(void) { } op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- func, maybe_self, args[oparg])) { - (void)callable; - (void)self_or_null; (void)args; func = sym_new_not_null(ctx); maybe_self = sym_new_not_null(ctx); } op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { - (void)(self_or_null); - (void)(callable); PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); co = get_code_with_logging((this_instr + 2)); @@ -717,27 +687,18 @@ dummy_func(void) { } op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame *)) { - (void)callable; - (void)self_or_null; - (void)args; - (void)kwnames; new_frame = NULL; ctx->done = true; } op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) { (void)type_version; - (void)callable; - (void)null; (void)args; self = sym_new_not_null(ctx); init = sym_new_not_null(ctx); } op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { - (void)self; - (void)init; - (void)args; init_frame = NULL; ctx->done = true; } @@ -806,7 +767,7 @@ dummy_func(void) { corresponding_check_stack = this_instr; } - op (_CHECK_STACK_SPACE_OPERAND, ( -- )) { + op (_CHECK_STACK_SPACE_OPERAND, (framesize/2 -- )) { (void)framesize; /* We should never see _CHECK_STACK_SPACE_OPERANDs. * They are only created at the end of this pass. */ @@ -848,7 +809,6 @@ dummy_func(void) { op(_UNPACK_SEQUENCE, (seq -- values[oparg])) { /* This has to be done manually */ - (void)seq; for (int i = 0; i < oparg; i++) { values[i] = sym_new_unknown(ctx); } @@ -856,7 +816,6 @@ dummy_func(void) { op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8])) { /* This has to be done manually */ - (void)seq; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { values[i] = sym_new_unknown(ctx); @@ -865,7 +824,6 @@ dummy_func(void) { op(_ITER_NEXT_RANGE, (iter -- iter, next)) { next = sym_new_type(ctx, &PyLong_Type); - (void)iter; } op(_GUARD_IS_TRUE_POP, (flag -- )) { @@ -917,7 +875,6 @@ dummy_func(void) { } op(_LOAD_SPECIAL, (owner -- attr, self_or_null)) { - (void)owner; attr = sym_new_not_null(ctx); self_or_null = sym_new_unknown(ctx); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 11f1815a977798..dde41d9407ee33 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -645,16 +645,7 @@ } case _BINARY_SUBSCR_INIT_CALL: { - JitOptSymbol *getitem; - JitOptSymbol *sub; - JitOptSymbol *container; _Py_UOpsAbstractFrame *new_frame; - getitem = stack_pointer[-1]; - sub = stack_pointer[-2]; - container = stack_pointer[-3]; - (void)container; - (void)sub; - (void)getitem; new_frame = NULL; ctx->done = true; stack_pointer[-3] = (JitOptSymbol *)new_frame; @@ -816,12 +807,9 @@ } case _UNPACK_SEQUENCE: { - JitOptSymbol *seq; JitOptSymbol **values; - seq = stack_pointer[-1]; values = &stack_pointer[-1]; /* This has to be done manually */ - (void)seq; for (int i = 0; i < oparg; i++) { values[i] = sym_new_unknown(ctx); } @@ -869,12 +857,9 @@ } case _UNPACK_EX: { - JitOptSymbol *seq; JitOptSymbol **values; - seq = stack_pointer[-1]; values = &stack_pointer[-1]; /* This has to be done manually */ - (void)seq; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { values[i] = sym_new_unknown(ctx); @@ -1178,13 +1163,10 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - JitOptSymbol *owner; JitOptSymbol *attr; - owner = stack_pointer[-1]; uint16_t offset = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)offset; - (void)owner; stack_pointer[-1] = attr; break; } @@ -1258,11 +1240,8 @@ } case _CHECK_ATTR_WITH_HINT: { - JitOptSymbol *owner; JitOptSymbol *dict; - owner = stack_pointer[-1]; dict = sym_new_not_null(ctx); - (void)owner; stack_pointer[0] = dict; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1270,16 +1249,10 @@ } case _LOAD_ATTR_WITH_HINT: { - JitOptSymbol *dict; - JitOptSymbol *owner; JitOptSymbol *attr; - dict = stack_pointer[-1]; - owner = stack_pointer[-2]; uint16_t hint = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)hint; - (void)owner; - (void)dict; stack_pointer[-2] = attr; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1287,13 +1260,10 @@ } case _LOAD_ATTR_SLOT: { - JitOptSymbol *owner; JitOptSymbol *attr; - owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)index; - (void)owner; stack_pointer[-1] = attr; break; } @@ -1303,24 +1273,18 @@ } case _LOAD_ATTR_CLASS: { - JitOptSymbol *owner; JitOptSymbol *attr; - owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; attr = sym_new_not_null(ctx); (void)descr; - (void)owner; stack_pointer[-1] = attr; break; } case _LOAD_ATTR_PROPERTY_FRAME: { - JitOptSymbol *owner; _Py_UOpsAbstractFrame *new_frame; - owner = stack_pointer[-1]; PyObject *fget = (PyObject *)this_instr->operand0; (void)fget; - (void)owner; new_frame = NULL; ctx->done = true; stack_pointer[-1] = (JitOptSymbol *)new_frame; @@ -1352,13 +1316,7 @@ } case _COMPARE_OP: { - JitOptSymbol *right; - JitOptSymbol *left; JitOptSymbol *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - (void)left; - (void)right; if (oparg & 16) { res = sym_new_type(ctx, &PyBool_Type); } @@ -1376,13 +1334,7 @@ } case _COMPARE_OP_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; JitOptSymbol *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - (void)left; - (void)right; res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -1391,13 +1343,7 @@ } case _COMPARE_OP_INT: { - JitOptSymbol *right; - JitOptSymbol *left; JitOptSymbol *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - (void)left; - (void)right; res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -1406,13 +1352,7 @@ } case _COMPARE_OP_STR: { - JitOptSymbol *right; - JitOptSymbol *left; JitOptSymbol *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - (void)left; - (void)right; res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -1421,13 +1361,7 @@ } case _IS_OP: { - JitOptSymbol *right; - JitOptSymbol *left; JitOptSymbol *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - (void)left; - (void)right; res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -1436,13 +1370,7 @@ } case _CONTAINS_OP: { - JitOptSymbol *right; - JitOptSymbol *left; JitOptSymbol *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - (void)left; - (void)right; res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -1635,11 +1563,8 @@ } case _ITER_NEXT_RANGE: { - JitOptSymbol *iter; JitOptSymbol *next; - iter = stack_pointer[-1]; next = sym_new_type(ctx, &PyLong_Type); - (void)iter; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1653,11 +1578,8 @@ } case _LOAD_SPECIAL: { - JitOptSymbol *owner; JitOptSymbol *attr; JitOptSymbol *self_or_null; - owner = stack_pointer[-1]; - (void)owner; attr = sym_new_not_null(ctx); self_or_null = sym_new_unknown(ctx); stack_pointer[-1] = attr; @@ -1764,16 +1686,10 @@ case _MAYBE_EXPAND_METHOD: { JitOptSymbol **args; - JitOptSymbol *self_or_null; - JitOptSymbol *callable; JitOptSymbol *func; JitOptSymbol *maybe_self; args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; args = &stack_pointer[-oparg]; - (void)callable; - (void)self_or_null; (void)args; func = sym_new_not_null(ctx); maybe_self = sym_new_not_null(ctx); @@ -1787,17 +1703,11 @@ /* _MONITOR_CALL is not a viable micro-op for tier 2 */ case _PY_FRAME_GENERAL: { - JitOptSymbol *self_or_null; - JitOptSymbol *callable; _Py_UOpsAbstractFrame *new_frame; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - (void)(self_or_null); - (void)(callable); PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); co = get_code_with_logging((this_instr + 2)); if (co == NULL) { ctx->done = true; @@ -1811,12 +1721,9 @@ } case _CHECK_FUNCTION_VERSION: { - JitOptSymbol *self_or_null; JitOptSymbol *callable; - self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand0; - (void)self_or_null; if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(callable))); REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); @@ -1866,7 +1773,6 @@ JitOptSymbol **callable; self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - (void)callable; callable[0] = sym_new_not_null(ctx); self_or_null[0] = sym_new_not_null(ctx); break; @@ -1896,7 +1802,6 @@ } } } - (void)self_or_null; break; } @@ -1909,13 +1814,10 @@ case _INIT_CALL_PY_EXACT_ARGS: { JitOptSymbol **args; JitOptSymbol *self_or_null; - JitOptSymbol *callable; _Py_UOpsAbstractFrame *new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; int argcount = oparg; - (void)callable; PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); stack_pointer += -2 - oparg; @@ -2010,18 +1912,12 @@ case _CHECK_AND_ALLOCATE_OBJECT: { JitOptSymbol **args; - JitOptSymbol *null; - JitOptSymbol *callable; JitOptSymbol *self; JitOptSymbol *init; args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; args = &stack_pointer[-oparg]; uint32_t type_version = (uint32_t)this_instr->operand0; (void)type_version; - (void)callable; - (void)null; (void)args; self = sym_new_not_null(ctx); init = sym_new_not_null(ctx); @@ -2031,16 +1927,7 @@ } case _CREATE_INIT_FRAME: { - JitOptSymbol **args; - JitOptSymbol *init; - JitOptSymbol *self; _Py_UOpsAbstractFrame *init_frame; - args = &stack_pointer[-oparg]; - init = stack_pointer[-1 - oparg]; - self = stack_pointer[-2 - oparg]; - (void)self; - (void)init; - (void)args; init_frame = NULL; ctx->done = true; stack_pointer[-2 - oparg] = (JitOptSymbol *)init_frame; @@ -2174,19 +2061,7 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 */ case _PY_FRAME_KW: { - JitOptSymbol *kwnames; - JitOptSymbol **args; - JitOptSymbol *self_or_null; - JitOptSymbol *callable; _Py_UOpsAbstractFrame *new_frame; - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - (void)callable; - (void)self_or_null; - (void)args; - (void)kwnames; new_frame = NULL; ctx->done = true; stack_pointer[-3 - oparg] = (JitOptSymbol *)new_frame; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 57cc587428c2b9..c127d0334d9cf8 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -621,6 +621,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "_PyErr_Occurred", "_PyEval_FrameClearAndPop", "_PyFloat_FromDouble_ConsumeInputs", + "_PyFrame_GetBytecode", "_PyFrame_GetCode", "_PyFrame_IsIncomplete", "_PyFrame_PushUnchecked", diff --git a/Tools/jit/jit.h b/Tools/jit/jit.h index 47da64cb12bd24..f767ef68127eb7 100644 --- a/Tools/jit/jit.h +++ b/Tools/jit/jit.h @@ -2,3 +2,7 @@ // pointer with __attribute__((preserve_none)), since this attribute may not be // supported by the compiler used to build the rest of the interpreter. typedef jit_func __attribute__((preserve_none)) jit_func_preserve_none; + +#define PATCH_VALUE(TYPE, NAME, ALIAS) \ + PyAPI_DATA(void) ALIAS; \ + TYPE NAME = (TYPE)(uintptr_t)&ALIAS; diff --git a/Tools/jit/shim.c b/Tools/jit/shim.c index f0cffa2f049d26..ebd4e9bc858b73 100644 --- a/Tools/jit/shim.c +++ b/Tools/jit/shim.c @@ -9,16 +9,7 @@ _Py_CODEUNIT * _JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) { - // This is subtle. The actual trace will return to us once it exits, so we - // need to make sure that we stay alive until then. If our trace side-exits - // into another trace, and this trace is then invalidated, the code for - // *this function* will be freed and we'll crash upon return: - PyAPI_DATA(void) _JIT_EXECUTOR; - PyObject *executor = (PyObject *)(uintptr_t)&_JIT_EXECUTOR; - Py_INCREF(executor); // Note that this is *not* a tail call: - PyAPI_DATA(void) _JIT_CONTINUE; - _Py_CODEUNIT *target = ((jit_func_preserve_none)&_JIT_CONTINUE)(frame, stack_pointer, tstate); - Py_SETREF(tstate->previous_executor, executor); - return target; + PATCH_VALUE(jit_func_preserve_none, call, _JIT_CONTINUE); + return call(frame, stack_pointer, tstate); } diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 6ea04f5640ef05..0b7d077d78ce7d 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -32,28 +32,22 @@ #undef CURRENT_OPERAND1 #define CURRENT_OPERAND1() (_operand1) -#undef ENABLE_SPECIALIZATION -#define ENABLE_SPECIALIZATION (0) - -#undef GOTO_ERROR -#define GOTO_ERROR(LABEL) \ - do { \ - goto LABEL ## _tier_two; \ - } while (0) +#undef CURRENT_TARGET +#define CURRENT_TARGET() (_target) #undef GOTO_TIER_TWO -#define GOTO_TIER_TWO(EXECUTOR) \ -do { \ - OPT_STAT_INC(traces_executed); \ - __attribute__((musttail)) \ - return ((jit_func_preserve_none)((EXECUTOR)->jit_side_entry))(frame, stack_pointer, tstate); \ +#define GOTO_TIER_TWO(EXECUTOR) \ +do { \ + OPT_STAT_INC(traces_executed); \ + jit_func_preserve_none jitted = (EXECUTOR)->jit_side_entry; \ + __attribute__((musttail)) return jitted(frame, stack_pointer, tstate); \ } while (0) #undef GOTO_TIER_ONE -#define GOTO_TIER_ONE(TARGET) \ -do { \ +#define GOTO_TIER_ONE(TARGET) \ +do { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - return TARGET; \ + return TARGET; \ } while (0) #undef LOAD_IP @@ -61,15 +55,15 @@ do { \ do { \ } while (0) -#define PATCH_VALUE(TYPE, NAME, ALIAS) \ - PyAPI_DATA(void) ALIAS; \ - TYPE NAME = (TYPE)(uintptr_t)&ALIAS; +#undef LLTRACE_RESUME_FRAME +#define LLTRACE_RESUME_FRAME() \ + do { \ + } while (0) -#define PATCH_JUMP(ALIAS) \ -do { \ - PyAPI_DATA(void) ALIAS; \ - __attribute__((musttail)) \ - return ((jit_func_preserve_none)&ALIAS)(frame, stack_pointer, tstate); \ +#define PATCH_JUMP(ALIAS) \ +do { \ + PATCH_VALUE(jit_func_preserve_none, jump, ALIAS); \ + __attribute__((musttail)) return jump(frame, stack_pointer, tstate); \ } while (0) #undef JUMP_TO_JUMP_TARGET @@ -78,14 +72,6 @@ do { \ #undef JUMP_TO_ERROR #define JUMP_TO_ERROR() PATCH_JUMP(_JIT_ERROR_TARGET) -#undef WITHIN_STACK_BOUNDS -#define WITHIN_STACK_BOUNDS() 1 - -#undef LLTRACE_RESUME_FRAME -#define LLTRACE_RESUME_FRAME() \ - do { \ - } while (0) - #define TIER_TWO 2 __attribute__((preserve_none)) _Py_CODEUNIT * @@ -106,16 +92,13 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState PATCH_VALUE(uint32_t, _operand0_hi, _JIT_OPERAND0_HI) PATCH_VALUE(uint32_t, _operand0_lo, _JIT_OPERAND0_LO) uint64_t _operand0 = ((uint64_t)_operand0_hi << 32) | _operand0_lo; - PATCH_VALUE(uint32_t, _operand1_hi, _JIT_OPERAND1_HI) PATCH_VALUE(uint32_t, _operand1_lo, _JIT_OPERAND1_LO) uint64_t _operand1 = ((uint64_t)_operand1_hi << 32) | _operand1_lo; #endif PATCH_VALUE(uint32_t, _target, _JIT_TARGET) - OPT_STAT_INC(uops_executed); UOP_STAT_INC(uopcode, execution_count); - switch (uopcode) { // The actual instruction definition gets inserted here: CASE @@ -123,15 +106,4 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState Py_UNREACHABLE(); } PATCH_JUMP(_JIT_CONTINUE); - // Labels that the instruction implementations expect to exist: - -error_tier_two: - tstate->previous_executor = (PyObject *)current_executor; - GOTO_TIER_ONE(NULL); -exit_to_tier1: - tstate->previous_executor = (PyObject *)current_executor; - GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target); -exit_to_tier1_dynamic: - tstate->previous_executor = (PyObject *)current_executor; - GOTO_TIER_ONE(frame->instr_ptr); }