diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index de3529670a845b..24999b298e6046 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -46,6 +46,7 @@ typedef struct _err_stackitem { } _PyErr_StackItem; +extern __thread struct _ts *_Py_tls_tstate; // The PyThreadState typedef is in Include/pystate.h. struct _ts { diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 50d906c4c6ddec..46826eff92b3a6 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -189,9 +189,8 @@ typedef struct _Py_AuditHookEntry { struct _gilstate_runtime_state { int check_enabled; - /* Assuming the current thread holds the GIL, this is the - PyThreadState for the current thread. */ - _Py_atomic_address tstate_current; + /* This is the PyThreadState for the thread holding the GIL. */ + _Py_atomic_address gil_holder; /* The single PyInterpreterState used by this process' GILState implementation */ @@ -328,10 +327,6 @@ _Py_ThreadCanHandlePendingCalls(void) /* Variable and macro for in-line access to current thread and interpreter state */ -static inline PyThreadState* _PyRuntimeState_GetThreadState(_PyRuntimeState *runtime) { - return (PyThreadState*)_Py_atomic_load_relaxed(&runtime->gilstate.tstate_current); -} - /* Get the current Python thread state. Efficient macro reading directly the 'gilstate.tstate_current' atomic @@ -341,7 +336,7 @@ static inline PyThreadState* _PyRuntimeState_GetThreadState(_PyRuntimeState *run The caller must hold the GIL. See also PyThreadState_Get() and PyThreadState_GET(). */ -#define _PyThreadState_GET() _PyRuntimeState_GetThreadState(&_PyRuntime) +#define _PyThreadState_GET() _Py_tls_tstate /* Redefine PyThreadState_GET() as an alias to _PyThreadState_GET() */ #undef PyThreadState_GET diff --git a/Python/ceval.c b/Python/ceval.c index 836457d1a57a43..0a24e7a74e64cc 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -355,8 +355,7 @@ _PyEval_Fini(void) void PyEval_AcquireLock(void) { - _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); ensure_tstate_not_null(__func__, tstate); take_gil(tstate); @@ -366,7 +365,7 @@ void PyEval_ReleaseLock(void) { _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); /* This function must succeed when the current thread state is NULL. We therefore avoid PyThreadState_Get() which dumps a fatal error in debug mode. */ @@ -419,7 +418,7 @@ _PyEval_ReInitThreads(_PyRuntimeState *runtime) return; } recreate_gil(gil); - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); ensure_tstate_not_null(__func__, tstate); take_gil(tstate); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index b7019e3bb67abb..cc17bd45a06db0 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1108,8 +1108,7 @@ _Py_InitializeMain(void) if (_PyStatus_EXCEPTION(status)) { return status; } - _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); return pyinit_main(tstate); } @@ -1337,7 +1336,7 @@ Py_FinalizeEx(void) } /* Get current thread state and interpreter pointer */ - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); PyInterpreterState *interp = tstate->interp; // Wrap up existing "threading"-module-created, non-daemon threads. @@ -2207,7 +2206,7 @@ fatal_error(const char *prefix, const char *msg, int status) _PyRuntimeState *runtime = &_PyRuntime; fatal_error_dump_runtime(stream, runtime); - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); PyInterpreterState *interp = NULL; if (tstate != NULL) { interp = tstate->interp; diff --git a/Python/pystate.c b/Python/pystate.c index 6331a854c810f8..686310b2b7269b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -32,9 +32,9 @@ extern "C" { #endif #define _PyRuntimeGILState_GetThreadState(gilstate) \ - ((PyThreadState*)_Py_atomic_load_relaxed(&(gilstate)->tstate_current)) + ((PyThreadState*)_Py_atomic_load_relaxed(&(gilstate)->gil_holder)) #define _PyRuntimeGILState_SetThreadState(gilstate, value) \ - _Py_atomic_store_relaxed(&(gilstate)->tstate_current, \ + _Py_atomic_store_relaxed(&(gilstate)->gil_holder, \ (uintptr_t)(value)) static void @@ -46,6 +46,7 @@ ensure_tstate_not_null(const char *func, PyThreadState *tstate) } } +__thread struct _ts *_Py_tls_tstate = NULL; /* Forward declarations */ static PyThreadState *_PyGILState_GetThisThreadState(struct _gilstate_runtime_state *gilstate); @@ -860,6 +861,7 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate) struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate; tstate_delete_common(tstate, gilstate); _PyRuntimeGILState_SetThreadState(gilstate, NULL); + _Py_tls_tstate = NULL; _PyEval_ReleaseLock(tstate); PyMem_RawFree(tstate); } @@ -934,9 +936,11 @@ PyThreadState_Get(void) PyThreadState * _PyThreadState_Swap(struct _gilstate_runtime_state *gilstate, PyThreadState *newts) { - PyThreadState *oldts = _PyRuntimeGILState_GetThreadState(gilstate); + assert(_PyRuntimeGILState_GetThreadState(gilstate) == _Py_tls_tstate); + PyThreadState *oldts = _Py_tls_tstate; _PyRuntimeGILState_SetThreadState(gilstate, newts); + _Py_tls_tstate = newts; /* It should not be possible for more than one thread state to be used for a thread. Check this the best we can in debug builds. @@ -1013,7 +1017,7 @@ int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) { _PyRuntimeState *runtime = &_PyRuntime; - PyInterpreterState *interp = _PyRuntimeState_GetThreadState(runtime)->interp; + PyInterpreterState *interp = _PyThreadState_GET()->interp; /* Although the GIL is held, a few C API functions can be called * without the GIL held, and in particular some that create and diff --git a/Python/sysmodule.c b/Python/sysmodule.c index c78a6273805221..a5c530cc69ef15 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -288,7 +288,7 @@ _PySys_ClearAuditHooks(void) { /* Must be finalizing to clear hooks */ _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *ts = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *ts = _PyThreadState_GET(); PyThreadState *finalizing = _PyRuntimeState_GetFinalizing(runtime); assert(!ts || finalizing == ts); if (!ts || finalizing != ts) { @@ -318,7 +318,7 @@ int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) { _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); /* Invoke existing audit hooks to allow them an opportunity to abort. */ /* Cannot invoke hooks until we are initialized */