8000 gh-105716: Support Background Threads in Subinterpreters Consistently by ericsnowcurrently · Pull Request #109921 · python/cpython · GitHub
[go: up one dir, main page]

Skip to content

gh-105716: Support Background Threads in Subinterpreters Consistently #109921

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
4e88c2b
Always wait for threads in subinterpreters.
ericsnowcurrently Sep 25, 2023
d08ac68
Add PyInterpreterState_*RunningMain().
ericsnowcurrently Sep 26, 2023
3b07482
Drop interpreter_exists().
ericsnowcurrently Sep 26, 2023
70c4753
Drop PyThreadState_IsRunning().
ericsnowcurrently Sep 26, 2023
678cb3b
Add a TODO.
ericsnowcurrently Sep 26, 2023
5be5a07
For now use the earliest thread state.
ericsnowcurrently Sep 26, 2023
81a1e0b
Add tests.
ericsnowcurrently Sep 26, 2023
c3f497b
Mark the main interpreter as running __main__.
ericsnowcurrently Sep 26, 2023
effa5b4
Call PyInterpreterState_SetNotRunningMain() in the right place.
ericsnowcurrently Sep 26, 2023
41c54b0
Check PyInterpreterState_IsRunningMain() preemptively, for now.
ericsnowcurrently Sep 26, 2023
5fbd940
Add docs.
ericsnowcurrently Sep 26, 2023
8b54c2e
Add a NEWS entry.
ericsnowcurrently Sep 26, 2023
5d1df61
Fix a typo.
ericsnowcurrently Sep 27, 2023
1034a67
Update the docs.
ericsnowcurrently Sep 27, 2023
8000
d57729e
Update the docs.
ericsnowcurrently Sep 27, 2023
4219e2c
Update the docs.
ericsnowcurrently Sep 27, 2023
bfdace2
Drop a dead line.
ericsnowcurrently Sep 27, 2023
ef8af92
Drop an unused parameter.
ericsnowcurrently Sep 27, 2023
566cd08
Drop an unnecessary variable.
ericsnowcurrently Sep 27, 2023
d14b87a
Update TODO comments.
ericsnowcurrently Sep 27, 2023
1f2a321
Merge branch 'main' into subinterpreters-allow-background-threads
ericsnowcurrently Sep 27, 2023
55d7090
Merge branch 'main' into subinterpreters-allow-background-threads
ericsnowcurrently Sep 29, 2023
399859f
Do not expose the "Running" C-API (for now).
ericsnowcurrently Oct 2, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8000
Prev Previous commit
Next Next commit
Add PyInterpreterState_*RunningMain().
  • Loading branch information
ericsnowcurrently committed Sep 26, 2023
commit d08ac68ded822c4043690523c72ea82afc6fd202
6 changes: 6 additions & 0 deletions Include/cpython/pystate.h
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,12 @@ struct _ts {
* if it is NULL. */
PyAPI_FUNC(PyThreadState *) _PyThreadState_UncheckedGet(void);

PyAPI_FUNC(int) PyInterpreterState_SetRunningMain(PyInterpreterState *);
PyAPI_FUNC(void) PyInterpreterState_SetNotRunningMain(PyInterpreterState *);
PyAPI_FUNC(int) PyInterpreterState_IsRunningMain(PyInterpreterState *);
PyAPI_FUNC(int) PyThreadState_IsRunning(PyThreadState *);


// Disable tracing and profiling.
PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate);

Expand Down
2 changes: 2 additions & 0 deletions Include/internal/pycore_interp.h
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ struct _is {
uint64_t next_unique_id;
/* The linked list of threads, newest first. */
PyThreadState *head;
/* The thread currently executing in the __main__ module, if any. */
PyThreadState *main;
/* Used in Modules/_threadmodule.c. */
long count;
/* Support for runtime thread stack size tuning.
Expand Down
67 changes: 21 additions & 46 deletions Modules/_xxsubinterpretersmodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,8 @@ _sharedexception_bind(PyObject *exc, _sharedexception *sharedexc)
}

static void
_sharedexception_apply(_sharedexception *exc, PyObject *wrapperclass)
_sharedexception_apply(_sharedexception *exc, PyObject *wrapperclass,
PyInterpreterState *interp)
{
if (exc->name != NULL) {
if (exc->msg != NULL) {
Expand Down Expand Up @@ -358,41 +359,14 @@ exceptions_init(PyObject *mod)
}

static int
_is_running(PyInterpreterState *interp)
{
PyThreadState *tstate = PyInterpreterState_ThreadHead(interp);
if (PyThreadState_Next(tstate) != NULL) {
PyErr_SetString(PyExc_RuntimeError,
"interpreter has more than one thread");
return -1;
}

assert(!PyErr_Occurred());
struct _PyInterpreterFrame *frame = tstate->current_frame;
if (frame == NULL) {
return 0;
}
return 1;
}

static int
_ensure_not_running(PyInterpreterState *interp)
_run_script(PyInterpreterState *interp, const char *codestr,
_sharedns *shared, _sharedexception *sharedexc)
{
int is_running = _is_running(interp);
if (is_running < 0) {
return -1;
}
if (is_running) {
PyErr_Format(PyExc_RuntimeError, "interpreter already running");
if (PyInterpreterState_SetRunningMain(interp) < 0) {
// We skip going through the shared exception.
return -1;
}
return 0;
}

static int
_run_script(PyInterpreterState *interp, const char *codestr,
_sharedns *shared, _sharedexception *sharedexc)
{
PyObject *excval = NULL;
PyObject *main_mod = PyUnstable_InterpreterState_GetMainModule(interp);
if (main_mod == NULL) {
Expand All @@ -415,6 +389,7 @@ _run_script(PyInterpreterState *interp, const char *codestr,

// Run the string (see PyRun_SimpleStringFlags).
PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL);
PyInterpreterState_SetNotRunningMain(interp);
Py_DECREF(ns);
if (result == NULL) {
goto error;
Expand All @@ -437,16 +412,14 @@ _run_script(PyInterpreterState *interp, const char *codestr,
}
Py_XDECREF(excval);
assert(!PyErr_Occurred());
PyInterpreterState_SetNotRunningMain(interp);
return -1;
}

static int
_run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp,
const char *codestr, PyObject *shareables)
{
if (_ensure_not_running(interp) < 0) {
return -1;
}
module_state *state = get_module_state(mod);

_sharedns *shared = _get_shared_ns(shareables);
Expand All @@ -456,9 +429,11 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp,

// Switch to interpreter.
PyThreadState *save_tstate = NULL;
if (interp != PyInterpreterState_Get()) {
PyThreadState *tcur = PyThreadState_Get();
PyThreadState *tstate = tcur;
if (interp != tstate->interp) {
// XXX Using the "head" thread isn't strictly correct.
PyThreadState *tstate = PyInterpreterState_ThreadHead(interp);
tstate = PyInterpreterState_ThreadHead(interp);
// XXX Possible GILState issues?
save_tstate = PyThreadState_Swap(tstate);
}
Expand All @@ -475,11 +450,13 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp,
// Propagate any exception out to the caller.
if (exc.name != NULL) {
assert(state != NULL);
_sharedexception_apply(&exc, state->RunFailedError);
_sharedexception_apply(&exc, state->RunFailedError, interp);
}
else if (result != 0) {
// We were unable to allocate a shared exception.
PyErr_NoMemory();
if (!PyErr_Occurred()) {
// We were unable to allocate a shared exception.
PyErr_NoMemory();
}
}

if (shared != NULL) {
Expand Down Expand Up @@ -574,7 +551,8 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds)
// Ensure the interpreter isn't running.
/* XXX We *could* support destroying a running interpreter but
aren't going to worry about it for now. */
if (_ensure_not_running(interp) < 0) {
if (PyInterpreterState_IsRunningMain(interp)) {
PyErr_Format(PyExc_RuntimeError, "interpreter running");
return NULL;
}

Expand Down Expand Up @@ -748,11 +726,7 @@ interp_is_running(PyObject *self, PyObject *args, PyObject *kwds)
if (interp == NULL) {
return NULL;
}
int is_running = _is_running(interp);
if (is_running < 0) {
return NULL;
}
if (is_running) {
if (PyInterpreterState_IsRunningMain(interp)) {
Py_RETURN_TRUE;
}
Py_RETURN_FALSE;
Expand All @@ -763,6 +737,7 @@ PyDoc_STRVAR(is_running_doc,
\n\
Return whether or not the identified interpreter is running.");


static PyMethodDef module_functions[] = {
{"create", _PyCFunction_CAST(interp_create),
METH_VARARGS | METH_KEYWORDS, create_doc},
Expand Down
71 changes: 71 additions & 0 deletions Python/pystate.c
Original file line number Diff line number Diff line change
Expand Up @@ -1095,6 +1095,50 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime)
#endif


int
PyInterpreterState_SetRunningMain(PyInterpreterState *interp)
{
if (interp->threads.main != NULL) {
PyErr_SetString(PyExc_RuntimeError,
"interpreter already running");
return -1;
}
PyThreadState *tstate = current_fast_get(&_PyRuntime);
_Py_EnsureTstateNotNULL(tstate);
if (tstate->interp != interp) {
PyErr_SetString(PyExc_RuntimeError,
"current tstate has wrong interpreter");
return -1;
}
interp->threads.main = tstate;
return 0;
}

void
PyInterpreterState_SetNotRunningMain(PyInterpreterState *interp)
{
assert(interp->threads.main == current_fast_get(&_PyRuntime));
interp->threads.main = NULL;
}

static int interpreter_exists(PyInterpreterState *);

int
PyInterpreterState_IsRunningMain(PyInterpreterState *interp)
{
HEAD_LOCK(interp->runtime);
int exists = interpreter_exists(interp);
HEAD_UNLOCK(interp->runtime);
if (!exists) {
return 0;
}
if (interp->threads.main == NULL) {
return 0;
}
return PyThreadState_IsRunning(interp->threads.main);
}


//----------
// accessors
//----------
Expand Down Expand Up @@ -1204,6 +1248,22 @@ PyInterpreterState_GetDict(PyInterpreterState *interp)
// look up an interpreter state
//-----------------------------

/* This must be called with the "head" lock held. */
static int
interpreter_exists(PyInterpreterState *interp)
{
PyInterpreterState *actual = interp->runtime->interpreters.head;
while (actual != NULL) {
if (actual == interp) {
return 1;
}
actual = actual->next;
}
// It's a dangling pointer.
return 0;
}


/* Return the interpreter associated with the current OS thread.

The GIL must be held.
Expand Down Expand Up @@ -1683,6 +1743,13 @@ _PyThreadState_DeleteExcept(PyThreadState *tstate)
}


int
PyThreadState_IsRunning(PyThreadState *tstate)
{
return tstate->current_frame != NULL;
}


//----------
// accessors
//----------
Expand Down Expand Up @@ -2805,6 +2872,10 @@ _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry)
}


/*************/
/* Other API */
/*************/

_PyFrameEvalFunction
_PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp)
{
Expand Down
0