From 247de37af954aa5a383cbf9273b3131feacfe0b5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 18 Feb 2019 16:51:52 -0700 Subject: [PATCH 01/15] Add _PyInterpreterState_GetMainModule(). --- Include/cpython/pystate.h | 1 + Modules/_xxsubinterpretersmodule.c | 2 +- Python/pystate.c | 10 ++++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 3fca78f9ddf238..2a288a9709c960 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -33,6 +33,7 @@ typedef struct { PyAPI_FUNC(_PyCoreConfig *) _PyInterpreterState_GetCoreConfig(PyInterpreterState *); PyAPI_FUNC(_PyMainInterpreterConfig *) _PyInterpreterState_GetMainConfig(PyInterpreterState *); +PyAPI_FUNC(PyObject *) _PyInterpreterState_GetMainModule(PyInterpreterState *); /* State unique per thread */ diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 79c9def7262913..e693d3a4060597 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1875,7 +1875,7 @@ _run_script(PyInterpreterState *interp, const char *codestr, PyObject *excval = NULL; PyObject *tb = NULL; - PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); + PyObject *main_mod = _PyInterpreterState_GetMainModule(interp); if (main_mod == NULL) { goto error; } diff --git a/Python/pystate.c b/Python/pystate.c index d612d3a171b620..e5f39b58852400 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -425,6 +425,16 @@ _PyInterpreterState_GetMainConfig(PyInterpreterState *interp) return &interp->config; } +PyObject * +_PyInterpreterState_GetMainModule(PyInterpreterState *interp) +{ + if (interp->modules == NULL) { + PyErr_SetString(PyExc_RuntimeError, "interpreter not initialized"); + return NULL; + } + return PyMapping_GetItemString(interp->modules, "__main__"); +} + /* Default implementation for _PyThreadState_GetFrame */ static struct _frame * threadstate_getframe(PyThreadState *self) From b498efa1fe5500bb99451ab6b6b0eb8cb4cdcc31 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 18 Feb 2019 16:55:13 -0700 Subject: [PATCH 02/15] Use PyInterpreterState_GetID(). --- Modules/_xxsubinterpretersmodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index e693d3a4060597..f5eb29af8f6353 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1345,7 +1345,7 @@ _channel_send(_channels *channels, int64_t id, PyObject *obj) } // Add the data to the channel. - int res = _channel_add(chan, interp->id, data); + int res = _channel_add(chan, PyInterpreterState_GetID(interp), data); PyThread_release_lock(mutex); if (res != 0) { _PyCrossInterpreterData_Release(data); @@ -1373,7 +1373,7 @@ _channel_recv(_channels *channels, int64_t id) // Past this point we are responsible for releasing the mutex. // Pop off the next item from the channel. - _PyCrossInterpreterData *data = _channel_next(chan, interp->id); + _PyCrossInterpreterData *data = _channel_next(chan, PyInterpreterState_GetID(interp)); PyThread_release_lock(mutex); if (data == NULL) { if (!PyErr_Occurred()) { @@ -1410,7 +1410,7 @@ _channel_drop(_channels *channels, int64_t id, int send, int recv) // Past this point we are responsible for releasing the mutex. // Close one or both of the two ends. - int res = _channel_close_interpreter(chan, interp->id, send-recv); + int res = _channel_close_interpreter(chan, PyInterpreterState_GetID(interp), send-recv); PyThread_release_lock(mutex); return res; } From 3732ddeef890a753d12b9a636f5da3848a08670c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 18 Feb 2019 16:57:02 -0700 Subject: [PATCH 03/15] Fix memory leaks. --- Modules/_xxsubinterpretersmodule.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index f5eb29af8f6353..c52b128fa9be45 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -71,6 +71,8 @@ struct _sharednsitem { _PyCrossInterpreterData data; }; +static void _sharednsitem_clear(struct _sharednsitem *); // forward + static int _sharednsitem_init(struct _sharednsitem *item, PyObject *key, PyObject *value) { @@ -79,6 +81,7 @@ _sharednsitem_init(struct _sharednsitem *item, PyObject *key, PyObject *value) return -1; } if (_PyObject_GetCrossInterpreterData(value, &item->data) != 0) { + _sharednsitem_clear(item); return -1; } return 0; @@ -89,6 +92,7 @@ _sharednsitem_clear(struct _sharednsitem *item) { if (item->name != NULL) { PyMem_Free(item->name); + item->name = NULL; } _PyCrossInterpreterData_Release(&item->data); } @@ -1339,8 +1343,8 @@ _channel_send(_channels *channels, int64_t id, PyObject *obj) return -1; } if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { - PyMem_Free(data); PyThread_release_lock(mutex); + PyMem_Free(data); return -1; } From 165e96cbd33c58d362f626a9e08a1ccb7926d423 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 18 Feb 2019 16:57:37 -0700 Subject: [PATCH 04/15] Move _PyCrossInterpreterData (and functions) to the public API. --- Include/cpython/pystate.h | 57 +++++++++++++++++++++++++++++++ Include/internal/pycore_pystate.h | 54 ----------------------------- 2 files changed, 57 insertions(+), 54 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 2a288a9709c960..3ded3446fe3fc9 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -215,6 +215,63 @@ PyAPI_FUNC(PyThreadState *) PyThreadState_Next(PyThreadState *); typedef struct _frame *(*PyThreadFrameGetter)(PyThreadState *self_); +/* cross-interpreter data */ + +struct _xid; + +// _PyCrossInterpreterData is similar to Py_buffer as an effectively +// opaque struct that holds data outside the object machinery. This +// is necessary to pass safely between interpreters in the same process. +typedef struct _xid { + // data is the cross-interpreter-safe derivation of a Python object + // (see _PyObject_GetCrossInterpreterData). It will be NULL if the + // new_object func (below) encodes the data. + void *data; + // obj is the Python object from which the data was derived. This + // is non-NULL only if the data remains bound to the object in some + // way, such that the object must be "released" (via a decref) when + // the data is released. In that case the code that sets the field, + // likely a registered "crossinterpdatafunc", is responsible for + // ensuring it owns the reference (i.e. incref). + PyObject *obj; + // interp is the ID of the owning interpreter of the original + // object. It corresponds to the active interpreter when + // _PyObject_GetCrossInterpreterData() was called. This should only + // be set by the cross-interpreter machinery. + // + // We use the ID rather than the PyInterpreterState to avoid issues + // with deleted interpreters. + int64_t interp; + // new_object is a function that returns a new object in the current + // interpreter given the data. The resulting object (a new + // reference) will be equivalent to the original object. This field + // is required. + PyObject *(*new_object)(struct _xid *); + // free is called when the data is released. If it is NULL then + // nothing will be done to free the data. For some types this is + // okay (e.g. bytes) and for those types this field should be set + // to NULL. However, for most the data was allocated just for + // cross-interpreter use, so it must be freed when + // _PyCrossInterpreterData_Release is called or the memory will + // leak. In that case, at the very least this field should be set + // to PyMem_RawFree (the default if not explicitly set to NULL). + // The call will happen with the original interpreter activated. + void (*free)(void *); +} _PyCrossInterpreterData; + +PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); +PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); +PyAPI_FUNC(void) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); + +PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); + +/* cross-interpreter data registry */ + +typedef int (*crossinterpdatafunc)(PyObject *, struct _xid *); + +PyAPI_FUNC(int) _PyCrossInterpreterData_Register_Class(PyTypeObject *, crossinterpdatafunc); +PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index f6c61e7bcbd6d5..9b57c57b02dd82 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -92,66 +92,12 @@ PyAPI_FUNC(void) _PyInterpreterState_IDIncref(struct _is *); PyAPI_FUNC(void) _PyInterpreterState_IDDecref(struct _is *); -/* cross-interpreter data */ - -struct _xid; - -// _PyCrossInterpreterData is similar to Py_buffer as an effectively -// opaque struct that holds data outside the object machinery. This -// is necessary to pass safely between interpreters in the same process. -typedef struct _xid { - // data is the cross-interpreter-safe derivation of a Python object - // (see _PyObject_GetCrossInterpreterData). It will be NULL if the - // new_object func (below) encodes the data. - void *data; - // obj is the Python object from which the data was derived. This - // is non-NULL only if the data remains bound to the object in some - // way, such that the object must be "released" (via a decref) when - // the data is released. In that case the code that sets the field, - // likely a registered "crossinterpdatafunc", is responsible for - // ensuring it owns the reference (i.e. incref). - PyObject *obj; - // interp is the ID of the owning interpreter of the original - // object. It corresponds to the active interpreter when - // _PyObject_GetCrossInterpreterData() was called. This should only - // be set by the cross-interpreter machinery. - // - // We use the ID rather than the PyInterpreterState to avoid issues - // with deleted interpreters. - int64_t interp; - // new_object is a function that returns a new object in the current - // interpreter given the data. The resulting object (a new - // reference) will be equivalent to the original object. This field - // is required. - PyObject *(*new_object)(struct _xid *); - // free is called when the data is released. If it is NULL then - // nothing will be done to free the data. For some types this is - // okay (e.g. bytes) and for those types this field should be set - // to NULL. However, for most the data was allocated just for - // cross-interpreter use, so it must be freed when - // _PyCrossInterpreterData_Release is called or the memory will - // leak. In that case, at the very least this field should be set - // to PyMem_RawFree (the default if not explicitly set to NULL). - // The call will happen with the original interpreter activated. - void (*free)(void *); -} _PyCrossInterpreterData; - -typedef int (*crossinterpdatafunc)(PyObject *, _PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); - -PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); -PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); -PyAPI_FUNC(void) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); - /* cross-interpreter data registry */ /* For now we use a global registry of shareable classes. An alternative would be to add a tp_* slot for a class's crossinterpdatafunc. It would be simpler and more efficient. */ -PyAPI_FUNC(int) _PyCrossInterpreterData_Register_Class(PyTypeObject *, crossinterpdatafunc); -PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); - struct _xidregitem; struct _xidregitem { From 892ea20bcdc19786151ff874010389e7263519b3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 18 Feb 2019 18:01:31 -0700 Subject: [PATCH 05/15] Add _PyInterpreterID_Type and helpers to public API. --- Include/cpython/interpreteridobject.h | 13 + Include/interpreteridobject.h | 17 ++ Makefile.pre.in | 1 + Modules/_xxsubinterpretersmodule.c | 334 ++------------------------ Objects/interpreteridobject.c | 305 +++++++++++++++++++++++ 5 files changed, 354 insertions(+), 316 deletions(-) create mode 100644 Include/cpython/interpreteridobject.h create mode 100644 Include/interpreteridobject.h create mode 100644 Objects/interpreteridobject.c diff --git a/Include/cpython/interpreteridobject.h b/Include/cpython/interpreteridobject.h new file mode 100644 index 00000000000000..04e38f7e8214a1 --- /dev/null +++ b/Include/cpython/interpreteridobject.h @@ -0,0 +1,13 @@ +#ifndef Py_CPYTHON_INTERPRETERIDOBJECT_H +# error "this header file must not be included directly" +#endif + +/* Interpreter ID Object */ + +PyAPI_DATA(PyTypeObject) _PyInterpreterID_Type; + +PyAPI_FUNC(PyObject *) _PyInterpreterID_New(int64_t); +PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *); +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterID_LookUp(PyObject *); + +PyAPI_FUNC(int64_t) _Py_CoerceID(PyObject *); diff --git a/Include/interpreteridobject.h b/Include/interpreteridobject.h new file mode 100644 index 00000000000000..e744fcdc9ff189 --- /dev/null +++ b/Include/interpreteridobject.h @@ -0,0 +1,17 @@ +#ifndef Py_INTERPRETERIDOBJECT_H +#define Py_INTERPRETERIDOBJECT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_INTERPRETERIDOBJECT_H +# include "cpython/interpreteridobject.h" +# undef Py_CPYTHON_INTERPRETERIDOBJECT_H +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERPRETERIDOBJECT_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 5ae69480bf16a4..6590377fe07311 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -410,6 +410,7 @@ OBJECT_OBJS= \ Objects/floatobject.o \ Objects/frameobject.o \ Objects/funcobject.o \ + Objects/interpreteridobject.o \ Objects/iterobject.o \ Objects/listobject.o \ Objects/longobject.o \ diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index c52b128fa9be45..d02a313a4ec14d 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -4,7 +4,7 @@ #include "Python.h" #include "frameobject.h" -#include "pycore_pystate.h" +#include "interpreteridobject.h" static char * @@ -31,38 +31,6 @@ _get_current(void) return _PyInterpreterState_Get(); } -static int64_t -_coerce_id(PyObject *orig) -{ - PyObject *pyid = PyNumber_Long(orig); - if (pyid == NULL) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_Format(PyExc_TypeError, - "'id' must be a non-negative int, got %R", orig); - } - else { - PyErr_Format(PyExc_ValueError, - "'id' must be a non-negative int, got %R", orig); - } - return -1; - } - int64_t id = PyLong_AsLongLong(pyid); - Py_DECREF(pyid); - if (id == -1 && PyErr_Occurred() != NULL) { - if (!PyErr_ExceptionMatches(PyExc_OverflowError)) { - PyErr_Format(PyExc_ValueError, - "'id' must be a non-negative int, got %R", orig); - } - return -1; - } - if (id < 0) { - PyErr_Format(PyExc_ValueError, - "'id' must be a non-negative int, got %R", orig); - return -1; - } - return id; -} - /* data-sharing-specific code ***********************************************/ @@ -1485,7 +1453,7 @@ channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) cid = ((channelid *)id)->id; } else { - cid = _coerce_id(id); + cid = _Py_CoerceID(id); if (cid < 0) { return NULL; } @@ -1978,272 +1946,6 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, return result; } -/* InterpreterID class */ - -static PyTypeObject InterpreterIDtype; - -typedef struct interpid { - PyObject_HEAD - int64_t id; -} interpid; - -static interpid * -newinterpid(PyTypeObject *cls, int64_t id, int force) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpID(id); - if (interp == NULL) { - if (force) { - PyErr_Clear(); - } - else { - return NULL; - } - } - - interpid *self = PyObject_New(interpid, cls); - if (self == NULL) { - return NULL; - } - self->id = id; - - if (interp != NULL) { - _PyInterpreterState_IDIncref(interp); - } - return self; -} - -static PyObject * -interpid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"id", "force", NULL}; - PyObject *idobj; - int force = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|$p:InterpreterID.__init__", kwlist, - &idobj, &force)) { - return NULL; - } - - // Coerce and check the ID. - int64_t id; - if (PyObject_TypeCheck(idobj, &InterpreterIDtype)) { - id = ((interpid *)idobj)->id; - } - else { - id = _coerce_id(idobj); - if (id < 0) { - return NULL; - } - } - - return (PyObject *)newinterpid(cls, id, force); -} - -static void -interpid_dealloc(PyObject *v) -{ - int64_t id = ((interpid *)v)->id; - PyInterpreterState *interp = _PyInterpreterState_LookUpID(id); - if (interp != NULL) { - _PyInterpreterState_IDDecref(interp); - } - else { - // already deleted - PyErr_Clear(); - } - Py_TYPE(v)->tp_free(v); -} - -static PyObject * -interpid_repr(PyObject *self) -{ - PyTypeObject *type = Py_TYPE(self); - const char *name = _PyType_Name(type); - interpid *id = (interpid *)self; - return PyUnicode_FromFormat("%s(%" PRId64 ")", name, id->id); -} - -static PyObject * -interpid_str(PyObject *self) -{ - interpid *id = (interpid *)self; - return PyUnicode_FromFormat("%" PRId64 "", id->id); -} - -PyObject * -interpid_int(PyObject *self) -{ - interpid *id = (interpid *)self; - return PyLong_FromLongLong(id->id); -} - -static PyNumberMethods interpid_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - 0, /* nb_bool */ - 0, /* nb_invert */ - 0, /* nb_lshift */ - 0, /* nb_rshift */ - 0, /* nb_and */ - 0, /* nb_xor */ - 0, /* nb_or */ - (unaryfunc)interpid_int, /* nb_int */ - 0, /* nb_reserved */ - 0, /* nb_float */ - - 0, /* nb_inplace_add */ - 0, /* nb_inplace_subtract */ - 0, /* nb_inplace_multiply */ - 0, /* nb_inplace_remainder */ - 0, /* nb_inplace_power */ - 0, /* nb_inplace_lshift */ - 0, /* nb_inplace_rshift */ - 0, /* nb_inplace_and */ - 0, /* nb_inplace_xor */ - 0, /* nb_inplace_or */ - - 0, /* nb_floor_divide */ - 0, /* nb_true_divide */ - 0, /* nb_inplace_floor_divide */ - 0, /* nb_inplace_true_divide */ - - (unaryfunc)interpid_int, /* nb_index */ -}; - -static Py_hash_t -interpid_hash(PyObject *self) -{ - interpid *id = (interpid *)self; - PyObject *obj = PyLong_FromLongLong(id->id); - if (obj == NULL) { - return -1; - } - Py_hash_t hash = PyObject_Hash(obj); - Py_DECREF(obj); - return hash; -} - -static PyObject * -interpid_richcompare(PyObject *self, PyObject *other, int op) -{ - if (op != Py_EQ && op != Py_NE) { - Py_RETURN_NOTIMPLEMENTED; - } - - if (!PyObject_TypeCheck(self, &InterpreterIDtype)) { - Py_RETURN_NOTIMPLEMENTED; - } - - interpid *id = (interpid *)self; - int equal; - if (PyObject_TypeCheck(other, &InterpreterIDtype)) { - interpid *otherid = (interpid *)other; - equal = (id->id == otherid->id); - } - else { - other = PyNumber_Long(other); - if (other == NULL) { - PyErr_Clear(); - Py_RETURN_NOTIMPLEMENTED; - } - int64_t otherid = PyLong_AsLongLong(other); - Py_DECREF(other); - if (otherid == -1 && PyErr_Occurred() != NULL) { - return NULL; - } - if (otherid < 0) { - equal = 0; - } - else { - equal = (id->id == otherid); - } - } - - if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; -} - -PyDoc_STRVAR(interpid_doc, -"A interpreter ID identifies a interpreter and may be used as an int."); - -static PyTypeObject InterpreterIDtype = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "interpreters.InterpreterID", /* tp_name */ - sizeof(interpid), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)interpid_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)interpid_repr, /* tp_repr */ - &interpid_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - interpid_hash, /* tp_hash */ - 0, /* tp_call */ - (reprfunc)interpid_str, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_LONG_SUBCLASS, /* tp_flags */ - interpid_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - interpid_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - interpid_new, /* tp_new */ -}; - -static PyObject * -_get_id(PyInterpreterState *interp) -{ - PY_INT64_T id = PyInterpreterState_GetID(interp); - if (id < 0) { - return NULL; - } - return (PyObject *)newinterpid(&InterpreterIDtype, id, 0); -} - -static PyInterpreterState * -_look_up(PyObject *requested_id) -{ - int64_t id; - if (PyObject_TypeCheck(requested_id, &InterpreterIDtype)) { - id = ((interpid *)requested_id)->id; - } - else { - id = PyLong_AsLongLong(requested_id); - if (id == -1 && PyErr_Occurred() != NULL) { - return NULL; - } - assert(id <= INT64_MAX); - } - return _PyInterpreterState_LookUpID(id); -} - /* module level code ********************************************************/ @@ -2290,7 +1992,7 @@ interp_create(PyObject *self, PyObject *args) if (_PyInterpreterState_IDInitref(tstate->interp) != 0) { goto error; }; - return _get_id(tstate->interp); + return _PyInterpreterState_GetIDObject(tstate->interp); error: // XXX Possible GILState issues? @@ -2322,7 +2024,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) } // Look up the interpreter. - PyInterpreterState *interp = _look_up(id); + PyInterpreterState *interp = _PyInterpreterID_LookUp(id); if (interp == NULL) { return NULL; } @@ -2378,7 +2080,7 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) interp = PyInterpreterState_Head(); while (interp != NULL) { - id = _get_id(interp); + id = _PyInterpreterState_GetIDObject(interp); if (id == NULL) { Py_DECREF(ids); return NULL; @@ -2410,7 +2112,7 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) if (interp == NULL) { return NULL; } - return _get_id(interp); + return _PyInterpreterState_GetIDObject(interp); } PyDoc_STRVAR(get_current_doc, @@ -2424,7 +2126,7 @@ interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored)) { // Currently, 0 is always the main interpreter. PY_INT64_T id = 0; - return (PyObject *)newinterpid(&InterpreterIDtype, id, 0); + return _PyInterpreterID_New(id); } PyDoc_STRVAR(get_main_doc, @@ -2450,7 +2152,7 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) } // Look up the interpreter. - PyInterpreterState *interp = _look_up(id); + PyInterpreterState *interp = _PyInterpreterID_LookUp(id); if (interp == NULL) { return NULL; } @@ -2520,7 +2222,7 @@ interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyInterpreterState *interp = _look_up(id); + PyInterpreterState *interp = _PyInterpreterID_LookUp(id); if (interp == NULL) { return NULL; } @@ -2572,7 +2274,7 @@ channel_destroy(PyObject *self, PyObject *args, PyObject *kwds) "O:channel_destroy", kwlist, &id)) { return NULL; } - int64_t cid = _coerce_id(id); + int64_t cid = _Py_CoerceID(id); if (cid < 0) { return NULL; } @@ -2636,7 +2338,7 @@ channel_send(PyObject *self, PyObject *args, PyObject *kwds) "OO:channel_send", kwlist, &id, &obj)) { return NULL; } - int64_t cid = _coerce_id(id); + int64_t cid = _Py_CoerceID(id); if (cid < 0) { return NULL; } @@ -2661,7 +2363,7 @@ channel_recv(PyObject *self, PyObject *args, PyObject *kwds) "O:channel_recv", kwlist, &id)) { return NULL; } - int64_t cid = _coerce_id(id); + int64_t cid = _Py_CoerceID(id); if (cid < 0) { return NULL; } @@ -2687,7 +2389,7 @@ channel_close(PyObject *self, PyObject *args, PyObject *kwds) &id, &send, &recv, &force)) { return NULL; } - int64_t cid = _coerce_id(id); + int64_t cid = _Py_CoerceID(id); if (cid < 0) { return NULL; } @@ -2739,7 +2441,7 @@ channel_release(PyObject *self, PyObject *args, PyObject *kwds) &id, &send, &recv, &force)) { return NULL; } - int64_t cid = _coerce_id(id); + int64_t cid = _Py_CoerceID(id); if (cid < 0) { return NULL; } @@ -2841,8 +2543,8 @@ PyInit__xxsubinterpreters(void) if (PyType_Ready(&ChannelIDtype) != 0) { return NULL; } - InterpreterIDtype.tp_base = &PyLong_Type; - if (PyType_Ready(&InterpreterIDtype) != 0) { + _PyInterpreterID_Type.tp_base = &PyLong_Type; + if (PyType_Ready(&_PyInterpreterID_Type) != 0) { return NULL; } @@ -2866,8 +2568,8 @@ PyInit__xxsubinterpreters(void) if (PyDict_SetItemString(ns, "ChannelID", (PyObject *)&ChannelIDtype) != 0) { return NULL; } - Py_INCREF(&InterpreterIDtype); - if (PyDict_SetItemString(ns, "InterpreterID", (PyObject *)&InterpreterIDtype) != 0) { + Py_INCREF(&_PyInterpreterID_Type); + if (PyDict_SetItemString(ns, "InterpreterID", (PyObject *)&_PyInterpreterID_Type) != 0) { return NULL; } diff --git a/Objects/interpreteridobject.c b/Objects/interpreteridobject.c new file mode 100644 index 00000000000000..c6cbc827070e55 --- /dev/null +++ b/Objects/interpreteridobject.c @@ -0,0 +1,305 @@ +/* InterpreterID object */ + +#include "Python.h" +#include "internal/pycore_pystate.h" +#include "interpreteridobject.h" + + +int64_t +_Py_CoerceID(PyObject *orig) +{ + PyObject *pyid = PyNumber_Long(orig); + if (pyid == NULL) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Format(PyExc_TypeError, + "'id' must be a non-negative int, got %R", orig); + } + else { + PyErr_Format(PyExc_ValueError, + "'id' must be a non-negative int, got %R", orig); + } + return -1; + } + int64_t id = PyLong_AsLongLong(pyid); + Py_DECREF(pyid); + if (id == -1 && PyErr_Occurred() != NULL) { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_Format(PyExc_ValueError, + "'id' must be a non-negative int, got %R", orig); + } + return -1; + } + if (id < 0) { + PyErr_Format(PyExc_ValueError, + "'id' must be a non-negative int, got %R", orig); + return -1; + } + return id; +} + +typedef struct interpid { + PyObject_HEAD + int64_t id; +} interpid; + +static interpid * +newinterpid(PyTypeObject *cls, int64_t id, int force) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpID(id); + if (interp == NULL) { + if (force) { + PyErr_Clear(); + } + else { + return NULL; + } + } + + interpid *self = PyObject_New(interpid, cls); + if (self == NULL) { + return NULL; + } + self->id = id; + + if (interp != NULL) { + _PyInterpreterState_IDIncref(interp); + } + return self; +} + +static PyObject * +interpid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "force", NULL}; + PyObject *idobj; + int force = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O|$p:InterpreterID.__init__", kwlist, + &idobj, &force)) { + return NULL; + } + + // Coerce and check the ID. + int64_t id; + if (PyObject_TypeCheck(idobj, &_PyInterpreterID_Type)) { + id = ((interpid *)idobj)->id; + } + else { + id = _Py_CoerceID(idobj); + if (id < 0) { + return NULL; + } + } + + return (PyObject *)newinterpid(cls, id, force); +} + +static void +interpid_dealloc(PyObject *v) +{ + int64_t id = ((interpid *)v)->id; + PyInterpreterState *interp = _PyInterpreterState_LookUpID(id); + if (interp != NULL) { + _PyInterpreterState_IDDecref(interp); + } + else { + // already deleted + PyErr_Clear(); + } + Py_TYPE(v)->tp_free(v); +} + +static PyObject * +interpid_repr(PyObject *self) +{ + PyTypeObject *type = Py_TYPE(self); + const char *name = _PyType_Name(type); + interpid *id = (interpid *)self; + return PyUnicode_FromFormat("%s(%" PRId64 ")", name, id->id); +} + +static PyObject * +interpid_str(PyObject *self) +{ + interpid *id = (interpid *)self; + return PyUnicode_FromFormat("%" PRId64 "", id->id); +} + +PyObject * +interpid_int(PyObject *self) +{ + interpid *id = (interpid *)self; + return PyLong_FromLongLong(id->id); +} + +static PyNumberMethods interpid_as_number = { + 0, /* nb_add */ + 0, /* nb_subtract */ + 0, /* nb_multiply */ + 0, /* nb_remainder */ + 0, /* nb_divmod */ + 0, /* nb_power */ + 0, /* nb_negative */ + 0, /* nb_positive */ + 0, /* nb_absolute */ + 0, /* nb_bool */ + 0, /* nb_invert */ + 0, /* nb_lshift */ + 0, /* nb_rshift */ + 0, /* nb_and */ + 0, /* nb_xor */ + 0, /* nb_or */ + (unaryfunc)interpid_int, /* nb_int */ + 0, /* nb_reserved */ + 0, /* nb_float */ + + 0, /* nb_inplace_add */ + 0, /* nb_inplace_subtract */ + 0, /* nb_inplace_multiply */ + 0, /* nb_inplace_remainder */ + 0, /* nb_inplace_power */ + 0, /* nb_inplace_lshift */ + 0, /* nb_inplace_rshift */ + 0, /* nb_inplace_and */ + 0, /* nb_inplace_xor */ + 0, /* nb_inplace_or */ + + 0, /* nb_floor_divide */ + 0, /* nb_true_divide */ + 0, /* nb_inplace_floor_divide */ + 0, /* nb_inplace_true_divide */ + + (unaryfunc)interpid_int, /* nb_index */ +}; + +static Py_hash_t +interpid_hash(PyObject *self) +{ + interpid *id = (interpid *)self; + PyObject *obj = PyLong_FromLongLong(id->id); + if (obj == NULL) { + return -1; + } + Py_hash_t hash = PyObject_Hash(obj); + Py_DECREF(obj); + return hash; +} + +static PyObject * +interpid_richcompare(PyObject *self, PyObject *other, int op) +{ + if (op != Py_EQ && op != Py_NE) { + Py_RETURN_NOTIMPLEMENTED; + } + + if (!PyObject_TypeCheck(self, &_PyInterpreterID_Type)) { + Py_RETURN_NOTIMPLEMENTED; + } + + interpid *id = (interpid *)self; + int equal; + if (PyObject_TypeCheck(other, &_PyInterpreterID_Type)) { + interpid *otherid = (interpid *)other; + equal = (id->id == otherid->id); + } + else { + other = PyNumber_Long(other); + if (other == NULL) { + PyErr_Clear(); + Py_RETURN_NOTIMPLEMENTED; + } + int64_t otherid = PyLong_AsLongLong(other); + Py_DECREF(other); + if (otherid == -1 && PyErr_Occurred() != NULL) { + return NULL; + } + if (otherid < 0) { + equal = 0; + } + else { + equal = (id->id == otherid); + } + } + + if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(interpid_doc, +"A interpreter ID identifies a interpreter and may be used as an int."); + +PyTypeObject _PyInterpreterID_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "interpreters.InterpreterID", /* tp_name */ + sizeof(interpid), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)interpid_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + (reprfunc)interpid_repr, /* tp_repr */ + &interpid_as_number, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + interpid_hash, /* tp_hash */ + 0, /* tp_call */ + (reprfunc)interpid_str, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_LONG_SUBCLASS, /* tp_flags */ + interpid_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + interpid_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + interpid_new, /* tp_new */ +}; + +PyObject *_PyInterpreterID_New(int64_t id) +{ + return (PyObject *)newinterpid(&_PyInterpreterID_Type, id, 0); +} + +PyObject * +_PyInterpreterState_GetIDObject(PyInterpreterState *interp) +{ + PY_INT64_T id = PyInterpreterState_GetID(interp); + if (id < 0) { + return NULL; + } + return (PyObject *)newinterpid(&_PyInterpreterID_Type, id, 0); +} + +PyInterpreterState * +_PyInterpreterID_LookUp(PyObject *requested_id) +{ + int64_t id; + if (PyObject_TypeCheck(requested_id, &_PyInterpreterID_Type)) { + id = ((interpid *)requested_id)->id; + } + else { + id = PyLong_AsLongLong(requested_id); + if (id == -1 && PyErr_Occurred() != NULL) { + return NULL; + } + assert(id <= INT64_MAX); + } + return _PyInterpreterState_LookUpID(id); +} From b40080096ffaa76249091ae4c209b9d5ccb5c604 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 18 Feb 2019 18:14:01 -0700 Subject: [PATCH 06/15] Initialize the ID refcount in _PyInterpreterState_GetIDObject(). --- Modules/_xxsubinterpretersmodule.c | 20 +++++++++----------- Objects/interpreteridobject.c | 3 +++ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index d02a313a4ec14d..3c616addf821d6 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1989,17 +1989,15 @@ interp_create(PyObject *self, PyObject *args) PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); return NULL; } - if (_PyInterpreterState_IDInitref(tstate->interp) != 0) { - goto error; - }; - return _PyInterpreterState_GetIDObject(tstate->interp); - -error: - // XXX Possible GILState issues? - save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); - return NULL; + PyObject *idobj = _PyInterpreterState_GetIDObject(tstate->interp); + if (idobj == NULL) { + // XXX Possible GILState issues? + save_tstate = PyThreadState_Swap(tstate); + Py_EndInterpreter(tstate); + PyThreadState_Swap(save_tstate); + return NULL; + } + return idobj; } PyDoc_STRVAR(create_doc, diff --git a/Objects/interpreteridobject.c b/Objects/interpreteridobject.c index c6cbc827070e55..46fd2c0c65a02b 100644 --- a/Objects/interpreteridobject.c +++ b/Objects/interpreteridobject.c @@ -280,6 +280,9 @@ PyObject *_PyInterpreterID_New(int64_t id) PyObject * _PyInterpreterState_GetIDObject(PyInterpreterState *interp) { + if (_PyInterpreterState_IDInitref(interp) != 0) { + return NULL; + }; PY_INT64_T id = PyInterpreterState_GetID(interp); if (id < 0) { return NULL; From 582e52043f89393b6e11e99bab7bd4a86adf4ec3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 23 Feb 2019 12:29:31 -0800 Subject: [PATCH 07/15] Fix InterpreterID intialization. --- Makefile.pre.in | 1 + Objects/interpreteridobject.c | 2 +- Objects/object.c | 2 ++ setup.py | 4 +--- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 6590377fe07311..6e1b32280e1dff 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1010,6 +1010,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/funcobject.h \ $(srcdir)/Include/genobject.h \ $(srcdir)/Include/import.h \ + $(srcdir)/Include//interpreteridobject.h \ $(srcdir)/Include/intrcheck.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ diff --git a/Objects/interpreteridobject.c b/Objects/interpreteridobject.c index 46fd2c0c65a02b..464b19ddf44048 100644 --- a/Objects/interpreteridobject.c +++ b/Objects/interpreteridobject.c @@ -232,7 +232,7 @@ PyDoc_STRVAR(interpid_doc, PyTypeObject _PyInterpreterID_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "interpreters.InterpreterID", /* tp_name */ + "InterpreterID", /* tp_name */ sizeof(interpid), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)interpid_dealloc, /* tp_dealloc */ diff --git a/Objects/object.c b/Objects/object.c index cf5264b5d80baf..b446d598130a25 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -5,6 +5,7 @@ #include "pycore_pystate.h" #include "pycore_context.h" #include "frameobject.h" +#include "interpreteridobject.h" #ifdef __cplusplus extern "C" { @@ -1806,6 +1807,7 @@ _PyTypes_Init(void) INIT_TYPE(&PySeqIter_Type, "sequence iterator"); INIT_TYPE(&PyCoro_Type, "coroutine"); INIT_TYPE(&_PyCoroWrapper_Type, "coroutine wrapper"); + INIT_TYPE(&_PyInterpreterID_Type, "interpreter ID"); return _Py_INIT_OK(); #undef INIT_TYPE diff --git a/setup.py b/setup.py index b96961073b987b..c278f08b8e6d2f 100644 --- a/setup.py +++ b/setup.py @@ -784,9 +784,7 @@ def detect_simple_extensions(self): self.add(Extension('syslog', ['syslogmodule.c'])) # Python interface to subinterpreter C-API. - self.add(Extension('_xxsubinterpreters', - ['_xxsubinterpretersmodule.c'], - define_macros=[('Py_BUILD_CORE', '')])) + self.add(Extension('_xxsubinterpreters', ['_xxsubinterpretersmodule.c'])) # # Here ends the simple stuff. From here on, modules need certain From 586d2bfe339d43a63c1a93cefac3a1015dd6c3b4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 1 Mar 2019 14:17:00 -0700 Subject: [PATCH 08/15] Fix the makefile. --- Makefile.pre.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 6e1b32280e1dff..0a981abd73d6b9 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1010,7 +1010,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/funcobject.h \ $(srcdir)/Include/genobject.h \ $(srcdir)/Include/import.h \ - $(srcdir)/Include//interpreteridobject.h \ + $(srcdir)/Include/interpreteridobject.h \ $(srcdir)/Include/intrcheck.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ @@ -1075,6 +1075,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/abstract.h \ $(srcdir)/Include/cpython/coreconfig.h \ $(srcdir)/Include/cpython/dictobject.h \ + $(srcdir)/Include/cpython/interpreteridobject.h \ $(srcdir)/Include/cpython/object.h \ $(srcdir)/Include/cpython/objimpl.h \ $(srcdir)/Include/cpython/pyerrors.h \ From 405f0242b7a5d62ce2f2ce699f60767225fd5493 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 26 Feb 2019 09:27:28 -0700 Subject: [PATCH 09/15] Make a function static. --- Objects/interpreteridobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/interpreteridobject.c b/Objects/interpreteridobject.c index 464b19ddf44048..4901788ac39ed5 100644 --- a/Objects/interpreteridobject.c +++ b/Objects/interpreteridobject.c @@ -125,7 +125,7 @@ interpid_str(PyObject *self) return PyUnicode_FromFormat("%" PRId64 "", id->id); } -PyObject * +static PyObject * interpid_int(PyObject *self) { interpid *id = (interpid *)self; From 1ab3501a29d3f6a15cfeb97d11c454438d851b83 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 1 Mar 2019 14:54:14 -0700 Subject: [PATCH 10/15] Do not call PyType_Ready() twice. --- Modules/_xxsubinterpretersmodule.c | 4 ---- Objects/interpreteridobject.c | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 3c616addf821d6..6576bf5c1ce714 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -2541,10 +2541,6 @@ PyInit__xxsubinterpreters(void) if (PyType_Ready(&ChannelIDtype) != 0) { return NULL; } - _PyInterpreterID_Type.tp_base = &PyLong_Type; - if (PyType_Ready(&_PyInterpreterID_Type) != 0) { - return NULL; - } /* Create the module */ PyObject *module = PyModule_Create(&interpretersmodule); diff --git a/Objects/interpreteridobject.c b/Objects/interpreteridobject.c index 4901788ac39ed5..dd142b043d0ae7 100644 --- a/Objects/interpreteridobject.c +++ b/Objects/interpreteridobject.c @@ -262,7 +262,7 @@ PyTypeObject _PyInterpreterID_Type = { 0, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ - 0, /* tp_base */ + &PyLong_Type, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ From ba9b3686efd1c872b5dbbccd5924945b9eab0ee8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sun, 24 Feb 2019 15:35:08 -0800 Subject: [PATCH 11/15] Fix the Windows build. --- PCbuild/pythoncore.vcxproj | 2 ++ PCbuild/pythoncore.vcxproj.filters | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 992c84f8cc81ee..0bfae52ea39ac9 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -154,6 +154,7 @@ + @@ -352,6 +353,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 293bded66c3165..e5165f3a0a6b05 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -489,6 +489,9 @@ Include + + Include + Modules @@ -1061,6 +1064,9 @@ Objects + + Objects + Modules From 5ad779d3486e4908a4820705a74dcb5f1b466081 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 26 Feb 2019 09:32:08 -0700 Subject: [PATCH 12/15] _PyCrossInterpreterData_Register_Class -> _PyCrossInterpreterData_RegisterClass. --- Include/cpython/pystate.h | 2 +- Modules/_xxsubinterpretersmodule.c | 2 +- Python/pystate.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 3ded3446fe3fc9..96ffb2886a6573 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -269,7 +269,7 @@ PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); typedef int (*crossinterpdatafunc)(PyObject *, struct _xid *); -PyAPI_FUNC(int) _PyCrossInterpreterData_Register_Class(PyTypeObject *, crossinterpdatafunc); +PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); #ifdef __cplusplus diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 6576bf5c1ce714..bba53d68b21636 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -2567,7 +2567,7 @@ PyInit__xxsubinterpreters(void) return NULL; } - if (_PyCrossInterpreterData_Register_Class(&ChannelIDtype, _channelid_shared)) { + if (_PyCrossInterpreterData_RegisterClass(&ChannelIDtype, _channelid_shared)) { return NULL; } diff --git a/Python/pystate.c b/Python/pystate.c index e5f39b58852400..fc2acca4b55e7a 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1380,7 +1380,7 @@ _register_xidata(PyTypeObject *cls, crossinterpdatafunc getdata) static void _register_builtins_for_crossinterpreter_data(void); int -_PyCrossInterpreterData_Register_Class(PyTypeObject *cls, +_PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, crossinterpdatafunc getdata) { if (!PyType_Check(cls)) { From c7aaefd41c7eb25d6d9744a8903510297354899c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 26 Feb 2019 09:40:01 -0700 Subject: [PATCH 13/15] Clarify a comment. --- Include/cpython/pystate.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 96ffb2886a6573..ab4a9375205b2c 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -240,7 +240,9 @@ typedef struct _xid { // be set by the cross-interpreter machinery. // // We use the ID rather than the PyInterpreterState to avoid issues - // with deleted interpreters. + // with deleted interpreters. Note that IDs are never re-used, so + // each one will always correspond to a specific interpreter + // (whether still alive or not). int64_t interp; // new_object is a function that returns a new object in the current // interpreter given the data. The resulting object (a new From b0b4ddb9d0b34d4cd43b7c7dbee5fb9967242fc7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 26 Feb 2019 09:44:07 -0700 Subject: [PATCH 14/15] Add the C++ guard. --- Include/cpython/interpreteridobject.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Include/cpython/interpreteridobject.h b/Include/cpython/interpreteridobject.h index 04e38f7e8214a1..cb72c2b0895a6c 100644 --- a/Include/cpython/interpreteridobject.h +++ b/Include/cpython/interpreteridobject.h @@ -2,6 +2,10 @@ # error "this header file must not be included directly" #endif +#ifdef __cplusplus +extern "C" { +#endif + /* Interpreter ID Object */ PyAPI_DATA(PyTypeObject) _PyInterpreterID_Type; @@ -11,3 +15,7 @@ PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *); PyAPI_FUNC(PyInterpreterState *) _PyInterpreterID_LookUp(PyObject *); PyAPI_FUNC(int64_t) _Py_CoerceID(PyObject *); + +#ifdef __cplusplus +} +#endif From 16c0b7b97d04d026fb86faa5b4ac0775b2b5f18b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 1 Mar 2019 16:21:45 -0700 Subject: [PATCH 15/15] Only destroy interpreters (by ID refcount) when created by the module. --- Include/cpython/pystate.h | 3 +++ Include/internal/pycore_pystate.h | 1 + Modules/_xxsubinterpretersmodule.c | 1 + Python/pystate.c | 14 +++++++++++++- 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index ab4a9375205b2c..5439d07e6af3ea 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -30,6 +30,9 @@ typedef struct { (_PyMainInterpreterConfig){.install_signal_handlers = -1} /* Note: _PyMainInterpreterConfig_INIT sets other fields to 0/NULL */ +PyAPI_FUNC(int) _PyInterpreterState_RequiresIDRef(PyInterpreterState *); +PyAPI_FUNC(void) _PyInterpreterState_RequireIDRef(PyInterpreterState *, int); + PyAPI_FUNC(_PyCoreConfig *) _PyInterpreterState_GetCoreConfig(PyInterpreterState *); PyAPI_FUNC(_PyMainInterpreterConfig *) _PyInterpreterState_GetMainConfig(PyInterpreterState *); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 9b57c57b02dd82..7e78297da69ab2 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -30,6 +30,7 @@ struct _is { int64_t id; int64_t id_refcount; + int requires_idref; PyThread_type_lock id_mutex; int finalizing; diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index bba53d68b21636..1cf43b7ac76e04 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1997,6 +1997,7 @@ interp_create(PyObject *self, PyObject *args) PyThreadState_Swap(save_tstate); return NULL; } + _PyInterpreterState_RequireIDRef(tstate->interp, 1); return idobj; } diff --git a/Python/pystate.c b/Python/pystate.c index fc2acca4b55e7a..99a01efa6c72ed 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -403,7 +403,7 @@ _PyInterpreterState_IDDecref(PyInterpreterState *interp) int64_t refcount = interp->id_refcount; PyThread_release_lock(interp->id_mutex); - if (refcount == 0) { + if (refcount == 0 && interp->requires_idref) { // XXX Using the "head" thread isn't strictly correct. PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); // XXX Possible GILState issues? @@ -413,6 +413,18 @@ _PyInterpreterState_IDDecref(PyInterpreterState *interp) } } +int +_PyInterpreterState_RequiresIDRef(PyInterpreterState *interp) +{ + return interp->requires_idref; +} + +void +_PyInterpreterState_RequireIDRef(PyInterpreterState *interp, int required) +{ + interp->requires_idref = required ? 1 : 0; +} + _PyCoreConfig * _PyInterpreterState_GetCoreConfig(PyInterpreterState *interp) {