8000 gh-76785: Improved Subinterpreters Compatibility with 3.12 (gh-115424) · python/cpython@514b1c9 · GitHub
[go: up one dir, main page]
Skip to content

Commit 514b1c9

Browse files
ericsnowcurrentlyericsnowcurrently
authored
gh-76785: Improved Subinterpreters Compatibility with 3.12 (gh-115424)
For the most part, these changes make is substantially easier to backport subinterpreter-related code to 3.12, especially the related modules (e.g. _xxsubinterpreters). The main motivation is to support releasing a PyPI package with the 3.13 capabilities compiled for 3.12. A lot of the changes here involve either hiding details behind macros/functions or splitting up some files.
1 parent 206f73d commit 514b1c9

12 files changed

+857
-719
lines changed

‎Include/internal/pycore_code.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,15 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11+
12+
// We hide some of the newer PyCodeObject fields behind macros.
13+
// This helps with backporting certain changes to 3.12.
14+
#define _PyCode_HAS_EXECUTORS(CODE) \
15+
(CODE->co_executors != NULL)
16+
#define _PyCode_HAS_INSTRUMENTATION(CODE) \
17+
(CODE->_co_instrumentation_version > 0)
18+
19+
1120
#define CODE_MAX_WATCHERS 8
1221

1322
/* PEP 659

‎Include/internal/pycore_crossinterp.h

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,11 @@ struct _xid {
8787
PyAPI_FUNC(_PyCrossInterpreterData *) _PyCrossInterpreterData_New(void);
8888
PyAPI_FUNC(void) _PyCrossInterpreterData_Free(_PyCrossInterpreterData *data);
8989

90+
#define _PyCrossInterpreterData_DATA(DATA) ((DATA)->data)
91+
#define _PyCrossInterpreterData_OBJ(DATA) ((DATA)->obj)
92+
#define _PyCrossInterpreterData_INTERPID(DATA) ((DATA)->interpid)
93+
// Users should not need getters for "new_object" or "free".
94+
9095

9196
/* defining cross-interpreter data */
9297

@@ -101,6 +106,25 @@ PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize(
101106
PyAPI_FUNC(void) _PyCrossInterpreterData_Clear(
102107
PyInterpreterState *, _PyCrossInterpreterData *);
103108

109+
// Normally the Init* functions are sufficient. The only time
110+
// additional initialization might be needed is to set the "free" func,
111+
// though that should be infrequent.
112+
#define _PyCrossInterpreterData_SET_FREE(DATA, FUNC) \
113+
do { \
114+
(DATA)->free = (FUNC); \
115+
} while (0)
116+
// Additionally, some shareable types are essentially light wrappers
117+
// around other shareable types. The crossinterpdatafunc of the wrapper
118+
// can often be implemented by calling the wrapped object's
119+
// crossinterpdatafunc and then changing the "new_object" function.
120+
// We have _PyCrossInterpreterData_SET_NEW_OBJECT() here for that,
121+
// but might be better to have a function like
122+
// _PyCrossInterpreterData_AdaptToWrapper() instead.
123+
#define _PyCrossInterpreterData_SET_NEW_OBJECT(DATA, FUNC) \
124+
do { \
125+
(DATA)->new_object = (FUNC); \
126+
} while (0)
127+
104128

105129
/* using cross-interpreter data */
106130

@@ -170,6 +194,8 @@ extern void _PyXI_Fini(PyInterpreterState *interp);
170194
extern PyStatus _PyXI_InitTypes(PyInterpreterState *interp);
171195
extern void _PyXI_FiniTypes(PyInterpreterState *interp);
172196

197+
#define _PyInterpreterState_GetXIState(interp) (&(interp)->xi)
198+
173199

174200
/***************************/
175201
/* short-term data sharing */

‎Include/internal/pycore_tstate.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,13 @@ extern "C" {
1313
#include "pycore_brc.h" // struct _brc_thread_state
1414

1515

16+
static inline void
17+
_PyThreadState_SetWhence(PyThreadState *tstate, int whence)
18+
{
19+
tstate->_whence = whence;
20+
}
21+
22+
1623
// Every PyThreadState is actually allocated as a _PyThreadStateImpl. The
1724
// PyThreadState fields are exposed as part of the C API, although most fields
1825
// are intended to be private. The _PyThreadStateImpl fields not exposed.

‎Makefile.pre.in

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1671,6 +1671,14 @@ Modules/pwdmodule.o: $(srcdir)/Modules/pwdmodule.c $(srcdir)/Modules/posixmodule
16711671

16721672
Modules/signalmodule.o: $(srcdir)/Modules/signalmodule.c $(srcdir)/Modules/posixmodule.h
16731673

1674+
Modules/_xxsubinterpretersmodule.o: $(srcdir)/Modules/_xxsubinterpretersmodule.c $(srcdir)/Modules/_interpreters_common.h
1675+
1676+
Modules/_xxinterpqueuesmodule.o: $(srcdir)/Modules/_xxinterpqueuesmodule.c $(srcdir)/Modules/_interpreters_common.h
1677+
1678+
Modules/_xxinterpchannelsmodule.o: $(srcdir)/Modules/_xxinterpchannelsmodule.c $(srcdir)/Modules/_interpreters_common.h
1679+
1680+
Python/crossinterp.o: $(srcdir)/Python/crossinterp.c $(srcdir)/Python/crossinterp_data_lookup.h $(srcdir)/Python/crossinterp_exceptions.h
1681+
16741682
Python/dynload_shlib.o: $(srcdir)/Python/dynload_shlib.c Makefile
16751683
$(CC) -c $(PY_CORE_CFLAGS) \
16761684
-DSOABI='"$(SOABI)"' \

‎Modules/_interpreters_common.h

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
2+
#define _RESOLVE_MODINIT_FUNC_NAME(NAME) \
3+
PyInit_ ## NAME
4+
#define RESOLVE_MODINIT_FUNC_NAME(NAME) \
5+
_RESOLVE_MODINIT_FUNC_NAME(NAME)
6+
7+
8+
static int
9+
ensure_xid_class(PyTypeObject *cls, crossinterpdatafunc getdata)
10+
{
11+
//assert(cls->tp_flags & Py_TPFLAGS_HEAPTYPE);
12+
return _PyCrossInterpreterData_RegisterClass(cls, getdata);
13+
}

‎Modules/_xxinterpchannelsmodule.c

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
#include <sched.h> // sched_yield()
1818
#endif
1919

20+
#include "_interpreters_common.h"
21+
2022

2123
/*
2224
This module has the following process-global state:
@@ -80,7 +82,9 @@ channel's queue, which are safely managed via the _PyCrossInterpreterData_*()
8082
API.. The module does not create any objects that are shared globally.
8183
*/
8284

83-
#define MODULE_NAME "_xxinterpchannels"
85+
#define MODULE_NAME _xxinterpchannels
86+
#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME)
87+
#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME)
8488

8589

8690
#define GLOBAL_MALLOC(TYPE) \
@@ -101,7 +105,7 @@ static int
101105
register_xid_class(PyTypeObject *cls, crossinterpdatafunc shared,
102106
struct xid_class_registry *classes)
103107
{
104-
int res = _PyCrossInterpreterData_RegisterClass(cls, shared);
108+
int res = ensure_xid_class(cls, shared);
105109
if (res == 0) {
106110
assert(classes->count < MAX_XID_CLASSES);
107111
// The class has refs elsewhere, so we need to incref here.
@@ -167,7 +171,7 @@ _get_current_interp(void)
167171
static PyObject *
168172
_get_current_module(void)
169173
{
170-
PyObject *name = PyUnicode_FromString(MODULE_NAME);
174+
PyObject *name = PyUnicode_FromString(MODULE_NAME_STR);
171175
if (name == NULL) {
172176
return NULL;
173177
}
@@ -217,7 +221,7 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base)
217221
}
218222

219223
#define ADD_NEW_EXCEPTION(MOD, NAME, BASE) \
220-
add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE)
224+
add_new_exception(MOD, MODULE_NAME_STR "." Py_STRINGIFY(NAME), BASE)
221225

222226
static PyTypeObject *
223227
add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared,
@@ -299,7 +303,7 @@ _get_current_module_state(void)
299303
if (mod == NULL) {
300304
// XXX import it?
301305
PyErr_SetString(PyExc_RuntimeError,
302-
MODULE_NAME " module not imported yet");
306+
MODULE_NAME_STR " module not imported yet");
303307
return NULL;
304308
}
305309
module_state *state = get_module_state(mod);
@@ -784,7 +788,7 @@ _channelqueue_clear_interpreter(_channelqueue *queue, int64_t interpid)
784788
while (next != NULL) {
785789
_channelitem *item = next;
786790
next = item->next;
787-
if (item->data->interpid == interpid) {
791+
if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) {
788792
if (prev == NULL) {
789793
queue->first = item->next;
790794
}
@@ -2126,7 +2130,7 @@ static PyStructSequence_Field channel_info_fields[] = {
21262130
};
21272131

21282132
static PyStructSequence_Desc channel_info_desc = {
2129-
.name = MODULE_NAME ".ChannelInfo",
2133+
.name = MODULE_NAME_STR ".ChannelInfo",
21302134
.doc = channel_info_doc,
21312135
.fields = channel_info_fields,
21322136
.n_in_sequence = 8,
@@ -2474,10 +2478,11 @@ struct _channelid_xid {
24742478
static PyObject *
24752479
_channelid_from_xid(_PyCrossInterpreterData *data)
24762480
{
2477-
struct _channelid_xid *xid = (struct _channelid_xid *)data->data;
2481+
struct _channelid_xid *xid = \
2482+
(struct _channelid_xid *)_PyCrossInterpreterData_DATA(data);
24782483

24792484
// It might not be imported yet, so we can't use _get_current_module().
2480-
PyObject *mod = PyImport_ImportModule(MODULE_NAME);
2485+
PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR);
24812486
if (mod == NULL) {
24822487
return NULL;
24832488
}
@@ -2530,7 +2535,8 @@ _channelid_shared(PyThreadState *tstate, PyObject *obj,
25302535
{
25312536
return -1;
25322537
}
2533-
struct _channelid_xid *xid = (struct _channelid_xid *)data->data;
2538+
struct _channelid_xid *xid = \
2539+
(struct _channelid_xid *)_PyCrossInterpreterData_DATA(data);
25342540
xid->cid = ((channelid *)obj)->cid;
25352541
xid->end = ((channelid *)obj)->end;
25362542
xid->resolve = ((channelid *)obj)->resolve;
@@ -2601,7 +2607,7 @@ static PyType_Slot channelid_typeslots[] = {
26012607
};
26022608

26032609
static PyType_Spec channelid_typespec = {
2604-
.name = MODULE_NAME ".ChannelID",
2610+
.name = MODULE_NAME_STR ".ChannelID",
26052611
.basicsize = sizeof(channelid),
26062612
.flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
26072613
Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),
@@ -2680,7 +2686,7 @@ _channelend_shared(PyThreadState *tstate, PyObject *obj,
26802686
if (res < 0) {
26812687
return -1;
26822688
}
2683-
data->new_object = _channelend_from_xid;
2689+
_PyCrossInterpreterData_SET_NEW_OBJECT(data, _channelend_from_xid);
26842690
return 0;
26852691
}
26862692

@@ -3379,7 +3385,7 @@ module_free(void *mod)
33793385

33803386
static struct PyModuleDef moduledef = {
33813387
.m_base = PyModuleDef_HEAD_INIT,
3382-
.m_name = MODULE_NAME,
3388+
.m_name = MODULE_NAME_STR,
33833389
.m_doc = module_doc,
33843390
.m_size = sizeof(module_state),
33853391
.m_methods = module_functions,
@@ -3390,7 +3396,7 @@ static struct PyModuleDef moduledef = {
33903396
};
33913397

33923398
PyMODINIT_FUNC
3393-
PyInit__xxinterpchannels(void)
3399+
MODINIT_FUNC_NAME(void)
33943400
{
33953401
return PyModuleDef_Init(&moduledef);
33963402
}

‎Modules/_xxinterpqueuesmodule.c

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,12 @@
88
#include "Python.h"
99
#include "pycore_crossinterp.h" // struct _xid
1010

11+
#include "_interpreters_common.h"
1112

12-
#define MODULE_NAME "_xxinterpqueues"
13+
14+
#define MODULE_NAME _xxinterpqueues
15+
#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME)
16+
#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME)
1317

1418

1519
#define GLOBAL_MALLOC(TYPE) \
@@ -64,7 +68,7 @@ _get_current_interp(void)
6468
static PyObject *
6569
_get_current_module(void)
6670
{
67-
PyObject *name = PyUnicode_FromString(MODULE_NAME);
71+
PyObject *name = PyUnicode_FromString(MODULE_NAME_STR);
6872
if (name == NULL) {
6973
return NULL;
7074
}
@@ -602,7 +606,7 @@ _queue_clear_interpreter(_queue *queue, int64_t interpid)
602606
while (next != NULL) {
603607
_queueitem *item = next;
604608
next = item->next;
605-
if (item->data->interpid == interpid) {
609+
if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) {
606610
if (prev == NULL) {
607611
queue->items.first = item->next;
608612
}
@@ -1062,7 +1066,7 @@ set_external_queue_type(PyObject *module, PyTypeObject *queue_type)
10621066
}
10631067
state->queue_type = (PyTypeObject *)Py_NewRef(queue_type);
10641068

1065-
if (_PyCrossInterpreterData_RegisterClass(queue_type, _queueobj_shared) < 0) {
1069+
if (ensure_xid_class(queue_type, _queueobj_shared) < 0) {
10661070
return -1;
10671071
}
10681072

@@ -1130,7 +1134,7 @@ _queueid_xid_free(void *data)
11301134
static PyObject *
11311135
_queueobj_from_xid(_PyCrossInterpreterData *data)
11321136
{
1133-
int64_t qid = *(int64_t *)data->data;
1137+
int64_t qid = *(int64_t *)_PyCrossInterpreterData_DATA(data);
11341138
PyObject *qidobj = PyLong_FromLongLong(qid);
11351139
if (qidobj == NULL) {
11361140
return NULL;
@@ -1140,7 +1144,7 @@ _queueobj_from_xid(_PyCrossInterpreterData *data)
11401144
if (mod == NULL) {
11411145
// XXX import it?
11421146
PyErr_SetString(PyExc_RuntimeError,
1143-
MODULE_NAME " module not imported yet");
1147+
MODULE_NAME_STR " module not imported yet");
11441148
return NULL;
11451149
}
11461150

@@ -1181,7 +1185,7 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj,
11811185
_PyCrossInterpreterData_Init(data, tstate->interp, raw, NULL,
11821186
_queueobj_from_xid);
11831187
Py_DECREF(qidobj);
1184-
data->free = _queueid_xid_free;
1188+
_PyCrossInterpreterData_SET_FREE(data, _queueid_xid_free);
11851189
return 0;
11861190
}
11871191

@@ -1670,7 +1674,7 @@ module_free(void *mod)
16701674

16711675
static struct PyModuleDef moduledef = {
16721676
.m_base = PyModuleDef_HEAD_INIT,
1673-
.m_name = MODULE_NAME,
1677+
.m_name = MODULE_NAME_STR,
16741678
.m_doc = module_doc,
16751679
.m_size = sizeof(module_state),
16761680
.m_methods = module_functions,
@@ -1681,7 +1685,7 @@ static struct PyModuleDef moduledef = {
16811685
};
16821686

16831687
PyMODINIT_FUNC
1684-
PyInit__xxinterpqueues(void)
1688+
MODINIT_FUNC_NAME(void)
16851689
{
16861690
return PyModuleDef_Init(&moduledef);
16871691
}

0 commit comments

Comments
 (0)
0