8000 gh-111506: Add _Py_OPAQUE_PYOBJECT to hide PyObject layout & related … · python/cpython@c7d24b8 · GitHub
[go: up one dir, main page]
Skip to content

Commit c7d24b8

Browse files
encukouvstinner
encukou
and
vstinner
authored
gh-111506: Add _Py_OPAQUE_PYOBJECT to hide PyObject layout & related API (GH-136505)
Allow Py_LIMITED_API for (Py_GIL_DISABLED && _Py_OPAQUE_PYOBJECT) API that's removed when _Py_OPAQUE_PYOBJECT is defined: - PyObject_HEAD - _PyObject_EXTRA_INIT - PyObject_HEAD_INIT - PyObject_VAR_HEAD - struct _object (i.e. PyObject) (opaque) - struct PyVarObject (opaque) - Py_SIZE - Py_SET_TYPE - Py_SET_SIZE - PyModuleDef_Base (opaque) - PyModuleDef_HEAD_INIT - PyModuleDef (opaque) - _Py_IsImmortal - _Py_IsStaticImmortal Note that the `_Py_IsImmortal` removal (and a few other issues) means _Py_OPAQUE_PYOBJECT only works with limited API 3.14+ now. Co-authored-by: Victor Stinner <vstinner@python.org>
1 parent db47f4d commit c7d24b8

File tree

8 files changed

+123
-25
lines changed

8 files changed

+123
-25
lines changed

‎Include/Python.h

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -45,19 +45,19 @@
4545
# endif
4646
#endif
4747

48-
// gh-111506: The free-threaded build is not compatible with the limited API
49-
// or the stable ABI.
50-
#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED)
51-
# error "The limited API is not currently supported in the free-threaded build"
52-
#endif
48+
#if defined(Py_GIL_DISABLED)
49+
# if defined(Py_LIMITED_API) && !defined(_Py_OPAQUE_PYOBJECT)
50+
# error "Py_LIMITED_API is not currently supported in the free-threaded build"
51+
# endif
5352

54-
#if defined(Py_GIL_DISABLED) && defined(_MSC_VER)
55-
# include <intrin.h> // __readgsqword()
56-
#endif
53+
# if defined(_MSC_VER)
54+
# include <intrin.h> // __readgsqword()
55+
# endif
5756

58-
#if defined(Py_GIL_DISABLED) && defined(__MINGW32__)
59-
# include <intrin.h> // __readgsqword()
60-
#endif
57+
# if defined(__MINGW32__)
58+
# include <intrin.h> // __readgsqword()
59+
# endif
60+
#endif // Py_GIL_DISABLED
6161

6262
// Include Python header files
6363
#include "pyport.h"

‎Include/moduleobject.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ PyAPI_FUNC(PyObject *) PyModuleDef_Init(PyModuleDef*);
3636
PyAPI_DATA(PyTypeObject) PyModuleDef_Type;
3737
#endif
3838

39+
#ifndef _Py_OPAQUE_PYOBJECT
3940
typedef struct PyModuleDef_Base {
4041
PyObject_HEAD
4142
/* The function used to re-initialize the module.
@@ -63,6 +64,7 @@ typedef struct PyModuleDef_Base {
6364
0, /* m_index */ \
6465
_Py_NULL, /* m_copy */ \
6566
}
67+
#endif // _Py_OPAQUE_PYOBJECT
6668

6769
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000
6870
/* New in 3.5 */
@@ -104,6 +106,8 @@ struct PyModuleDef_Slot {
104106
PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil);
105107
#endif
106108

109+
110+
#ifndef _Py_OPAQUE_PYOBJECT
107111
struct PyModuleDef {
108112
PyModuleDef_Base m_base;
109113
const char* m_name;
@@ -115,6 +119,7 @@ struct PyModuleDef {
115119
inquiry m_clear;
116120
freefunc m_free;
117121
};
122+
#endif // _Py_OPAQUE_PYOBJECT
118123

119124
#ifdef __cplusplus
120125
}

‎Include/object.h

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,11 @@ whose size is determined when the object is allocated.
5656
# define Py_REF_DEBUG
5757
#endif
5858

59+
#if defined(_Py_OPAQUE_PYOBJECT) && !defined(Py_LIMITED_API)
60+
# error "_Py_OPAQUE_PYOBJECT only makes sense with Py_LIMITED_API"
61+
#endif
62+
63+
#ifndef _Py_OPAQUE_PYOBJECT
5964
/* PyObject_HEAD defines the initial segment of every PyObject. */
6065
#define PyObject_HEAD PyObject ob_base;
6166

@@ -99,6 +104,8 @@ whose size is determined when the object is allocated.
99104
* not necessarily a byte count.
100105
*/
101106
#define PyObject_VAR_HEAD PyVarObject ob_base;
107+
#endif // !defined(_Py_OPAQUE_PYOBJECT)
108+
102109
#define Py_INVALID_SIZE (Py_ssize_t)-1
103110

104111
/* PyObjects are given a minimum alignment so that the least significant bits
@@ -112,7 +119,9 @@ whose size is determined when the object is allocated.
112119
* by hand. Similarly every pointer to a variable-size Python object can,
113120
* in addition, be cast to PyVarObject*.
114121
*/
115-
#ifndef Py_GIL_DISABLED
122+
#ifdef _Py_OPAQUE_PYOBJECT
123+
/* PyObject is opaque */
124+
#elif !defined(Py_GIL_DISABLED)
116125
struct _object {
117126
#if (defined(__GNUC__) || defined(__clang__)) \
118127
&& !(defined __STDC_VERSION__ && __STDC_VERSION__ >= 201112L)
@@ -168,15 +177,18 @@ struct _object {
168177
Py_ssize_t ob_ref_shared; // shared (atomic) reference count
169178
PyTypeObject *ob_type;
170179
};
171-
#endif
180+
#endif // !defined(_Py_OPAQUE_PYOBJECT)
172181

173182
/* Cast argument to PyObject* type. */
174183
#define _PyObject_CAST(op) _Py_CAST(PyObject*, (op))
175184

176-
typedef struct {
185+
#ifndef _Py_OPAQUE_PYOBJECT
186+
struct PyVarObject {
177187
PyObject ob_base;
178188
Py_ssize_t ob_size; /* Number of items in variable part */
179-
} PyVarObject;
189+
};
190+
#endif
191+
typedef struct PyVarObject PyVarObject;
180192

181193
/* Cast argument to PyVarObject* type. */
182194
#define _PyVarObject_CAST(op) _Py_CAST(PyVarObject*, (op))
@@ -286,6 +298,7 @@ PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob);
286298
PyAPI_DATA(PyTypeObject) PyLong_Type;
287299
PyAPI_DATA(PyTypeObject) PyBool_Type;
288300

301+
#ifndef _Py_OPAQUE_PYOBJECT
289302
// bpo-39573: The Py_SET_SIZE() function must be used to set an object size.
290303
static inline Py_ssize_t Py_SIZE(PyObject *ob) {
291304
assert(Py_TYPE(ob) != &PyLong_Type);
@@ -295,6 +308,7 @@ static inline Py_ssize_t Py_SIZE(PyObject *ob) {
295308
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
296309
# define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob))
297310
#endif
311+
#endif // !defined(_Py_OPAQUE_PYOBJECT)
298312

299313
static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) {
300314
return Py_TYPE(ob) == type;
@@ -304,6 +318,7 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) {
304318
#endif
305319

306320

321+
#ifndef _Py_OPAQUE_PYOBJECT
307322
static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) {
308323
ob->ob_type = type;
309324
}
@@ -323,6 +338,7 @@ static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) {
323338
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
324339
# define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), (size))
325340
#endif
341+
#endif // !defined(_Py_OPAQUE_PYOBJECT)
326342

327343

328344
/*

‎Include/refcount.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob);
117117
#endif
118118
#endif
119119

120+
#ifndef _Py_OPAQUE_PYOBJECT
120121
static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op)
121122
{
122123
#if defined(Py_GIL_DISABLED)
@@ -140,6 +141,7 @@ static inline Py_ALWAYS_INLINE int _Py_IsStaticImmortal(PyObject *op)
140141
#endif
141142
}
142143
#define _Py_IsStaticImmortal(op) _Py_IsStaticImmortal(_PyObject_CAST(op))
144+
#endif // !defined(_Py_OPAQUE_PYOBJECT)
143145

144146
// Py_SET_REFCNT() implementation for stable ABI
145147
PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt);

‎Lib/test/test_cext/__init__.py

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,10 @@
1212
from test import support
1313

1414

15-
SOURCE = os.path.join(os.path.dirname(__file__), 'extension.c')
15+
SOURCES = [
16+
os.path.join(os.path.dirname(__file__), 'extension.c'),
17+
os.path.join(os.path.dirname(__file__), 'create_moduledef.c'),
18+
]
1619
SETUP = os.path.join(os.path.dirname(__file__), 'setup.py')
1720

1821

@@ -35,24 +38,31 @@ class BaseTests:
3538
def test_build(self):
3639
self.check_build('_test_cext')
3740

38-
def check_build(self, extension_name, std=None, limited=False):
41+
def check_build(self, extension_name, std=None, limited=False,
42+
opaque_pyobject=False):
3943
venv_dir = 'env'
4044
with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe:
4145
self._check_build(extension_name, python_exe,
42-
std=std, limited=limited)
46+
std=std, limited=limited,
47+
opaque_pyobject=opaque_pyobject)
4348

44-
def _check_build(self, extension_name, python_exe, std, limited):
49+
def _check_build(self, extension_name, python_exe, std, limited,
50+
opaque_pyobject):
4551
pkg_dir = 'pkg'
4652
os.mkdir(pkg_dir)
4753
shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP)))
48-
shutil.copy(SOURCE, os.path.join(pkg_dir, os.path.basename(SOURCE)))
54+
for source in SOURCES:
55+
dest = os.path.join(pkg_dir, os.path.basename(source))
56+
shutil.copy(source, dest)
4957

5058
def run_cmd(operation, cmd):
5159
env = os.environ.copy()
5260
if std:
5361
env['CPYTHON_TEST_STD'] = std
5462
if limited:
5563
env['CPYTHON_TEST_LIMITED'] = '1'
64+
if opaque_pyobject:
65+
env['CPYTHON_TEST_OPAQUE_PYOBJECT'] = '1'
5666
env['CPYTHON_TEST_EXT_NAME'] = extension_name
5767
env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API))
58 1241 68
if support.verbose:
@@ -107,6 +117,11 @@ def test_build_limited_c11(self):
107117
def test_build_c11(self):
108118
self.check_build('_test_c11_cext', std='c11')
109119

120+
def test_build_opaque_pyobject(self):
121+
# Test with _Py_OPAQUE_PYOBJECT
122+
self.check_build('_test_limited_opaque_cext', limited=True,
123+
opaque_pyobject=True)
124+
110125
@unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99")
111126
def test_build_c99(self):
112127
# In public docs, we say C API is compatible with C11. However,

‎Lib/test/test_cext/create_moduledef.c

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Workaround for testing _Py_OPAQUE_PYOBJECT.
2+
// See end of 'extension.c'
3+
4+
5+
#undef _Py_OPAQUE_PYOBJECT
6+
#undef Py_LIMITED_API
7+
#include "Python.h"
8+
9+
10+
// (repeated definition to avoid creating a header)
11+
extern PyObject *testcext_create_moduledef(
12+
const char *name, const char *doc,
13+
PyMethodDef *methods, PyModuleDef_Slot *slots);
14+
15+
PyObject *testcext_create_moduledef(
16+
const char *name, const char *doc,
17+
PyMethodDef *methods, PyModuleDef_Slot *slots) {
18+
19+
static struct PyModuleDef _testcext_module = {
20+
PyModuleDef_HEAD_INIT,
21+
};
22+
if (!_testcext_module.m_name) {
23+
_testcext_module.m_name = name;
24+
_testcext_module.m_doc = doc;
25+
_testcext_module.m_methods = methods;
26+
_testcext_module.m_slots = slots;
27+
}
28+
return PyModuleDef_Init(&_testcext_module);
29+
}

‎Lib/test/test_cext/extension.c

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@ _testcext_exec(
7878
return 0;
7979
}
8080

81+
#define _FUNC_NAME(NAME) PyInit_ ## NAME
82+
#define FUNC_NAME(NAME) _FUNC_NAME(NAME)
83+
8184
// Converting from function pointer to void* has undefined behavior, but
8285
// works on all known platforms, and CPython's module and type slots currently
8386
// need it.
@@ -96,9 +99,10 @@ static PyModuleDef_Slot _testcext_slots[] = {
9699

97100
_Py_COMP_DIAG_POP
98101

99-
100102
PyDoc_STRVAR(_testcext_doc, "C test extension.");
101103

104+
#ifndef _Py_OPAQUE_PYOBJECT
105+
102106
static struct PyModuleDef _testcext_module = {
103107
PyModuleDef_HEAD_INIT, // m_base
104108
STR(MODULE_NAME), // m_name
@@ -112,11 +116,30 @@ static struct PyModuleDef _testcext_module = {
112116
};
113117

114118

115-
#define _FUNC_NAME(NAME) PyInit_ ## NAME
116-
#define FUNC_NAME(NAME) _FUNC_NAME(NAME)
117-
118119
PyMODINIT_FUNC
119120
FUNC_NAME(MODULE_NAME)(void)
120121
{
121122
return PyModuleDef_Init(&_testcext_module);
122123
}
124+
125+
#else // _Py_OPAQUE_PYOBJECT
126+
127+
// Opaque PyObject means that PyModuleDef is also opaque and cannot be
128+
// declared statically. See PEP 793.
129+
// So, this part of module creation is split into a separate source file
130+
// which uses non-limited API.
131+
132+
// (repeated definition to avoid creating a header)
133+
extern PyObject *testcext_create_moduledef(
134+
const char *name, const char *doc,
135+
PyMethodDef *methods, PyModuleDef_Slot *slots);
136+
137+
138+
PyMODINIT_FUNC
139+
FUNC_NAME(MODULE_NAME)(void)
140+
{
141+
return testcext_create_moduledef(
142+
STR(MODULE_NAME), _testcext_doc, _testcext_methods, _testcext_slots);
143+
}
144+
145+
#endif // _Py_OPAQUE_PYOBJECT

‎Lib/test/test_cext/setup.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,11 @@ def main():
5959
std = os.environ.get("CPYTHON_TEST_STD", "")
6060
module_name = os.environ["CPYTHON_TEST_EXT_NAME"]
6161
limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", ""))
62+
opaque_pyobject = bool(os.environ.get("CPYTHON_TEST_OPAQUE_PYOBJECT", ""))
6263
internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0")))
6364

65+
sources = [SOURCE]
66+
6467
if not internal:
6568
cflags = list(PUBLIC_CFLAGS)
6669
else:
@@ -93,6 +96,11 @@ def main():
9396
version = sys.hexversion
9497
cflags.append(f'-DPy_LIMITED_API={version:#x}')
9598

99+
# Define _Py_OPAQUE_PYOBJECT macro
100+
if opaque_pyobject:
101+
cflags.append(f'-D_Py_OPAQUE_PYOBJECT')
102+
sources.append('create_moduledef.c')
103+
96104
if internal:
97105
cflags.append('-DTEST_INTERNAL_C_API=1')
98106

@@ -120,7 +128,7 @@ def main():
120128

121129
ext = Extension(
122130
module_name,
123-
sources=[SOURCE],
131+
sources=sources,
124132
extra_compile_args=cflags,
125133
include_dirs=include_dirs,
126134
library_dirs=library_dirs)

0 commit comments

Comments
 (0)
0