8000 gh-117142: ctypes: Migrate global closure freelist to thunk-type state by neonene · Pull Request #117874 · python/cpython · GitHub
[go: up one dir, main page]
Skip to content

gh-117142: ctypes: Migrate global closure freelist to thunk-type state #117874

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 11 commits into from
Closed

gh-117142: ctypes: Migrate global closure freelist to thunk-type state #117874

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
f02a06d
globals-to-fix.tsv
neonene Apr 14, 2024
58d6beb
move free_list to thunk type state
neonene Apr 14, 2024
c9b1154
add a test
neonene Apr 14, 2024
71d12e3
check USING_MALLOC_CLOSURE_DOT_C
neonene Apr 14, 2024
f502626
fix tag name
neonene Apr 14, 2024
a2a53a0
tune a test
neonene Apr 14, 2024
8b987a9
edit test
neonene Apr 15, 2024
9178936
experiment
neonene May 6, 2024
a78f36a
experiment retry
neonene May 6, 2024
a2ec9df
experiment retry
neonene May 6, 2024
de53f39
experiment retry
neonene May 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions Lib/test/test_ctypes/test_refcounts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,20 @@ def test_finalize(self):
)
script_helper.assert_python_ok("-c", script)

def test_many_closures_per_module(self):
# check if mmap() and munmap() get called multiple times
script = (
"import ctypes;"
"pyfunc = lambda: 0;"
"cfunc_type = ctypes.CFUNCTYPE(ctypes.c_int);"
"cfuncs = [cfunc_type(pyfunc) for i in range(500)];"
"cthunk_tp = type(cfuncs[0]._objects['0']);"
"m = type(cthunk_tp) is not type;" # require a custom meta type
"n_containers = m and cthunk_tp.get_ffi_closure_containers_count();"
"exit(n_containers and n_containers < 2)"
)
script_helper.assert_python_ok("-c", script)


if __name__ == '__main__':
unittest.main()
9 changes: 8 additions & 1 deletion Modules/_ctypes/_ctypes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5721,7 +5721,12 @@ _ctypes_add_types(PyObject *mod)
tp_base is the base type, defaults to 'object' aka PyBaseObject_Type.
*/
CREATE_TYPE(st->PyCArg_Type, &carg_spec, NULL, NULL);
CREATE_TYPE(st->PyCThunk_Type, &cthunk_spec, NULL, NULL);
#ifdef USING_MALLOC_CLOSURE_DOT_C
CREATE_TYPE(st->PyCThunkType_Type, &cthunk_type_spec, NULL, &PyType_Type);
#else
st->PyCThunkType_Type = NULL;
#endif
CREATE_TYPE(st->PyCThunk_Type, &cthunk_spec, st->PyCThunkType_Type, NULL);
CREATE_TYPE(st->PyCData_Type, &pycdata_spec, NULL, NULL);

// Common Metaclass
Expand Down Expand Up @@ -5900,6 +5905,7 @@ module_traverse(PyObject *module, visitproc visit, void *arg) {
#ifdef MS_WIN32
Py_VISIT(st->PyComError_Type);
#endif
Py_VISIT(st->PyCThunkType_Type);
Py_VISIT(st->PyCType_Type);
return 0;
}
Expand Down Expand Up @@ -5935,6 +5941,7 @@ module_clear(PyObject *module) {
#ifdef MS_WIN32
Py_CLEAR(st->PyComError_Type);
#endif
Py_CLEAR(st->PyCThunkType_Type);
Py_CLEAR(st->PyCType_Type);
return 0;
}
Expand Down
5 changes: 3 additions & 2 deletions Modules/_ctypes/callbacks.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ CThunkObject_dealloc(PyObject *myself)
PyObject_GC_UnTrack(self);
(void)CThunkObject_clear(myself);
if (self->pcl_write) {
Py_ffi_closure_free(self->pcl_write);
Py_ffi_closure_free(tp, self->pcl_write);
}
PyObject_GC_Del(self);
Py_DECREF(tp);
Expand Down Expand Up @@ -364,7 +364,8 @@ CThunkObject *_ctypes_alloc_callback(ctypes_state *st,

assert(CThunk_CheckExact(st, (PyObject *)p));

p->pcl_write = Py_ffi_closure_alloc(sizeof(ffi_closure), &p->pcl_exec);
p->pcl_write = Py_ffi_closure_alloc(st->PyCThunk_Type,
sizeof(ffi_closure), &p->pcl_exec);
if (p->pcl_write == NULL) {
PyErr_NoMemory();
goto error;
Expand Down
10 changes: 7 additions & 3 deletions Modules/_ctypes/ctypes.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#undef USING_MALLOC_CLOSURE_DOT_C
#if defined (__SVR4) && defined (__sun)
# include <alloca.h>
#endif
Expand Down Expand Up @@ -43,6 +44,7 @@ typedef struct {
PyTypeObject *DictRemover_Type;
PyTypeObject *PyCArg_Type;
PyTypeObject *PyCField_Type;
PyTypeObject *PyCThunkType_Type;
PyTypeObject *PyCThunk_Type;
PyTypeObject *StructParam_Type;
PyTypeObject *PyCType_Type;
Expand Down Expand Up @@ -119,6 +121,9 @@ get_module_state_by_def_final(PyTypeObject *cls)
extern PyType_Spec carg_spec;
extern PyType_Spec cfield_spec;
extern PyType_Spec cthunk_spec;
#ifdef USING_MALLOC_CLOSURE_DOT_C
extern PyType_Spec cthunk_type_spec;
#endif

typedef struct tagPyCArgObject PyCArgObject;
typedef struct tagCDataObject CDataObject;
Expand Down Expand Up @@ -453,11 +458,10 @@ PyObject *_ctypes_get_errobj(ctypes_state *st, int **pspace);
void Py_ffi_closure_free(void *p);
void *Py_ffi_closure_alloc(size_t size, void** codeloc);
#else
#define Py_ffi_closure_free ffi_closure_free
#define Py_ffi_closure_alloc ffi_closure_alloc
#define Py_ffi_closure_free(tp, p) ffi_closure_free(p)
#define Py_ffi_closure_alloc(tp, size, codeloc) ffi_closure_alloc(size, codeloc)
#endif


/****************************************************************
* Accessing StgInfo -- these are inlined for performance reasons.
*/
Expand Down
147 changes: 0 additions & 147 deletions Modules/_ctypes/malloc_closure.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,148 +1 @@
#ifndef Py_BUILD_CORE_BUILTIN
# define Py_BUILD_CORE_MODULE 1
#endif

#include <Python.h>
#include <ffi.h>
#ifdef MS_WIN32
# include <windows.h>
#else
# include <sys/mman.h>
# include <unistd.h> // sysconf()
# if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
# define MAP_ANONYMOUS MAP_ANON
# endif
#endif
#include "ctypes.h"

/* BLOCKSIZE can be adjusted. Larger blocksize will take a larger memory
overhead, but allocate less blocks from the system. It may be that some
systems have a limit of how many mmap'd blocks can be open.
*/

#define BLOCKSIZE _pagesize

/* #define MALLOC_CLOSURE_DEBUG */ /* enable for some debugging output */


/******************************************************************/

typedef union _tagITEM {
ffi_closure closure;
union _tagITEM *next;
} ITEM;

static ITEM *free_list;
static int _pagesize;

static void more_core(void)
{
ITEM *item;
int count, i;

/* determine the pagesize */
#ifdef MS_WIN32
if (!_pagesize) {
SYSTEM_INFO systeminfo;
GetSystemInfo(&systeminfo);
_pagesize = systeminfo.dwPageSize;
}
#else
if (!_pagesize) {
#ifdef _SC_PAGESIZE
_pagesize = sysconf(_SC_PAGESIZE);
#else
_pagesize = getpagesize();
#endif
}
#endif

/* calculate the number of nodes to allocate */
count = BLOCKSIZE / sizeof(ITEM);

/* allocate a memory block */
#ifdef MS_WIN32
item = (ITEM *)VirtualAlloc(NULL,
count * sizeof(ITEM),
MEM_COMMIT,
PAGE_EXECUTE_READWRITE);
if (item == NULL)
return;
#else
item = (ITEM *)mmap(NULL,
count * sizeof(ITEM),
PROT_READ | PROT_WRITE | PROT_EXEC,
MAP_PRIVATE | MAP_ANONYMOUS,
-1,
0);
if (item == (void *)MAP_FAILED)
return;
#endif

#ifdef MALLOC_CLOSURE_DEBUG
printf("block at %p allocated (%d bytes), %d ITEMs\n",
item, count * (int)sizeof(ITEM), count);
#endif
/* put them into the free list */
for (i = 0; i < count; ++i) {
item->next = free_list;
free_list = item;
++item;
}
}

/******************************************************************/

/* put the item back into the free list */
void Py_ffi_closure_free(void *p)
{
#ifdef HAVE_FFI_CLOSURE_ALLOC
#ifdef USING_APPLE_OS_LIBFFI
# ifdef HAVE_BUILTIN_AVAILABLE
if (__builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)) {
# else
if (ffi_closure_free != NULL) {
# endif
#endif
ffi_closure_free(p);
return;
#ifdef USING_APPLE_OS_LIBFFI
}
#endif
#endif
ITEM *item = (ITEM *)p;
item->next = free_list;
free_list = item;
}

/* return one item from the free list, allocating more if needed */
void *Py_ffi_closure_alloc(size_t size, void** codeloc)
{
#ifdef HAVE_FFI_CLOSURE_ALLOC
#ifdef USING_APPLE_OS_LIBFFI
# ifdef HAVE_BUILTIN_AVAILABLE
if (__builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)) {
# else
if (ffi_closure_alloc != NULL) {
# endif
#endif
return ffi_closure_alloc(size, codeloc);
#ifdef USING_APPLE_OS_LIBFFI
}
#endif
#endif
ITEM *item;
if (!free_list)
more_core();
if (!free_list)
return NULL;
item = free_list;
free_list = item->next;
#ifdef _M_ARM
// set Thumb bit so that blx is called correctly
*codeloc = (ITEM*)((uintptr_t)item | 1);
#else
*codeloc = (void *)item;
#endif
return (void *)item;
}
2 changes: 0 additions & 2 deletions Tools/c-analyzer/cpython/globals-to-fix.tsv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -447,7 +447,6 @@ Modules/_tkinter.c - trbInCmd -
## other
Include/datetime.h - PyDateTimeAPI -
Modules/_ctypes/cfield.c _ctypes_get_fielddesc initialized -
Modules/_ctypes/malloc_closure.c - _pagesize -
Modules/_cursesmodule.c - initialised -
Modules/_cursesmodule.c - initialised_setupterm -
Modules/_cursesmodule.c - initialisedcolors -
Expand All @@ -461,7 +460,6 @@ Modules/readline.c - libedit_history_start -
## state

Modules/_ctypes/cfield.c - formattable -
Modules/_ctypes/malloc_closure.c - free_list -
Modules/_curses_panel.c - lop -
Modules/_ssl/debughelpers.c _PySSL_keylog_callback lock -
Modules/_tkinter.c - quitMainLoop -
Expand Down
Loading
0