From ae9c004dbd7effcae4cf39eae0df6838b61471eb Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 25 Feb 2024 19:56:06 +0100 Subject: [PATCH 1/9] gh-93502: Add new C-API functions to trace object creation and destruction --- Doc/c-api/init.rst | 42 +++++++++++ Include/cpython/object.h | 10 +++ Include/internal/pycore_object.h | 2 +- Include/internal/pycore_runtime.h | 7 ++ Include/internal/pycore_runtime_init.h | 3 + ...4-02-26-13-14-52.gh-issue-93502.JMWRvA.rst | 4 + Modules/_testcapimodule.c | 75 +++++++++++++++++++ Objects/object.c | 35 +++++++-- Python/tracemalloc.c | 9 ++- 9 files changed, 179 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-26-13-14-52.gh-issue-93502.JMWRvA.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index e7199ad5e0c1b1..e7f067d5036156 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1746,6 +1746,48 @@ Python-level trace functions in previous versions. .. versionadded:: 3.12 +.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) + + The type of the trace function registered using :c:func:`PyRefTracer_SetTracer` + The first parameter is a Python object that has been just created (when **event** + is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** + is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer + that was provided when :c:func:`PyRefTracer_SetTracer` was called. + +.. versionadded:: 3.13 + +.. c:var:: int PyRefTracer_CREATE + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been created. + +.. c:var:: int PyRefTracer_DESTROY + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been destroyed. + +.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) + + Register a reference tracer function. The function will be called when a new Python + has been created or when an object is going to be destroyed. If **data** is provided + it must be an opaque pointer that will be provided when the tracer function is called. + + Not that tracer functions **must not** create Python objects inside or otherwise the + call will be re-entrant. + + The GIL must be held when calling this function. + +.. versionadded:: 3.13 + +.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) + + Get the registered reference tracer function and the value of the opaque data pointer that + was registered when :c:func:`PyRefTracer_SetTracer` was called. If no tracer was registered + this function will return NULL and will set the **data** pointer to NULL. + + The GIL must be held when calling this function. + +.. versionadded:: 3.13 .. _advanced-debugging: diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 7512bb70c760fd..c50030a3325d06 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -507,3 +507,13 @@ PyAPI_FUNC(int) PyType_Unwatch(int watcher_id, PyObject *type); * assigned, or 0 if a new tag could not be assigned. */ PyAPI_FUNC(int) PyUnstable_Type_AssignVersionTag(PyTypeObject *type); + + +typedef enum { + PyRefTracer_CREATE = 0, + PyRefTracer_DESTROY = 1, +} PyRefTracerEvent; + +typedef int (*PyRefTracer)(PyObject *, PyRefTracerEvent event, void *); +PyAPI_FUNC(int) PyRefTracer_SetTracer(PyRefTracer tracer, void *data); +PyAPI_FUNC(PyRefTracer) PyRefTracer_GetTracer(void**); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 34a83ea228e8b1..f01063bc613a5e 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -252,7 +252,7 @@ extern int _PyDict_CheckConsistency(PyObject *mp, int check_content); when a memory block is reused from a free list. Internal function called by _Py_NewReference(). */ -extern int _PyTraceMalloc_NewReference(PyObject *op); +extern int _PyTraceMalloc_NewReference(PyObject *op, PyRefTracerEvent event, void*); // Fast inlined version of PyType_HasFeature() static inline int diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 0c9c59e85b2fcf..2855e329561535 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -125,6 +125,12 @@ typedef struct _Py_DebugOffsets { } tuple_object; } _Py_DebugOffsets; +/* Reference tracer state */ +struct _reftracer_runtime_state { + PyRefTracer tracer_func; + void* data; +}; + /* Full Python runtime state */ /* _PyRuntimeState holds the global state for the CPython runtime. @@ -229,6 +235,7 @@ typedef struct pyruntimestate { struct _fileutils_state fileutils; struct _faulthandler_runtime_state faulthandler; struct _tracemalloc_runtime_state tracemalloc; + struct _reftracer_runtime_state reftracer; // The rwmutex is used to prevent overlapping global and per-interpreter // stop-the-world events. Global stop-the-world events lock the mutex diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index d093047d4bc09d..7dbc761f361566 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -119,6 +119,9 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .faulthandler = _faulthandler_runtime_state_INIT, \ .tracemalloc = _tracemalloc_runtime_state_INIT, \ + .reftracer = { \ + .tracer_func = NULL, \ + }, \ .stoptheworld = { \ .is_global = 1, \ }, \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-26-13-14-52.gh-issue-93502.JMWRvA.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-26-13-14-52.gh-issue-93502.JMWRvA.rst new file mode 100644 index 00000000000000..524626950c02e6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-26-13-14-52.gh-issue-93502.JMWRvA.rst @@ -0,0 +1,4 @@ +Add two new functions to the C-API, :c:func:`PyRefTracer_SetTracer` and +:c:func:`PyRefTracer_GetTracer`, that allows to track object creation and +destruction the same way the :mod:`tracemalloc` module does. Patch by Pablo +Galindo diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b03f871b089c8a..b9fda3b0782e16 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3284,6 +3284,80 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) _Py_COMP_DIAG_POP } +struct simpletracer_data { + int create_count; + int destroy_count; + void* addresses[10]; +}; + +static int _simpletracer(PyObject *obj, PyRefTracerEvent event, void* data) { + struct simpletracer_data* the_data = (struct simpletracer_data*)data; + assert(the_data->create_count + the_data->destroy_count < 10); + the_data->addresses[the_data->create_count + the_data->destroy_count] = obj; + if (event == PyRefTracer_CREATE) { + the_data->create_count++; + } else { + the_data->destroy_count++; + } + return 0; +} + +static PyObject * +test_reftracer(PyObject *ob, PyObject *Py_UNUSED(ignored)) +{ + struct simpletracer_data tracer_data = {0}; + void* the_data = (void*)&tracer_data; + // Install a simple tracer function + PyRefTracer_SetTracer(_simpletracer, the_data); + + // Check that the tracer was correctly installed + void* data; + if (PyRefTracer_GetTracer(&data) != _simpletracer || data != the_data) { + PyErr_SetString(PyExc_ValueError, "The reftracer not correctly installed"); + PyRefTracer_SetTracer(NULL, NULL); + return NULL; + } + + // Create a bunch of objects + PyObject* obj = PyList_New(0); + if (obj == NULL) { + return NULL; + } + PyObject* obj2 = PyDict_New(); + if (obj2 == NULL) { + Py_DECREF(obj); + return NULL; + } + + // Kill all objects + Py_DECREF(obj); + Py_DECREF(obj2); + + // Remove the tracer + PyRefTracer_SetTracer(NULL, NULL); + + // Check that the tracer was removed + if (PyRefTracer_GetTracer(&data) != NULL || data != NULL) { + PyErr_SetString(PyExc_ValueError, "The reftracer was not correctly removed"); + return NULL; + } + + if (tracer_data.create_count != 2 || + tracer_data.addresses[0] != obj || + tracer_data.addresses[1] != obj2) { + PyErr_SetString(PyExc_ValueError, "The object creation was not correctly traced"); + return NULL; + } + + if (tracer_data.destroy_count != 2 || + tracer_data.addresses[2] != obj || + tracer_data.addresses[3] != obj2) { + PyErr_SetString(PyExc_ValueError, "The object destruction was not correctly traced"); + return NULL; + } + + Py_RETURN_NONE; +} static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, @@ -3320,6 +3394,7 @@ static PyMethodDef TestMethods[] = { {"test_get_type_name", test_get_type_name, METH_NOARGS}, {"test_get_type_qualname", test_get_type_qualname, METH_NOARGS}, {"test_get_type_dict", test_get_type_dict, METH_NOARGS}, + {"test_reftracer", test_reftracer, METH_NOARGS}, {"_test_thread_state", test_thread_state, METH_VARARGS}, #ifndef MS_WINDOWS {"_spawn_pthread_waiter", spawn_pthread_waiter, METH_NOARGS}, diff --git a/Objects/object.c b/Objects/object.c index df14fe0c6fbfec..cc0942acf2ea5f 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2367,9 +2367,6 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) static inline void new_reference(PyObject *op) { - if (_PyRuntime.tracemalloc.config.tracing) { - _PyTraceMalloc_NewReference(op); - } // Skip the immortal object check in Py_SET_REFCNT; always set refcnt to 1 #if !defined(Py_GIL_DISABLED) op->ob_refcnt = 1; @@ -2384,6 +2381,10 @@ new_reference(PyObject *op) #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif + if (_PyRuntime.reftracer.tracer_func != NULL) { + void* data = _PyRuntime.reftracer.data; + _PyRuntime.reftracer.tracer_func(op, PyRefTracer_CREATE, data); + } } void @@ -2404,12 +2405,13 @@ _Py_NewReferenceNoTotal(PyObject *op) void _Py_ResurrectReference(PyObject *op) { - if (_PyRuntime.tracemalloc.config.tracing) { - _PyTraceMalloc_NewReference(op); - } #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif + if (_PyRuntime.reftracer.tracer_func != NULL) { + void* data = _PyRuntime.reftracer.data; + _PyRuntime.reftracer.tracer_func(op, PyRefTracer_CREATE, data); + } } @@ -2883,6 +2885,11 @@ _Py_Dealloc(PyObject *op) Py_INCREF(type); #endif + if (_PyRuntime.reftracer.tracer_func != NULL) { + void* data = _PyRuntime.reftracer.data; + _PyRuntime.reftracer.tracer_func(op, PyRefTracer_DESTROY, data); + } + #ifdef Py_TRACE_REFS _Py_ForgetReference(op); #endif @@ -2970,3 +2977,19 @@ _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt) { Py_SET_REFCNT(ob, refcnt); } + +int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) { + assert(PyGILState_Check()); + _PyRuntime.reftracer.tracer_func = tracer; + _PyRuntime.reftracer.data = data; + return 0; +} + +PyRefTracer PyRefTracer_GetTracer(void** data) { + assert(PyGILState_Check()); + if (data != NULL) { + *data = _PyRuntime.reftracer.data; + } + return _PyRuntime.reftracer.tracer_func; +} + diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 19b64c619feb6a..0a4f4a3a2118fe 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -906,6 +906,8 @@ _PyTraceMalloc_Start(int max_nframe) return -1; } + _PyRuntime.reftracer.tracer_func = _PyTraceMalloc_NewReference; + if (tracemalloc_config.tracing) { /* hook already installed: do nothing */ return 0; @@ -1352,8 +1354,13 @@ _PyTraceMalloc_Fini(void) Do nothing if tracemalloc is not tracing memory allocations or if the object memory block is not already traced. */ int -_PyTraceMalloc_NewReference(PyObject *op) +_PyTraceMalloc_NewReference(PyObject *op, PyRefTracerEvent event, void* Py_UNUSED(ignore)) + { + if (event != PyRefTracer_CREATE) { + return 0; + } + assert(PyGILState_Check()); if (!tracemalloc_config.tracing) { From 4a96b8b855368dc9ec3ddaf5c476f0522d32d658 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 1 May 2024 13:59:21 +0100 Subject: [PATCH 2/9] Apply suggestions from code review Co-authored-by: Victor Stinner --- Doc/c-api/init.rst | 2 +- Include/internal/pycore_runtime.h | 4 ++-- Modules/_testcapimodule.c | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index e7f067d5036156..efbaed60a7f23a 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1748,7 +1748,7 @@ Python-level trace functions in previous versions. .. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) - The type of the trace function registered using :c:func:`PyRefTracer_SetTracer` + The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. The first parameter is a Python object that has been just created (when **event** is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 2855e329561535..4bb37fabbea494 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -128,7 +128,7 @@ typedef struct _Py_DebugOffsets { /* Reference tracer state */ struct _reftracer_runtime_state { PyRefTracer tracer_func; - void* data; + void* tracer_data; }; /* Full Python runtime state */ @@ -235,7 +235,7 @@ typedef struct pyruntimestate { struct _fileutils_state fileutils; struct _faulthandler_runtime_state faulthandler; struct _tracemalloc_runtime_state tracemalloc; - struct _reftracer_runtime_state reftracer; + struct _reftracer_runtime_state ref_tracer; // The rwmutex is used to prevent overlapping global and per-interpreter // stop-the-world events. Global stop-the-world events lock the mutex diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b9fda3b0782e16..0d92a1c8253f61 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3292,7 +3292,7 @@ struct simpletracer_data { static int _simpletracer(PyObject *obj, PyRefTracerEvent event, void* data) { struct simpletracer_data* the_data = (struct simpletracer_data*)data; - assert(the_data->create_count + the_data->destroy_count < 10); + assert(the_data->create_count + the_data->destroy_count < Py_ARRAY_LENGTH(the_data->addresses)); the_data->addresses[the_data->create_count + the_data->destroy_count] = obj; if (event == PyRefTracer_CREATE) { the_data->create_count++; @@ -3306,7 +3306,7 @@ static PyObject * test_reftracer(PyObject *ob, PyObject *Py_UNUSED(ignored)) { struct simpletracer_data tracer_data = {0}; - void* the_data = (void*)&tracer_data; + void* the_data = &tracer_data; // Install a simple tracer function PyRefTracer_SetTracer(_simpletracer, the_data); From 9b3ddce47e94d36dfe28fbacae02773f7354dd33 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 1 May 2024 14:14:40 +0100 Subject: [PATCH 3/9] Address feedback --- Include/internal/pycore_object.h | 2 +- Include/internal/pycore_runtime_init.h | 3 ++- Modules/_testcapimodule.c | 6 ++++-- Objects/object.c | 26 +++++++++++++------------- Python/tracemalloc.c | 5 ++--- 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index f01063bc613a5e..98e439e81c7bc4 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -252,7 +252,7 @@ extern int _PyDict_CheckConsistency(PyObject *mp, int check_content); when a memory block is reused from a free list. Internal function called by _Py_NewReference(). */ -extern int _PyTraceMalloc_NewReference(PyObject *op, PyRefTracerEvent event, void*); +extern int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, void*); // Fast inlined version of PyType_HasFeature() static inline int diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 7dbc761f361566..a0326bea08c2ba 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -119,8 +119,9 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .faulthandler = _faulthandler_runtime_state_INIT, \ .tracemalloc = _tracemalloc_runtime_state_INIT, \ - .reftracer = { \ + .ref_tracer = { \ .tracer_func = NULL, \ + .tracer_data = NULL, \ }, \ .stoptheworld = { \ .is_global = 1, \ diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 0d92a1c8253f61..ceb8c6eeae54c2 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3308,12 +3308,14 @@ test_reftracer(PyObject *ob, PyObject *Py_UNUSED(ignored)) struct simpletracer_data tracer_data = {0}; void* the_data = &tracer_data; // Install a simple tracer function - PyRefTracer_SetTracer(_simpletracer, the_data); + if (PyRefTracer_SetTracer(_simpletracer, the_data) != 0) { + return NULL; + } // Check that the tracer was correctly installed void* data; if (PyRefTracer_GetTracer(&data) != _simpletracer || data != the_data) { - PyErr_SetString(PyExc_ValueError, "The reftracer not correctly installed"); + PyErr_SetString(PyExc_AssertionError, "The reftracer not correctly installed"); PyRefTracer_SetTracer(NULL, NULL); return NULL; } diff --git a/Objects/object.c b/Objects/object.c index cc0942acf2ea5f..3ead45995daa46 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2381,9 +2381,9 @@ new_reference(PyObject *op) #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif - if (_PyRuntime.reftracer.tracer_func != NULL) { - void* data = _PyRuntime.reftracer.data; - _PyRuntime.reftracer.tracer_func(op, PyRefTracer_CREATE, data); + if (_PyRuntime.ref_tracer.tracer_func != NULL) { + void* data = _PyRuntime.ref_tracer.tracer_data; + _PyRuntime.ref_tracer.tracer_func(op, PyRefTracer_CREATE, data); } } @@ -2408,9 +2408,9 @@ _Py_ResurrectReference(PyObject *op) #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif - if (_PyRuntime.reftracer.tracer_func != NULL) { - void* data = _PyRuntime.reftracer.data; - _PyRuntime.reftracer.tracer_func(op, PyRefTracer_CREATE, data); + if (_PyRuntime.ref_tracer.tracer_func != NULL) { + void* data = _PyRuntime.ref_tracer.tracer_data; + _PyRuntime.ref_tracer.tracer_func(op, PyRefTracer_CREATE, data); } } @@ -2885,9 +2885,9 @@ _Py_Dealloc(PyObject *op) Py_INCREF(type); #endif - if (_PyRuntime.reftracer.tracer_func != NULL) { - void* data = _PyRuntime.reftracer.data; - _PyRuntime.reftracer.tracer_func(op, PyRefTracer_DESTROY, data); + if (_PyRuntime.ref_tracer.tracer_func != NULL) { + void* data = _PyRuntime.ref_tracer.tracer_data; + _PyRuntime.ref_tracer.tracer_func(op, PyRefTracer_DESTROY, data); } #ifdef Py_TRACE_REFS @@ -2980,16 +2980,16 @@ _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt) int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) { assert(PyGILState_Check()); - _PyRuntime.reftracer.tracer_func = tracer; - _PyRuntime.reftracer.data = data; + _PyRuntime.ref_tracer.tracer_func = tracer; + _PyRuntime.ref_tracer.tracer_data = data; return 0; } PyRefTracer PyRefTracer_GetTracer(void** data) { assert(PyGILState_Check()); if (data != NULL) { - *data = _PyRuntime.reftracer.data; + *data = _PyRuntime.ref_tracer.tracer_data; } - return _PyRuntime.reftracer.tracer_func; + return _PyRuntime.ref_tracer.tracer_func; } diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 0a4f4a3a2118fe..18388a15ee861f 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -906,7 +906,7 @@ _PyTraceMalloc_Start(int max_nframe) return -1; } - _PyRuntime.reftracer.tracer_func = _PyTraceMalloc_NewReference; + PyRefTracer_SetTracer(_PyTraceMalloc_TraceRef, NULL); if (tracemalloc_config.tracing) { /* hook already installed: do nothing */ @@ -1354,8 +1354,7 @@ _PyTraceMalloc_Fini(void) Do nothing if tracemalloc is not tracing memory allocations or if the object memory block is not already traced. */ int -_PyTraceMalloc_NewReference(PyObject *op, PyRefTracerEvent event, void* Py_UNUSED(ignore)) - +_PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, void* Py_UNUSED(ignore)) { if (event != PyRefTracer_CREATE) { return 0; From be2380d0fe3f158592d5c62a918b9572444fc220 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 1 May 2024 14:21:55 +0100 Subject: [PATCH 4/9] Address feedback --- Modules/_testcapimodule.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index a7a31249b9a10c..8ddc9c73a21e6c 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3227,7 +3227,7 @@ struct simpletracer_data { static int _simpletracer(PyObject *obj, PyRefTracerEvent event, void* data) { struct simpletracer_data* the_data = (struct simpletracer_data*)data; - assert(the_data->create_count + the_data->destroy_count < Py_ARRAY_LENGTH(the_data->addresses)); + assert(the_data->create_count + the_data->destroy_count < (int)Py_ARRAY_LENGTH(the_data->addresses)); the_data->addresses[the_data->create_count + the_data->destroy_count] = obj; if (event == PyRefTracer_CREATE) { the_data->create_count++; @@ -3240,11 +3240,15 @@ static int _simpletracer(PyObject *obj, PyRefTracerEvent event, void* data) { static PyObject * test_reftracer(PyObject *ob, PyObject *Py_UNUSED(ignored)) { + // Save the current tracer and data to restore it later + void* current_data; + PyRefTracer current_tracer = PyRefTracer_GetTracer(¤t_data); + struct simpletracer_data tracer_data = {0}; void* the_data = &tracer_data; // Install a simple tracer function if (PyRefTracer_SetTracer(_simpletracer, the_data) != 0) { - return NULL; + goto failed; } // Check that the tracer was correctly installed @@ -3252,18 +3256,18 @@ test_reftracer(PyObject *ob, PyObject *Py_UNUSED(ignored)) if (PyRefTracer_GetTracer(&data) != _simpletracer || data != the_data) { PyErr_SetString(PyExc_AssertionError, "The reftracer not correctly installed"); PyRefTracer_SetTracer(NULL, NULL); - return NULL; + goto failed; } // Create a bunch of objects PyObject* obj = PyList_New(0); if (obj == NULL) { - return NULL; + goto failed; } PyObject* obj2 = PyDict_New(); if (obj2 == NULL) { Py_DECREF(obj); - return NULL; + goto failed; } // Kill all objects @@ -3276,24 +3280,27 @@ test_reftracer(PyObject *ob, PyObject *Py_UNUSED(ignored)) // Check that the tracer was removed if (PyRefTracer_GetTracer(&data) != NULL || data != NULL) { PyErr_SetString(PyExc_ValueError, "The reftracer was not correctly removed"); - return NULL; + goto failed; } if (tracer_data.create_count != 2 || tracer_data.addresses[0] != obj || tracer_data.addresses[1] != obj2) { PyErr_SetString(PyExc_ValueError, "The object creation was not correctly traced"); - return NULL; + goto failed; } if (tracer_data.destroy_count != 2 || tracer_data.addresses[2] != obj || tracer_data.addresses[3] != obj2) { PyErr_SetString(PyExc_ValueError, "The object destruction was not correctly traced"); - return NULL; + goto failed; } - + PyRefTracer_SetTracer(current_tracer, current_data); Py_RETURN_NONE; +failed: + PyRefTracer_SetTracer(current_tracer, current_data); + return NULL; } static PyMethodDef TestMethods[] = { From a326f3f087c4e7b52e2dbee57d56e49e76ce30cf Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 1 May 2024 14:25:56 +0100 Subject: [PATCH 5/9] improve docs --- Doc/c-api/init.rst | 26 +++++++++++++++----------- Doc/whatsnew/3.13.rst | 5 +++++ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 82cec5747d0377..77c9c9f0907bb7 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1925,23 +1925,27 @@ Python-level trace functions in previous versions. object has been destroyed. .. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) - - Register a reference tracer function. The function will be called when a new Python - has been created or when an object is going to be destroyed. If **data** is provided - it must be an opaque pointer that will be provided when the tracer function is called. - - Not that tracer functions **must not** create Python objects inside or otherwise the - call will be re-entrant. - + + Register a reference tracer function. The function will be called when a new + Python has been created or when an object is going to be destroyed. If + **data** is provided it must be an opaque pointer that will be provided when + the tracer function is called. + + Not that tracer functions **must not** create Python objects inside or + otherwise the call will be re-entrant. The tracer also **must not** clear + any existing exception or set an exception. The GIL will be held every time + the tracer function is called. + The GIL must be held when calling this function. .. versionadded:: 3.13 .. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) - Get the registered reference tracer function and the value of the opaque data pointer that - was registered when :c:func:`PyRefTracer_SetTracer` was called. If no tracer was registered - this function will return NULL and will set the **data** pointer to NULL. + Get the registered reference tracer function and the value of the opaque data + pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. + If no tracer was registered this function will return NULL and will set the + **data** pointer to NULL. The GIL must be held when calling this function. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index ee50effd662f12..f3d156ec9df170 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1936,6 +1936,11 @@ New Features * Add :c:func:`PyType_GetModuleByDef` to the limited C API (Contributed by Victor Stinner in :gh:`116936`.) +* Add two new functions to the C-API, :c:func:`PyRefTracer_SetTracer` and + :c:func:`PyRefTracer_GetTracer`, that allows to track object creation and + destruction the same way the :mod:`tracemalloc` module does. (Contributed + by Pablo Galindo in :gh:`93502`.) + Porting to Python 3.13 ---------------------- From b1ce660719dd11bb040f6ad414656447e248d04b Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 1 May 2024 14:41:43 +0100 Subject: [PATCH 6/9] Fix linting --- Doc/c-api/init.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 77c9c9f0907bb7..0b6d98a39ee137 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1925,17 +1925,17 @@ Python-level trace functions in previous versions. object has been destroyed. .. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) - + Register a reference tracer function. The function will be called when a new Python has been created or when an object is going to be destroyed. If **data** is provided it must be an opaque pointer that will be provided when the tracer function is called. - + Not that tracer functions **must not** create Python objects inside or otherwise the call will be re-entrant. The tracer also **must not** clear any existing exception or set an exception. The GIL will be held every time the tracer function is called. - + The GIL must be held when calling this function. .. versionadded:: 3.13 From e74204f4000609d3bf6e7e08b8b66279c007a5db Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 1 May 2024 16:47:13 +0100 Subject: [PATCH 7/9] Apply suggestions from code review Co-authored-by: Victor Stinner --- Modules/_testcapimodule.c | 4 ++-- Objects/object.c | 14 ++++++++------ Python/tracemalloc.c | 4 +++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 8ddc9c73a21e6c..f5892fc5ed2a2c 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3255,7 +3255,7 @@ test_reftracer(PyObject *ob, PyObject *Py_UNUSED(ignored)) void* data; if (PyRefTracer_GetTracer(&data) != _simpletracer || data != the_data) { PyErr_SetString(PyExc_AssertionError, "The reftracer not correctly installed"); - PyRefTracer_SetTracer(NULL, NULL); + (void)PyRefTracer_SetTracer(NULL, NULL); goto failed; } @@ -3275,7 +3275,7 @@ test_reftracer(PyObject *ob, PyObject *Py_UNUSED(ignored)) Py_DECREF(obj2); // Remove the tracer - PyRefTracer_SetTracer(NULL, NULL); + (void)PyRefTracer_SetTracer(NULL, NULL); // Check that the tracer was removed if (PyRefTracer_GetTracer(&data) != NULL || data != NULL) { diff --git a/Objects/object.c b/Objects/object.c index 486d45aa3ee82c..79e4fb4dbbf7c6 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2386,9 +2386,10 @@ new_reference(PyObject *op) #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif - if (_PyRuntime.ref_tracer.tracer_func != NULL) { - void* data = _PyRuntime.ref_tracer.tracer_data; - _PyRuntime.ref_tracer.tracer_func(op, PyRefTracer_CREATE, data); + struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; + if (tracer->tracer_func != NULL) { + void* data = tracer->tracer_data; + tracer->tracer_func(op, PyRefTracer_CREATE, data); } } @@ -2847,9 +2848,10 @@ _Py_Dealloc(PyObject *op) Py_INCREF(type); #endif - if (_PyRuntime.ref_tracer.tracer_func != NULL) { - void* data = _PyRuntime.ref_tracer.tracer_data; - _PyRuntime.ref_tracer.tracer_func(op, PyRefTracer_DESTROY, data); + struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; + if (tracer->tracer_func != NULL) { + void* data = tracer->tracer_data; + tracer->tracer_func(op, PyRefTracer_DESTROY, data); } #ifdef Py_TRACE_REFS diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 18388a15ee861f..e3ec72062f6931 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -906,7 +906,9 @@ _PyTraceMalloc_Start(int max_nframe) return -1; } - PyRefTracer_SetTracer(_PyTraceMalloc_TraceRef, NULL); + if (PyRefTracer_SetTracer(_PyTraceMalloc_TraceRef, NULL) < 0) { + return -1; + } if (tracemalloc_config.tracing) { /* hook already installed: do nothing */ From 0a3c9ddf7c5fd031854aa13f97b0f2647805950c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 1 May 2024 16:53:18 +0100 Subject: [PATCH 8/9] improve docs --- Doc/c-api/init.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 0b6d98a39ee137..bb4556b1d64127 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1904,6 +1904,11 @@ Python-level trace functions in previous versions. .. versionadded:: 3.12 +Reference tracing +================= + +.. versionadded:: 3.13 + .. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. @@ -1929,7 +1934,8 @@ Python-level trace functions in previous versions. Register a reference tracer function. The function will be called when a new Python has been created or when an object is going to be destroyed. If **data** is provided it must be an opaque pointer that will be provided when - the tracer function is called. + the tracer function is called. Return ``0`` on success. Set an exception and + return ``-1`` on error. Not that tracer functions **must not** create Python objects inside or otherwise the call will be re-entrant. The tracer also **must not** clear From 7fc599ea320dc71708ec9689d6acce3955e9523e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 2 May 2024 19:00:30 +0200 Subject: [PATCH 9/9] Fixes --- Doc/c-api/init.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index bb4556b1d64127..9e118d4f36145f 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1935,7 +1935,7 @@ Reference tracing Python has been created or when an object is going to be destroyed. If **data** is provided it must be an opaque pointer that will be provided when the tracer function is called. Return ``0`` on success. Set an exception and - return ``-1`` on error. + return ``-1`` on error. Not that tracer functions **must not** create Python objects inside or otherwise the call will be re-entrant. The tracer also **must not** clear