From 07130345a9fc76873bdaf34c3929b13099bad4d0 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 15 Nov 2024 12:14:28 +0100 Subject: [PATCH 01/19] Add freelist of compact int objects --- Include/internal/pycore_freelist_state.h | 2 ++ Include/internal/pycore_long.h | 2 ++ Objects/longobject.c | 36 +++++++++++------------- Python/bytecodes.c | 21 +++++++------- 4 files changed, 32 insertions(+), 29 deletions(-) diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h index 4e04cf431e0b31..a1a94c1f2dc880 100644 --- a/Include/internal/pycore_freelist_state.h +++ b/Include/internal/pycore_freelist_state.h @@ -14,6 +14,7 @@ extern "C" { # define Py_dicts_MAXFREELIST 80 # define Py_dictkeys_MAXFREELIST 80 # define Py_floats_MAXFREELIST 100 +# define Py_ints_MAXFREELIST 100 # define Py_slices_MAXFREELIST 1 # define Py_contexts_MAXFREELIST 255 # define Py_async_gens_MAXFREELIST 80 @@ -35,6 +36,7 @@ struct _Py_freelist { struct _Py_freelists { struct _Py_freelist floats; + struct _Py_freelist ints; struct _Py_freelist tuples[PyTuple_MAXSAVESIZE]; struct _Py_freelist lists; struct _Py_freelist dicts; diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 196b4152280a35..e17a9dbf2d9c72 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -55,6 +55,8 @@ extern void _PyLong_FiniTypes(PyInterpreterState *interp); /* other API */ +void _PyLong_Free(PyLongObject *op); + #define _PyLong_SMALL_INTS _Py_SINGLETON(small_ints) // _PyLong_GetZero() and _PyLong_GetOne() must always be available diff --git a/Objects/longobject.c b/Objects/longobject.c index 4aa35685b509f2..2fd931afa39ceb 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6,6 +6,7 @@ #include "pycore_bitutils.h" // _Py_popcount32() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_call.h" // _PyObject_MakeTpCall +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_long.h" // _Py_SmallInts #include "pycore_object.h" // _PyObject_Init() #include "pycore_runtime.h" // _PY_NSMALLPOSINTS @@ -221,10 +222,14 @@ _PyLong_FromMedium(sdigit x) assert(!IS_SMALL_INT(x)); assert(is_medium_int(x)); /* We could use a freelist here */ - PyLongObject *v = PyObject_Malloc(sizeof(PyLongObject)); + + PyLongObject *v = _Py_FREELIST_POP(PyLongObject, ints); if (v == NULL) { - PyErr_NoMemory(); - return NULL; + v = PyObject_Malloc(sizeof(PyLongObject)); + if (v == NULL) { + PyErr_NoMemory(); + return NULL; + } } digit abs_x = x < 0 ? -x : x; _PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1); @@ -3614,24 +3619,17 @@ long_richcompare(PyObject *self, PyObject *other, int op) static void long_dealloc(PyObject *self) { - /* This should never get called, but we also don't want to SEGV if - * we accidentally decref small Ints out of existence. Instead, - * since small Ints are immortal, re-set the reference count. - */ - PyLongObject *pylong = (PyLongObject*)self; - if (pylong && _PyLong_IsCompact(pylong)) { - stwodigits ival = medium_value(pylong); - if (IS_SMALL_INT(ival)) { - PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival); - if (pylong == small_pylong) { - _Py_SetImmortal(self); - return; - } - } - } Py_TYPE(self)->tp_free(self); } +void _PyLong_Free(PyLongObject *op) { + if (_PyLong_IsCompact(op)) { + _Py_FREELIST_FREE(ints, op, PyObject_Free); + return; + } + PyObject_Free(op); +} + static Py_hash_t long_hash(PyObject *obj) { @@ -6615,7 +6613,7 @@ PyTypeObject PyLong_Type = { 0, /* tp_init */ 0, /* tp_alloc */ long_new, /* tp_new */ - PyObject_Free, /* tp_free */ + _PyLong_Free, /* tp_free */ .tp_vectorcall = long_vectorcall, .tp_version_tag = _Py_TYPE_VERSION_INT, }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 04983fd861ec59..141ba5a6868d09 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -26,6 +26,7 @@ #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject +#include "pycore_long.h" // void _PyLong_Free(PyLongObject *op); #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs #include "pycore_tuple.h" // _PyTuple_ITEMS() @@ -514,8 +515,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -527,8 +528,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -540,8 +541,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -797,7 +798,7 @@ dummy_func( PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); DEAD(sub_st); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -817,7 +818,7 @@ dummy_func( DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); DEAD(sub_st); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -838,7 +839,7 @@ dummy_func( PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); DEAD(sub_st); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -950,7 +951,7 @@ dummy_func( PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free ); DEAD(sub_st); PyStackRef_CLOSE(list_st); } From be58ade742e286cc1e957257c11eaf2de4f2f6e9 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 15 Nov 2024 12:32:12 +0100 Subject: [PATCH 02/19] fix build --- Objects/longobject.c | 4 ++-- Python/executor_cases.c.h | 20 ++++++++++---------- Python/generated_cases.c.h | 20 ++++++++++---------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 2fd931afa39ceb..009237c92d5e36 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -43,7 +43,7 @@ static inline void _Py_DECREF_INT(PyLongObject *op) { assert(PyLong_CheckExact(op)); - _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)_PyLong_Free); } static inline int @@ -6613,7 +6613,7 @@ PyTypeObject PyLong_Type = { 0, /* tp_init */ 0, /* tp_alloc */ long_new, /* tp_new */ - _PyLong_Free, /* tp_free */ + (freefunc)_PyLong_Free, /* tp_free */ .tp_vectorcall = long_vectorcall, .tp_version_tag = _Py_TYPE_VERSION_INT, }; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 494ace1bd85822..029782a233c0c9 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -626,8 +626,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -646,8 +646,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -666,8 +666,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -991,7 +991,7 @@ PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1033,7 +1033,7 @@ } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1072,7 +1072,7 @@ PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1247,7 +1247,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free ); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 77bf6ad3781f17..2e3e3a1e9c1d55 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -118,8 +118,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -287,8 +287,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -358,8 +358,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -584,7 +584,7 @@ PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -616,7 +616,7 @@ DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -648,7 +648,7 @@ PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -7707,7 +7707,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free ); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); From 3f50b54741aef4d404f8143973d68ffffc03cabb Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 15 Nov 2024 13:21:39 +0100 Subject: [PATCH 03/19] cleanup freelist at exit --- Objects/object.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/object.c b/Objects/object.c index 052dea9ad1feff..75e57391199592 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -889,6 +889,7 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear() // In the default build, freelists are per-interpreter and cleared in finalize_interp_types() clear_freelist(&freelists->floats, is_finalization, free_object); + clear_freelist(&freelists->ints, is_finalization, free_object); for (Py_ssize_t i = 0; i < PyTuple_MAXSAVESIZE; i++) { clear_freelist(&freelists->tuples[i], is_finalization, free_object); } From fa97302acab37799217c7dbbb62792a8dbbf8457 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 16 Nov 2024 19:59:35 +0100 Subject: [PATCH 04/19] fix memory leak; align with float implementation --- Include/internal/pycore_long.h | 3 ++- Objects/longobject.c | 34 ++++++++++++++++++++++++++++++++-- Objects/object.c | 2 +- Python/bytecodes.c | 22 +++++++++++----------- Python/executor_cases.c.h | 20 ++++++++++---------- Python/generated_cases.c.h | 20 ++++++++++---------- 6 files changed, 66 insertions(+), 35 deletions(-) diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index e17a9dbf2d9c72..1d1a3c9467a090 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -55,7 +55,8 @@ extern void _PyLong_FiniTypes(PyInterpreterState *interp); /* other API */ -void _PyLong_Free(PyLongObject *op); +void +_PyLong_ExactDealloc(PyObject *self); #define _PyLong_SMALL_INTS _Py_SINGLETON(small_ints) diff --git a/Objects/longobject.c b/Objects/longobject.c index 009237c92d5e36..3d7b28a7335b22 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -43,7 +43,7 @@ static inline void _Py_DECREF_INT(PyLongObject *op) { assert(PyLong_CheckExact(op)); - _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)_PyLong_Free); + _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)_PyLong_ExactDealloc); } static inline int @@ -3616,9 +3616,39 @@ long_richcompare(PyObject *self, PyObject *other, int op) Py_RETURN_RICHCOMPARE(result, 0, op); } +void +_PyLong_ExactDealloc(PyObject *self) +{ + assert(PyLong_CheckExact(self)); + + if (_PyLong_IsCompact((PyLongObject *)self)) { + _Py_FREELIST_FREE(ints, self, PyObject_Free); + return; + } + PyObject_Free(self); +} + static void long_dealloc(PyObject *self) { + PyLongObject *pylong = (PyLongObject*)self; + + if (pylong && _PyLong_IsCompact(pylong)) { + stwodigits ival = medium_value(pylong); + if (IS_SMALL_INT(ival)) { + PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival); + if (pylong == small_pylong) { + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref small Ints out of existence. Instead, + * since small Ints are immortal, re-set the reference count. + */ + // can we remove the next two lines? the immortal objects now have a fixed refcount + // in particular in the free-threading build this seeems safe + _Py_SetImmortal(self); + return; + } + } + } Py_TYPE(self)->tp_free(self); } @@ -6613,7 +6643,7 @@ PyTypeObject PyLong_Type = { 0, /* tp_init */ 0, /* tp_alloc */ long_new, /* tp_new */ - (freefunc)_PyLong_Free, /* tp_free */ + (freefunc)PyObject_Free, /* tp_free */ .tp_vectorcall = long_vectorcall, .tp_version_tag = _Py_TYPE_VERSION_INT, }; diff --git a/Objects/object.c b/Objects/object.c index 75e57391199592..64b7de85a6147e 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -889,7 +889,6 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear() // In the default build, freelists are per-interpreter and cleared in finalize_interp_types() clear_freelist(&freelists->floats, is_finalization, free_object); - clear_freelist(&freelists->ints, is_finalization, free_object); for (Py_ssize_t i = 0; i < PyTuple_MAXSAVESIZE; i++) { clear_freelist(&freelists->tuples[i], is_finalization, free_object); } @@ -907,6 +906,7 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree); } clear_freelist(&freelists->unicode_writers, is_finalization, PyMem_Free); + clear_freelist(&freelists->ints, is_finalization, free_object); } /* diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 141ba5a6868d09..c69706edd84203 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -26,7 +26,7 @@ #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject -#include "pycore_long.h" // void _PyLong_Free(PyLongObject *op); +#include "pycore_long.h" // void _PyLong_ExactDealloc(PyLongObject *op); #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs #include "pycore_tuple.h" // _PyTuple_ITEMS() @@ -515,8 +515,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -528,8 +528,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -541,8 +541,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -798,7 +798,7 @@ dummy_func( PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -818,7 +818,7 @@ dummy_func( DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -839,7 +839,7 @@ dummy_func( PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -951,7 +951,7 @@ dummy_func( PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free ); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc ); DEAD(sub_st); PyStackRef_CLOSE(list_st); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 029782a233c0c9..f7055eba714075 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -626,8 +626,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -646,8 +646,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -666,8 +666,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -991,7 +991,7 @@ PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1033,7 +1033,7 @@ } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1072,7 +1072,7 @@ PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1247,7 +1247,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free ); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc ); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2e3e3a1e9c1d55..73f376e4a870da 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -118,8 +118,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -287,8 +287,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -358,8 +358,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -584,7 +584,7 @@ PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -616,7 +616,7 @@ DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -648,7 +648,7 @@ PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -7707,7 +7707,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free ); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc ); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); From d72486f5eeb7ca8e6f8c9523bcb4b1b0828d74ba Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 16 Nov 2024 20:00:16 +0100 Subject: [PATCH 05/19] remove stale comment --- Objects/longobject.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 3d7b28a7335b22..eec395351b6a4d 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -221,7 +221,6 @@ _PyLong_FromMedium(sdigit x) { assert(!IS_SMALL_INT(x)); assert(is_medium_int(x)); - /* We could use a freelist here */ PyLongObject *v = _Py_FREELIST_POP(PyLongObject, ints); if (v == NULL) { From e07c2188052873c0e4298128bdbc7e92e39bb026 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 16 Nov 2024 20:01:21 +0100 Subject: [PATCH 06/19] remove unused function --- Objects/longobject.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index eec395351b6a4d..7b55fb272d16a8 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3651,14 +3651,6 @@ long_dealloc(PyObject *self) Py_TYPE(self)->tp_free(self); } -void _PyLong_Free(PyLongObject *op) { - if (_PyLong_IsCompact(op)) { - _Py_FREELIST_FREE(ints, op, PyObject_Free); - return; - } - PyObject_Free(op); -} - static Py_hash_t long_hash(PyObject *obj) { From 328e0c164346999aa43401813ce6ed55edd72a8d Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 16 Nov 2024 22:22:55 +0100 Subject: [PATCH 07/19] avoid some problematic decrefs --- Objects/longobject.c | 2 +- Python/bytecodes.c | 8 ++++---- Python/executor_cases.c.h | 8 ++++---- Python/generated_cases.c.h | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 7b55fb272d16a8..dc19d488530de0 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -43,7 +43,7 @@ static inline void _Py_DECREF_INT(PyLongObject *op) { assert(PyLong_CheckExact(op)); - _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)_PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)PyObject_Free); // needs to be converted to freelist? } static inline int diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c69706edd84203..933d79c163c713 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -516,7 +516,7 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -529,7 +529,7 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -542,7 +542,7 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -951,7 +951,7 @@ dummy_func( PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc ); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(list_st); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index f7055eba714075..d2c50b36a0edb1 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -627,7 +627,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -647,7 +647,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -667,7 +667,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1247,7 +1247,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc ); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 73f376e4a870da..b5b07d9c66a21c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -119,7 +119,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -288,7 +288,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -359,7 +359,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -7707,7 +7707,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc ); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); From e1dc2b3d872e91c7ac20cddf5d9aa4a617d0c255 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 16 Nov 2024 23:36:32 +0100 Subject: [PATCH 08/19] jit build --- Include/internal/pycore_long.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 1d1a3c9467a090..8bead00e70640c 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -55,8 +55,7 @@ extern void _PyLong_FiniTypes(PyInterpreterState *interp); /* other API */ -void -_PyLong_ExactDealloc(PyObject *self); +PyAPI_FUNC(void) _PyLong_ExactDealloc(PyObject *self); #define _PyLong_SMALL_INTS _Py_SINGLETON(small_ints) From 9df776b599995e4cb8d2dab60f0382f16895ebfa Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 16 Nov 2024 22:37:49 +0000 Subject: [PATCH 09/19] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst new file mode 100644 index 00000000000000..abba62811e3758 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst @@ -0,0 +1 @@ +Increase performance of int by adding a freelist for compact ints. From db8247e0aa20aa33a17af6b342cc138117302ab4 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 21 Nov 2024 03:13:37 +0800 Subject: [PATCH 10/19] Apply review suggestions --- Objects/longobject.c | 18 ++++++++++-------- Python/bytecodes.c | 6 +++--- Python/executor_cases.c.h | 6 +++--- Python/generated_cases.c.h | 6 +++--- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index dc19d488530de0..3d16a5d1e53d9a 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -43,7 +43,7 @@ static inline void _Py_DECREF_INT(PyLongObject *op) { assert(PyLong_CheckExact(op)); - _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)PyObject_Free); // needs to be converted to freelist? + _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor) _PyLong_ExactDealloc); } static inline int @@ -222,7 +222,9 @@ _PyLong_FromMedium(sdigit x) assert(!IS_SMALL_INT(x)); assert(is_medium_int(x)); - PyLongObject *v = _Py_FREELIST_POP(PyLongObject, ints); + // The small int cache is incompatible with _Py_NewReference which is called + // by _Py_FREELIST_POP. + PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP_MEM(ints); if (v == NULL) { v = PyObject_Malloc(sizeof(PyLongObject)); if (v == NULL) { @@ -3618,13 +3620,13 @@ long_richcompare(PyObject *self, PyObject *other, int op) void _PyLong_ExactDealloc(PyObject *self) { - assert(PyLong_CheckExact(self)); - - if (_PyLong_IsCompact((PyLongObject *)self)) { - _Py_FREELIST_FREE(ints, self, PyObject_Free); - return; + if (PyLong_CheckExact(self)) { + if (_PyLong_IsCompact((PyLongObject *)self)) { + _Py_FREELIST_FREE(ints, self, PyObject_Free); + return; + } } - PyObject_Free(self); + Py_TYPE(self)->tp_free(self); } static void diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f24ca0f62755d9..7d61fc48e21ea4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -516,7 +516,7 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -529,7 +529,7 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -542,7 +542,7 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 2a0e07fb1e8c2b..73be53d5a2ce46 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -627,7 +627,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -647,7 +647,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -667,7 +667,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e25d8b4785125c..1fb36cdde1a1c2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -119,7 +119,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -288,7 +288,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -359,7 +359,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } From d1e4aa268ed83dfcb35b18b040451a75bd003bee Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 21 Nov 2024 03:20:15 +0800 Subject: [PATCH 11/19] Fixup --- Objects/longobject.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 3d16a5d1e53d9a..d3cdba4365d4c2 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3620,13 +3620,11 @@ long_richcompare(PyObject *self, PyObject *other, int op) void _PyLong_ExactDealloc(PyObject *self) { - if (PyLong_CheckExact(self)) { - if (_PyLong_IsCompact((PyLongObject *)self)) { - _Py_FREELIST_FREE(ints, self, PyObject_Free); - return; - } + if (_PyLong_IsCompact((PyLongObject *)self)) { + _Py_FREELIST_FREE(ints, self, PyObject_Free); + return; } - Py_TYPE(self)->tp_free(self); + PyObject_Free(self); } static void @@ -3650,6 +3648,14 @@ long_dealloc(PyObject *self) } } } + + if (PyLong_CheckExact(self)) { + if (_PyLong_IsCompact((PyLongObject *)self)) { + _Py_FREELIST_FREE(ints, self, PyObject_Free); + return; + } + } + Py_TYPE(self)->tp_free(self); } From 644e85a5ae7a31b2cae09b9528928ff75d995cbf Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 21 Nov 2024 10:20:03 +0100 Subject: [PATCH 12/19] review comments part 1 --- Objects/longobject.c | 24 +++++++++++------------- Python/bytecodes.c | 2 +- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index d3cdba4365d4c2..1674a2e378fd20 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6,7 +6,7 @@ #include "pycore_bitutils.h" // _Py_popcount32() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_call.h" // _PyObject_MakeTpCall -#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() +#include "pycore_freelist.h" // _Py_FREELIST_FREE, _Py_FREELIST_POP #include "pycore_long.h" // _Py_SmallInts #include "pycore_object.h" // _PyObject_Init() #include "pycore_runtime.h" // _PY_NSMALLPOSINTS @@ -43,7 +43,7 @@ static inline void _Py_DECREF_INT(PyLongObject *op) { assert(PyLong_CheckExact(op)); - _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor) _PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED((PyObject *)op, _PyLong_ExactDealloc); } static inline int @@ -3631,26 +3631,24 @@ static void long_dealloc(PyObject *self) { PyLongObject *pylong = (PyLongObject*)self; + assert(pylong); - if (pylong && _PyLong_IsCompact(pylong)) { + if (_PyLong_IsCompact(pylong)) { stwodigits ival = medium_value(pylong); if (IS_SMALL_INT(ival)) { PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival); if (pylong == small_pylong) { /* This should never get called, but we also don't want to SEGV if - * we accidentally decref small Ints out of existence. Instead, - * since small Ints are immortal, re-set the reference count. - */ - // can we remove the next two lines? the immortal objects now have a fixed refcount - // in particular in the free-threading build this seeems safe + * we accidentally decref small Ints out of existence. Instead, + * since small Ints are immortal, re-set the reference count. + * + * With a fixed refcount for immortal objects this should not happen + */ _Py_SetImmortal(self); return; } } - } - - if (PyLong_CheckExact(self)) { - if (_PyLong_IsCompact((PyLongObject *)self)) { + if (PyLong_CheckExact(self)) { _Py_FREELIST_FREE(ints, self, PyObject_Free); return; } @@ -6642,7 +6640,7 @@ PyTypeObject PyLong_Type = { 0, /* tp_init */ 0, /* tp_alloc */ long_new, /* tp_new */ - (freefunc)PyObject_Free, /* tp_free */ + PyObject_Free, /* tp_free */ .tp_vectorcall = long_vectorcall, .tp_version_tag = _Py_TYPE_VERSION_INT, }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7d61fc48e21ea4..1112ed55000864 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -26,7 +26,7 @@ #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject -#include "pycore_long.h" // void _PyLong_ExactDealloc(PyLongObject *op); +#include "pycore_long.h" // _PyLong_ExactDealloc() #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs #include "pycore_tuple.h" // _PyTuple_ITEMS() From 9f86b6ef5beadeb3341f7afe856a4e5917f62f4d Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 21 Nov 2024 12:25:44 +0100 Subject: [PATCH 13/19] review comments part 2 --- Objects/longobject.c | 4 ++-- Python/bytecodes.c | 20 ++++++++++---------- Python/executor_cases.c.h | 20 ++++++++++---------- Python/generated_cases.c.h | 20 ++++++++++---------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 1674a2e378fd20..6eaa31bdcc18b3 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -222,8 +222,8 @@ _PyLong_FromMedium(sdigit x) assert(!IS_SMALL_INT(x)); assert(is_medium_int(x)); - // The small int cache is incompatible with _Py_NewReference which is called - // by _Py_FREELIST_POP. + // We use _Py_FREELIST_POP_MEM instead of _Py_FREELIST_POP because the new + // reference is created in _PyObject_Init PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP_MEM(ints); if (v == NULL) { v = PyObject_Malloc(sizeof(PyLongObject)); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 1112ed55000864..7b5ccc0708b17a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -515,8 +515,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -528,8 +528,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -541,8 +541,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -798,7 +798,7 @@ dummy_func( PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -818,7 +818,7 @@ dummy_func( DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -839,7 +839,7 @@ dummy_func( PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -951,7 +951,7 @@ dummy_func( PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(list_st); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 73be53d5a2ce46..0e5cdbfd806d51 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -626,8 +626,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -646,8 +646,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -666,8 +666,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -991,7 +991,7 @@ PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1033,7 +1033,7 @@ } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1072,7 +1072,7 @@ PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1247,7 +1247,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1fb36cdde1a1c2..95bc25e022dd2d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -118,8 +118,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -287,8 +287,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -358,8 +358,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -584,7 +584,7 @@ PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -616,7 +616,7 @@ DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -648,7 +648,7 @@ PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -7707,7 +7707,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); From 1e548bdced82f085b9baedf867e25180ba385d0a Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 23 Nov 2024 21:48:41 +0100 Subject: [PATCH 14/19] review suggestion --- Objects/longobject.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 6eaa31bdcc18b3..5b3f8167871d24 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -222,19 +222,17 @@ _PyLong_FromMedium(sdigit x) assert(!IS_SMALL_INT(x)); assert(is_medium_int(x)); - // We use _Py_FREELIST_POP_MEM instead of _Py_FREELIST_POP because the new - // reference is created in _PyObject_Init - PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP_MEM(ints); + PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints); if (v == NULL) { v = PyObject_Malloc(sizeof(PyLongObject)); if (v == NULL) { PyErr_NoMemory(); return NULL; } + _PyObject_Init((PyObject*)v, &PyLong_Type); } digit abs_x = x < 0 ? -x : x; _PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1); - _PyObject_Init((PyObject*)v, &PyLong_Type); v->long_value.ob_digit[0] = abs_x; return (PyObject*)v; } From 60220c05e6433f61ae8f6495f3f50e95f263aaec Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Mon, 2 Dec 2024 16:51:28 +0100 Subject: [PATCH 15/19] add small int check to _PyLong_ExactDealloc --- Objects/longobject.c | 50 +++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 5b3f8167871d24..fd4c2d515ae38b 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3615,10 +3615,32 @@ long_richcompare(PyObject *self, PyObject *other, int op) Py_RETURN_RICHCOMPARE(result, 0, op); } +static inline int +_PyLong_IsSmallInt(PyObject *self) +{ + PyLongObject *pylong = (PyLongObject *)self; + assert(_PyLong_IsCompact(pylong)); + stwodigits ival = medium_value(pylong); + if (IS_SMALL_INT(ival)) { + PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival); + if (pylong == small_pylong) { + return 1; + } + } + return 0; +} + void _PyLong_ExactDealloc(PyObject *self) { if (_PyLong_IsCompact((PyLongObject *)self)) { + #ifndef Py_GIL_DISABLED + if (_PyLong_IsSmallInt(self)) { + // See PEP 683, section Accidental De-Immortalizing for details + _Py_SetImmortal(self); + return; + } + #endif _Py_FREELIST_FREE(ints, self, PyObject_Free); return; } @@ -3628,23 +3650,17 @@ _PyLong_ExactDealloc(PyObject *self) static void long_dealloc(PyObject *self) { - PyLongObject *pylong = (PyLongObject*)self; - assert(pylong); - - if (_PyLong_IsCompact(pylong)) { - stwodigits ival = medium_value(pylong); - if (IS_SMALL_INT(ival)) { - PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival); - if (pylong == small_pylong) { - /* This should never get called, but we also don't want to SEGV if - * we accidentally decref small Ints out of existence. Instead, - * since small Ints are immortal, re-set the reference count. - * - * With a fixed refcount for immortal objects this should not happen - */ - _Py_SetImmortal(self); - return; - } + assert(self); + if (_PyLong_IsCompact((PyLongObject *)self)) { + if (_PyLong_IsSmallInt(self)) { + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref small Ints out of existence. Instead, + * since small Ints are immortal, re-set the reference count. + * + * See PEP 683, section Accidental De-Immortalizing for details + */ + _Py_SetImmortal(self); + return; } if (PyLong_CheckExact(self)) { _Py_FREELIST_FREE(ints, self, PyObject_Free); From 593d62182619db75659f4134bfb533db6c656438 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Mon, 2 Dec 2024 17:03:06 +0100 Subject: [PATCH 16/19] add COMPARE_OP_INT --- Objects/longobject.c | 1 + Python/bytecodes.c | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index fd4c2d515ae38b..0061b12aa22d88 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3633,6 +3633,7 @@ _PyLong_IsSmallInt(PyObject *self) void _PyLong_ExactDealloc(PyObject *self) { + assert(PyLong_CheckExact(self)); if (_PyLong_IsCompact((PyLongObject *)self)) { #ifndef Py_GIL_DISABLED if (_PyLong_IsSmallInt(self)) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 67af0096527d01..dbcb3078ddb45d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2477,9 +2477,9 @@ dummy_func( Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); DEAD(left); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); DEAD(right); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. From efde111a1b5ada77534807d5ce80bc9d668a7488 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 7 Dec 2024 23:11:19 +0100 Subject: [PATCH 17/19] review comment --- Objects/longobject.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 0061b12aa22d88..96d59f542a7c3c 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3616,7 +3616,7 @@ long_richcompare(PyObject *self, PyObject *other, int op) } static inline int -_PyLong_IsSmallInt(PyObject *self) +compact_int_is_small(PyObject *self) { PyLongObject *pylong = (PyLongObject *)self; assert(_PyLong_IsCompact(pylong)); @@ -3636,7 +3636,7 @@ _PyLong_ExactDealloc(PyObject *self) assert(PyLong_CheckExact(self)); if (_PyLong_IsCompact((PyLongObject *)self)) { #ifndef Py_GIL_DISABLED - if (_PyLong_IsSmallInt(self)) { + if (compact_int_is_small(self)) { // See PEP 683, section Accidental De-Immortalizing for details _Py_SetImmortal(self); return; @@ -3653,7 +3653,7 @@ long_dealloc(PyObject *self) { assert(self); if (_PyLong_IsCompact((PyLongObject *)self)) { - if (_PyLong_IsSmallInt(self)) { + if (compact_int_is_small(self)) { /* This should never get called, but we also don't want to SEGV if * we accidentally decref small Ints out of existence. Instead, * since small Ints are immortal, re-set the reference count. From 437c24c66d9429911ab9dba0c979aff2272e2790 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 7 Dec 2024 23:19:13 +0100 Subject: [PATCH 18/19] regenerate --- Python/executor_cases.c.h | 4 ++-- Python/generated_cases.c.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 90ee79aeb33a5c..bf970aa42df0cd 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3075,8 +3075,8 @@ Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2dafb0cd7de89b..c3c58d9972346a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3337,8 +3337,8 @@ Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } From b0349488f2e4239864517ead91281accb2c0d920 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 7 Dec 2024 23:46:05 +0100 Subject: [PATCH 19/19] Update Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst --- .../2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst index abba62811e3758..fd1570908c1fd6 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst @@ -1 +1 @@ -Increase performance of int by adding a freelist for compact ints. +Increase performance of :class:`int` by adding a freelist for compact ints.