8000 bpo-36389: _PyObject_CheckConsistency() available in release mode (GH… · python/cpython@6876257 · GitHub
[go: up one dir, main page]
Skip to content
8000

Commit 6876257

Browse files
vstinnervstinner
authored
bpo-36389: _PyObject_CheckConsistency() available in release mode (GH-16612)
bpo-36389, bpo-38376: The _PyObject_CheckConsistency() function is now also available in release mode. For example, it can be used to debug a crash in the visit_decref() function of the GC. Modify the following functions to also work in release mode: * _PyDict_CheckConsistency() * _PyObject_CheckConsistency() * _PyType_CheckConsistency() * _PyUnicode_CheckConsistency() Other changes: * _PyMem_IsPtrFreed(ptr) now also returns 1 if ptr is NULL (equals to 0). * _PyBytesWriter_CheckConsistency() now returns 1 and is only used with assert(). * Reorder _PyObject_Dump() to write safe fields first, and only attempt to render repr() at the end.
1 parent 321def8 commit 6876257

File tree

14 files changed

+148
-133
lines changed

14 files changed

+148
-133
lines changed

‎Include/cpython/unicodeobject.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,10 @@ typedef struct {
246246
} data; /* Canonical, smallest-form Unicode buffer */
247247
} PyUnicodeObject;
248248

249+
PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
250+
PyObject *op,
251+
int check_content);
252+
249253
/* Fast access macros */
250254
#define PyUnicode_WSTR_LENGTH(op) \
251255
(PyUnicode_IS_COMPACT_ASCII(op) ? \

‎Include/internal/pycore_object.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ extern "C" {
1111
#include "pycore_pystate.h" /* _PyRuntime.gc */
1212

1313
PyAPI_FUNC(int) _PyType_CheckConsistency(PyTypeObject *type);
14-
PyAPI_FUNC(int) _PyUnicode_CheckConsistency(PyObject *op, int check_content);
1514
PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content);
1615

1716
/* Tell the GC to track this object.

‎Include/internal/pycore_pymem.h

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -155,8 +155,9 @@ PyAPI_FUNC(int) _PyMem_SetDefaultAllocator(
155155
PyMemAllocatorEx *old_alloc);
156156

157157
/* Heuristic checking if a pointer value is newly allocated
158-
(uninitialized) or newly freed. The pointer is not dereferenced, only the
159-
pointer value is checked.
158+
(uninitialized), newly freed or NULL (is equal to zero).
159+
160+
The pointer is not dereferenced, only the pointer value is checked.
160161
161162
The heuristic relies on the debug hooks on Python memory allocators which
162163
fills newly allocated memory with CLEANBYTE (0xCD) and newly freed memory
@@ -166,11 +167,13 @@ static inline int _PyMem_IsPtrFreed(void *ptr)
166167
{
167168
uintptr_t value = (uintptr_t)ptr;
168169
#if SIZEOF_VOID_P == 8
169-
return (value == (uintptr_t)0xCDCDCDCDCDCDCDCD
170+
return (value == 0
171+
|| value == (uintptr_t)0xCDCDCDCDCDCDCDCD
170172
|| value == (uintptr_t)0xDDDDDDDDDDDDDDDD
171173
|| value == (uintptr_t)0xFDFDFDFDFDFDFDFD);
172174
#elif SIZEOF_VOID_P == 4
173-
return (value == (uintptr_t)0xCDCDCDCD
175+
return (value == 0
176+
|| value == (uintptr_t)0xCDCDCDCD
174177
|| value == (uintptr_t)0xDDDDDDDD
175178
|| value == (uintptr_t)0xFDFDFDFD);
176179
#else

‎Include/unicodeobject.h

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1032,16 +1032,6 @@ PyAPI_FUNC(int) PyUnicode_IsIdentifier(PyObject *s);
10321032

10331033
/* === Characters Type APIs =============================================== */
10341034

1035-
#if defined(Py_DEBUG) && !defined(Py_LIMITED_API)
1036-
PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
1037-
PyObject *op,
1038-
int check_content);
1039-
#elif !defined(NDEBUG)
1040-
/* For asserts that call _PyUnicode_CheckConsistency(), which would
1041-
* otherwise be a problem when building with asserts but without Py_DEBUG. */
1042-
#define _PyUnicode_CheckConsistency(op, check_content) PyUnicode_Check(op)
1043-
#endif
1044-
10451035
#ifndef Py_LIMITED_API
10461036
# define Py_CPYTHON_UNICODEOBJECT_H
10471037
# include "cpython/unicodeobject.h"

‎Lib/test/test_capi.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -719,6 +719,9 @@ def check_pyobject_is_freed(self, func_name):
719719
''')
720720
assert_python_ok('-c', code, PYTHONMALLOC=self.PYTHONMALLOC)
721721

722+
def test_pyobject_null_is_freed(self):
723+
self.check_pyobject_is_freed('check_pyobject_null_is_freed')
724+
722725
def test_pyobject_uninitialized_is_freed(self):
723726
self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed')
724727

‎Lib/test/test_gc.py

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -985,16 +985,19 @@ def test_refcount_errors(self):
985985
br'gcmodule\.c:[0-9]+: gc_decref: Assertion "gc_get_refs\(g\) > 0" failed.')
986986
self.assertRegex(stderr,
987987
br'refcount is too small')
988+
# "address : 0x7fb5062efc18"
989+
# "address : 7FB5062EFC18"
990+
address_regex = br'[0-9a-fA-Fx]+'
988991
self.assertRegex(stderr,
989-
br'object : \[1, 2, 3\]')
992+
br'object address : ' + address_regex)
990993
self.assertRegex(stderr,
991-
br'type : list')
994+
br'object refcount : 1')
992995
self.assertRegex(stderr,
993-
br'refcount: 1')
994-
# "address : 0x7fb5062efc18"
995-
# "address : 7FB5062EFC18"
996+
br'object type : ' + address_regex)
997+
self.assertRegex(stderr,
998+
br'object type name: list')
996999
self.assertRegex(stderr,
997-
br'address : [0-9a-fA-Fx]+')
1000+
br'object repr : \[1, 2, 3\]')
9981001

9991002

10001003
class GCTogglingTests(unittest.TestCase):

‎Misc/NEWS.d/next/C API/2019-10-07-17-15-09.bpo-36389.hFX_jD.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
The ``_PyObject_CheckConsistency()`` function is now also available in release
2+
mode. For example, it can be used to debug a crash in the ``visit_decref()``
3+
function of the GC.

‎Modules/_testcapimodule.c

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4611,6 +4611,14 @@ test_pyobject_is_freed(const char *test_name, PyObject *op)
46114611
}
46124612

46134613

4614+
static PyObject*
4615+
check_pyobject_null_is_freed(PyObject *self, PyObject *Py_UNUSED(args))
4616+
{
4617+
PyObject *op = NULL;
4618+
return test_pyobject_is_freed("check_pyobject_null_is_freed", op);
4619+
}
4620+
4621+
46144622
static PyObject*
46154623
check_pyobject_uninitialized_is_freed(PyObject *self, PyObject *Py_UNUSED(args))
46164624
{
@@ -5475,6 +5483,7 @@ static PyMethodDef TestMethods[] = {
54755483
{"pymem_api_misuse", pymem_api_misuse, METH_NOARGS},
54765484
{"pymem_malloc_without_gil", pymem_malloc_without_gil, METH_NOARGS},
54775485
{"pymem_getallocatorsname", test_pymem_getallocatorsname, METH_NOARGS},
5486+
{"check_pyobject_null_is_freed", check_pyobject_null_is_freed, METH_NOARGS},
54785487
{"check_pyobject_uninitialized_is_freed", check_pyobject_uninitialized_is_freed, METH_NOARGS},
54795488
{"check_pyobject_forbidden_bytes_is_freed", check_pyobject_forbidden_bytes_is_freed, METH_NOARGS},
54805489
{"check_pyobject_freed_is_freed", check_pyobject_freed_is_freed, METH_NOARGS},

‎Modules/gcmodule.c

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -375,7 +375,6 @@ update_refs(PyGC_Head *containers)
375375
static int
376376
visit_decref(PyObject *op, void *data)
377377
{
378-
assert(op != NULL);
379378
_PyObject_ASSERT(op, !_PyObject_IsFreed(op));
380379

381380
if (PyObject_IS_GC(op)) {
F987

‎Objects/bytesobject.c

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3206,10 +3206,10 @@ _PyBytesWriter_GetSize(_PyBytesWriter *writer, char *str)
32063206
return str - start;
32073207
}
32083208

3209-
Py_LOCAL_INLINE(void)
3209+
#ifndef NDEBUG
3210+
Py_LOCAL_INLINE(int)
32103211
_PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str)
32113212
{
3212-
#ifdef Py_DEBUG
32133213
char *start, *end;
32143214

32153215
if (writer->use_small_buffer) {
@@ -3239,15 +3239,16 @@ _PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str)
32393239
end = start + writer->allocated;
32403240
assert(str != NULL);
32413241
assert(start <= str && str <= end);
3242-
#endif
3242+
return 1;
32433243
}
3244+
#endif
32443245

32453246
void*
32463247
_PyBytesWriter_Resize(_PyBytesWriter *writer, void *str, Py_ssize_t size)
32473248
{
32483249
Py_ssize_t allocated, pos;
32493250

3250-
_PyBytesWriter_CheckConsistency(writer, str);
3251+
assert(_PyBytesWriter_CheckConsistency(writer, str));
32513252
assert(writer->allocated < size);
32523253

32533254
allocated = size;
@@ -3303,7 +3304,7 @@ _PyBytesWriter_Resize(_PyBytesWriter *writer, void *str, Py_ssize_t size)
33033304
writer->allocated = allocated;
33043305

33053306
str = _PyBytesWriter_AsString(writer) + pos;
3306-
_PyBytesWriter_CheckConsistency(writer, str);
3307+
assert(_PyBytesWriter_CheckConsistency(writer, str));
33073308
return str;
33083309

33093310
error:
@@ -3316,7 +3317,7 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size)
33163317
{
33173318
Py_ssize_t new_min_size;
33183319

3319-
_PyBytesWriter_CheckConsistency(writer, str);
3320+
assert(_PyBytesWriter_CheckConsistency(writer, str));
33203321
assert(size >= 0);
33213322

33223323
if (size == 0) {
@@ -3377,7 +3378,7 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, void *str)
33773378
Py_ssize_t size;
33783379
PyObject *result;
33793380

3380-
_PyBytesWriter_CheckConsistency(writer, str);
3381+
assert(_PyBytesWriter_CheckConsistency(writer, str));
33813382

33823383
size = _PyBytesWriter_GetSize(writer, str);
33833384
if (size == 0 && !writer->use_bytearray) {

‎Objects/dictobject.c

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -459,23 +459,26 @@ static PyObject *empty_values[1] = { NULL };
459459
int
460460
_PyDict_CheckConsistency(PyObject *op, int check_content)
461461
{
462-
#ifndef NDEBUG
463-
_PyObject_ASSERT(op, PyDict_Check(op));
462+
#define CHECK(expr) \
463+
do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG(op, Py_STRINGIFY(expr)); } } while (0)
464+
465+
assert(op != NULL);
466+
CHECK(PyDict_Check(op));
464467
PyDictObject *mp = (PyDictObject *)op;
465468

466469
PyDictKeysObject *keys = mp->ma_keys;
467470
int splitted = _PyDict_HasSplitTable(mp);
468471
Py_ssize_t usable = USABLE_FRACTION(keys->dk_size);
469472

470-
_PyObject_ASSERT(op, 0 <= mp->ma_used && mp->ma_used <= usable);
471-
_PyObject_ASSERT(op, IS_POWER_OF_2(keys->dk_size));
472-
_PyObject_ASSERT(op, 0 <= keys->dk_usable && keys->dk_usable <= usable);
473-
_PyObject_ASSERT(op, 0 <= keys->dk_nentries && keys->dk_nentries <= usable);
474-
_PyObject_ASSERT(op, keys->dk_usable + keys->dk_nentries <= usable);
473+
CHECK(0 <= mp->ma_used && mp->ma_used <= usable);
474+
CHECK(IS_POWER_OF_2(keys->dk_size));
475+
CHECK(0 <= keys->dk_usable && keys->dk_usable <= usable);
476+
CHECK(0 <= keys->dk_nentries && keys->dk_nentries <= usable);
477+
CHECK(keys->dk_usable + keys->dk_nentries <= usable);
475478

476479
if (!splitted) {
477480
/* combined table */
478-
_PyObject_ASSERT(op, keys->dk_refcnt == 1);
481+
CHECK(keys->dk_refcnt == 1);
479482
}
480483

481484
if (check_content) {
@@ -484,7 +487,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content)
484487

485488
for (i=0; i < keys->dk_size; i++) {
486489
Py_ssize_t ix = dictkeys_get_index(keys, i);
487-
_PyObject_ASSERT(op, DKIX_DUMMY <= ix && ix <= usable);
490+
CHECK(DKIX_DUMMY <= ix && ix <= usable);
488491
}
489492

490493
for (i=0; i < usable; i++) {
@@ -494,32 +497,33 @@ _PyDict_CheckConsistency(PyObject *op, int check_content)
494497
if (key != NULL) {
495498
if (PyUnicode_CheckExact(key)) {
496499
Py_hash_t hash = ((PyASCIIObject *)key)->hash;
497-
_PyObject_ASSERT(op, hash != -1);
498-
_PyObject_ASSERT(op, entry->me_hash == hash);
500+
CHECK(hash != -1);
501+
CHECK(entry->me_hash == hash);
499502
}
500503
else {
501504
/* test_dict fails if PyObject_Hash() is called again */
502-
_PyObject_ASSERT(op, entry->me_hash != -1);
505+
CHECK(entry->me_hash != -1);
503506
}
504507
if (!splitted) {
505-
_PyObject_ASSERT(op, entry->me_value != NULL);
508+
CHECK(entry->me_value != NULL);
506509
}
507510
}
508511

509512
if (splitted) {
510-
_PyObject_ASSERT(op, entry->me_value == NULL);
513+
CHECK(entry->me_value == NULL);
511514
}
512515
}
513516

514517
if (splitted) {
515518
/* splitted table */
516519
for (i=0; i < mp->ma_used; i++) {
517-
_PyObject_ASSERT(op, mp->ma_values[i] != NULL);
520+
CHECK(mp->ma_values[i] != NULL);
518521
}
519522
}
520523
}
521-
#endif
522524
return 1;
525+
526+
#undef CHECK
523527
}
524528

525529

0 commit comments

Comments
 (0)
0