From cc5fef4e2f103dbf273d6b93647aa29212883f1d Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Tue, 2 Aug 2022 17:29:35 +0100
Subject: [PATCH 1/7] Make sure that tp_dictoffset is correct with
 Py_TPFLAGS_MANAGED_DICT is set.

---
 Lib/test/test_capi.py        | 15 ++++++++++++
 Modules/_testcapi/heaptype.c | 45 ++++++++++++++++++++++++++++++++++++
 Objects/typeobject.c         | 40 +++++++++++++++++++++++++++++++-
 3 files changed, 99 insertions(+), 1 deletion(-)

diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index a88a17d3c55788..18716e7aaa1f38 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -538,6 +538,21 @@ def test_heaptype_with_dict(self):
         inst = _testcapi.HeapCTypeWithDict()
         self.assertEqual({}, inst.__dict__)
 
+    def test_heaptype_with_managed_dict(self):
+        inst = _testcapi.HeapCTypeWithManagedDict()
+        inst.foo = 42
+        self.assertEqual(inst.foo, 42)
+        self.assertEqual(inst.__dict__, {"foo": 42})
+
+        inst = _testcapi.HeapCTypeWithManagedDict()
+        self.assertEqual({}, inst.__dict__)
+
+        a = _testcapi.HeapCTypeWithManagedDict()
+        b = _testcapi.HeapCTypeWithManagedDict()
+        a.b = b
+        b.a = a
+        del a, b
+
     def test_heaptype_with_negative_dict(self):
         inst = _testcapi.HeapCTypeWithNegativeDict()
         inst.foo = 42
diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c
index 9fb0051ca125e4..08797787efbdf7 100644
--- a/Modules/_testcapi/heaptype.c
+++ b/Modules/_testcapi/heaptype.c
@@ -737,6 +737,45 @@ static PyType_Spec HeapCTypeWithDict_spec = {
     HeapCTypeWithDict_slots
 };
 
+static int
+heapmanaged_traverse(HeapCTypeObject *self, visitproc visit, void *arg)
+{
+    Py_VISIT(Py_TYPE(self));
+    return _PyObject_VisitManagedDict((PyObject *)self, visit, arg);
+}
+
+static void
+heapmanaged_clear(HeapCTypeObject *self)
+{
+    _PyObject_ClearManagedDict((PyObject *)self);
+}
+
+static void
+heapmanaged_dealloc(HeapCTypeObject *self)
+{
+    PyTypeObject *tp = Py_TYPE(self);
+    _PyObject_ClearManagedDict((PyObject *)self);
+    PyObject_GC_UnTrack(self);
+    PyObject_GC_Del(self);
+    Py_DECREF(tp);
+}
+
+static PyType_Slot HeapCTypeWithManagedDict_slots[] = {
+    {Py_tp_traverse, heapmanaged_traverse},
+    {Py_tp_getset, heapctypewithdict_getsetlist},
+    {Py_tp_clear, heapmanaged_clear},
+    {Py_tp_dealloc, heapmanaged_dealloc},
+    {0, 0},
+};
+
+static PyType_Spec  HeapCTypeWithManagedDict_spec = {
+    "_testcapi.HeapCTypeWithManagedDict",
+    sizeof(PyObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MANAGED_DICT,
+    HeapCTypeWithManagedDict_slots
+};
+
 static struct PyMemberDef heapctypewithnegativedict_members[] = {
     {"dictobj", T_OBJECT, offsetof(HeapCTypeWithDictObject, dict)},
     {"__dictoffset__", T_PYSSIZET, -(Py_ssize_t)sizeof(void*), READONLY},
@@ -925,6 +964,12 @@ _PyTestCapi_Init_Heaptype(PyObject *m) {
     }
     PyModule_AddObject(m, "HeapCTypeWithNegativeDict", HeapCTypeWithNegativeDict);
 
+    PyObject *HeapCTypeWithManagedDict = PyType_FromSpec(&HeapCTypeWithManagedDict_spec);
+    if (HeapCTypeWithManagedDict == NULL) {
+        return -1;
+    }
+    PyModule_AddObject(m, "HeapCTypeWithManagedDict", HeapCTypeWithManagedDict);
+
     PyObject *HeapCTypeWithWeakref = PyType_FromSpec(&HeapCTypeWithWeakref_spec);
     if (HeapCTypeWithWeakref == NULL) {
         return -1;
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index d33befc05d7ab8..491d121bf09b22 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -3078,7 +3078,7 @@ type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type)
     if (ctx->add_dict && ctx->base->tp_itemsize == 0) {
         assert((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
         type->tp_flags |= Py_TPFLAGS_MANAGED_DICT;
-        type->tp_dictoffset = -slotoffset - sizeof(PyObject *)*3;
+        type->tp_dictoffset = -type->tp_basicsize - (Py_ssize_t)sizeof(PyObject *)*3;
     }
 
     type->tp_basicsize = slotoffset;
@@ -6078,6 +6078,7 @@ inherit_special(PyTypeObject *type, PyTypeObject *base)
     COPYVAL(tp_itemsize);
     COPYVAL(tp_weaklistoffset);
     COPYVAL(tp_dictoffset);
+
 #undef COPYVAL
 
     /* Setup fast subclass flags */
@@ -6486,6 +6487,22 @@ type_ready_fill_dict(PyTypeObject *type)
     return 0;
 }
 
+static int
+type_ready_dict_offset(PyTypeObject *type)
+{
+    if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+        if (type->tp_dictoffset > 0) {
+            PyErr_Format(PyExc_TypeError,
+                        "type %s has the Py_TPFLAGS_MANAGED_DICT flag "
+                        "but tp_dictoffset is positive",
+                        type->tp_name);
+            return -1;
+        }
+        Py_ssize_t offset = -type->tp_basicsize - sizeof(PyObject *)*3;
+        type->tp_dictoffset = offset;
+    }
+    return 0;
+}
 
 static int
 type_ready_mro(PyTypeObject *type)
@@ -6694,6 +6711,24 @@ type_ready_post_checks(PyTypeObject *type)
                      type->tp_name);
         return -1;
     }
+    if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+        if (type->tp_itemsize != 0) {
+            PyErr_Format(PyExc_SystemError,
+                        "type %s has the Py_TPFLAGS_MANAGED_DICT flag "
+                        "but is variable sized",
+                        type->tp_name);
+            return -1;
+        }
+        Py_ssize_t size = type->tp_basicsize;
+        if (type->tp_dictoffset != -size - (Py_ssize_t)sizeof(PyObject *)*3) {
+            PyErr_Format(PyExc_SystemError,
+                        "type %s has the Py_TPFLAGS_MANAGED_DICT flag "
+                        "but tp_dictoffset is set to incompatible value",
+                        type->tp_name);
+            assert(0);
+            return -1;
+        }
+    }
     return 0;
 }
 
@@ -6733,6 +6768,9 @@ type_ready(PyTypeObject *type)
     if (type_ready_inherit(type) < 0) {
         return -1;
     }
+    if (type_ready_dict_offset(type) < 0) {
+        return -1;
+    }
     if (type_ready_set_hash(type) < 0) {
         return -1;
     }

From 1c1e488601925aa2d64490dfc97de60e6db1a37f Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Tue, 2 Aug 2022 18:10:07 +0100
Subject: [PATCH 2/7] Revert mistaken change.

---
 Objects/typeobject.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 491d121bf09b22..71339a1d11bbe3 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -3078,7 +3078,7 @@ type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type)
     if (ctx->add_dict && ctx->base->tp_itemsize == 0) {
         assert((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
         type->tp_flags |= Py_TPFLAGS_MANAGED_DICT;
-        type->tp_dictoffset = -type->tp_basicsize - (Py_ssize_t)sizeof(PyObject *)*3;
+        type->tp_dictoffset = -slotoffset - (Py_ssize_t)sizeof(PyObject *)*3;
     }
 
     type->tp_basicsize = slotoffset;

From 8bf2f63a5a69ecc976a4021aca9624ba8a0cb76d Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Wed, 3 Aug 2022 12:05:08 +0100
Subject: [PATCH 3/7] Avoid traversing managed dict twice when subclassing
 class with Py_TPFLAGS_MANAGED_DICT set.

---
 Lib/test/test_capi.py |  9 +++++++++
 Objects/typeobject.c  | 12 ++++++++----
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index 18716e7aaa1f38..cc7d68e8a61d1e 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -553,6 +553,15 @@ def test_heaptype_with_managed_dict(self):
         b.a = a
         del a, b
 
+    def test_sublclassing_managed_dict(self):
+
+        class C(_testcapi.HeapCTypeWithManagedDict):
+            pass
+
+        i = C()
+        i.spam = i
+        del i
+
     def test_heaptype_with_negative_dict(self):
         inst = _testcapi.HeapCTypeWithNegativeDict()
         inst.foo = 42
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 71339a1d11bbe3..70aa0052dcd89a 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1310,9 +1310,11 @@ subtype_traverse(PyObject *self, visitproc visit, void *arg)
     }
 
     if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
-        int err = _PyObject_VisitManagedDict(self, visit, arg);
-        if (err) {
-            return err;
+        if ((base->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
+            int err = _PyObject_VisitManagedDict(self, visit, arg);
+            if (err) {
+                return err;
+            }
         }
     }
     else if (type->tp_dictoffset != base->tp_dictoffset) {
@@ -1377,7 +1379,9 @@ subtype_clear(PyObject *self)
     /* Clear the instance dict (if any), to break cycles involving only
        __dict__ slots (as in the case 'self.__dict__ is self'). */
     if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
-        _PyObject_ClearManagedDict(self);
+        if ((base->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
+            _PyObject_ClearManagedDict(self);
+        }
     }
     else if (type->tp_dictoffset != base->tp_dictoffset) {
         PyObject **dictptr = _PyObject_ComputedDictPointer(self);

From 305dc9e60cc620b76bfebf2ac939725a3a760a2c Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Wed, 3 Aug 2022 13:02:05 +0100
Subject: [PATCH 4/7] Add news.

---
 .../next/C API/2022-08-03-13-01-57.gh-issue-92678.DLwONN.rst    | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 Misc/NEWS.d/next/C API/2022-08-03-13-01-57.gh-issue-92678.DLwONN.rst

diff --git a/Misc/NEWS.d/next/C API/2022-08-03-13-01-57.gh-issue-92678.DLwONN.rst b/Misc/NEWS.d/next/C API/2022-08-03-13-01-57.gh-issue-92678.DLwONN.rst
new file mode 100644
index 00000000000000..a3a209f5e41338
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2022-08-03-13-01-57.gh-issue-92678.DLwONN.rst	
@@ -0,0 +1,2 @@
+Support C extensions using managed dictionaries by setting the
+``Py_TPFLAGS_MANAGED_DICT`` flag.

From 00cf8449e306118ac08e5a1b0d564615d8cc9d07 Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Wed, 10 Aug 2022 15:31:09 +0100
Subject: [PATCH 5/7] Reject explicitly negative tp_dictoffsets, and set
 tp_dictoffset to -1 if using managed dicts. Allows use of managed dict on
 subclasses of variable sized objects like tuple.

---
 Lib/test/test_capi.py        | 10 -------
 Lib/test/test_sys.py         |  2 +-
 Modules/_testcapi/heaptype.c | 27 -----------------
 Modules/_testcapimodule.c    |  1 -
 Objects/object.c             |  1 +
 Objects/typeobject.c         | 56 ++++++++++++++++++------------------
 Tools/gdb/libpython.py       | 10 ++-----
 7 files changed, 32 insertions(+), 75 deletions(-)

diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index e4d20355d4303d..4596d4d94fe1a4 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -563,16 +563,6 @@ class C(_testcapi.HeapCTypeWithManagedDict):
         i.spam = i
         del i
 
-    def test_heaptype_with_negative_dict(self):
-        inst = _testcapi.HeapCTypeWithNegativeDict()
-        inst.foo = 42
-        self.assertEqual(inst.foo, 42)
-        self.assertEqual(inst.dictobj, inst.__dict__)
-        self.assertEqual(inst.dictobj, {"foo": 42})
-
-        inst = _testcapi.HeapCTypeWithNegativeDict()
-        self.assertEqual({}, inst.__dict__)
-
     def test_heaptype_with_weakref(self):
         inst = _testcapi.HeapCTypeWithWeakref()
         ref = weakref.ref(inst)
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
index 05fbcc19b6b7d7..78da09d6abc0f9 100644
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -1287,7 +1287,7 @@ class OverflowSizeof(int):
             def __sizeof__(self):
                 return int(self)
         self.assertEqual(sys.getsizeof(OverflowSizeof(sys.maxsize)),
-                         sys.maxsize + self.gc_headsize)
+                         sys.maxsize + self.gc_headsize*2)
         with self.assertRaises(OverflowError):
             sys.getsizeof(OverflowSizeof(sys.maxsize + 1))
         with self.assertRaises(ValueError):
diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c
index de990bac51d6b6..b6e47504f53cbf 100644
--- a/Modules/_testcapi/heaptype.c
+++ b/Modules/_testcapi/heaptype.c
@@ -800,27 +800,6 @@ static PyType_Spec  HeapCTypeWithManagedDict_spec = {
     HeapCTypeWithManagedDict_slots
 };
 
-static struct PyMemberDef heapctypewithnegativedict_members[] = {
-    {"dictobj", T_OBJECT, offsetof(HeapCTypeWithDictObject, dict)},
-    {"__dictoffset__", T_PYSSIZET, -(Py_ssize_t)sizeof(void*), READONLY},
-    {NULL} /* Sentinel */
-};
-
-static PyType_Slot HeapCTypeWithNegativeDict_slots[] = {
-    {Py_tp_members, heapctypewithnegativedict_members},
-    {Py_tp_getset, heapctypewithdict_getsetlist},
-    {Py_tp_dealloc, heapctypewithdict_dealloc},
-    {0, 0},
-};
-
-static PyType_Spec HeapCTypeWithNegativeDict_spec = {
-    "_testcapi.HeapCTypeWithNegativeDict",
-    sizeof(HeapCTypeWithDictObject),
-    0,
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
-    HeapCTypeWithNegativeDict_slots
-};
-
 typedef struct {
     PyObject_HEAD
     PyObject *weakreflist;
@@ -996,12 +975,6 @@ _PyTestCapi_Init_Heaptype(PyObject *m) {
     }
     PyModule_AddObject(m, "HeapCTypeWithDict2", HeapCTypeWithDict2);
 
-    PyObject *HeapCTypeWithNegativeDict = PyType_FromSpec(&HeapCTypeWithNegativeDict_spec);
-    if (HeapCTypeWithNegativeDict == NULL) {
-        return -1;
-    }
-    PyModule_AddObject(m, "HeapCTypeWithNegativeDict", HeapCTypeWithNegativeDict);
-
     PyObject *HeapCTypeWithManagedDict = PyType_FromSpec(&HeapCTypeWithManagedDict_spec);
     if (HeapCTypeWithManagedDict == NULL) {
         return -1;
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index 91bdeb8b6464df..fea8ae34849e11 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -5421,7 +5421,6 @@ clear_managed_dict(PyObject *self, PyObject *obj)
     Py_RETURN_NONE;
 }
 
-
 static PyObject *
 test_macros(PyObject *self, PyObject *Py_UNUSED(args))
 {
diff --git a/Objects/object.c b/Objects/object.c
index a90c6faf99db48..f0e4ec3100ff91 100644
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -1061,6 +1061,7 @@ _PyObject_ComputedDictPointer(PyObject *obj)
 
     assert((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
     dictoffset = tp->tp_dictoffset;
+    assert(dictoffset >= 0);
     if (dictoffset == 0)
         return NULL;
     if (dictoffset < 0) {
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index ebbacf7c325dbc..dfcbe61939bfde 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1312,18 +1312,21 @@ subtype_traverse(PyObject *self, visitproc visit, void *arg)
         assert(base);
     }
 
-    if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
-        if ((base->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
+    if (type->tp_dictoffset != base->tp_dictoffset) {
+        assert(base->tp_dictoffset == 0);
+        if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+            assert(type->tp_dictoffset == -1);
             int err = _PyObject_VisitManagedDict(self, visit, arg);
             if (err) {
                 return err;
             }
         }
-    }
-    else if (type->tp_dictoffset != base->tp_dictoffset) {
-        PyObject **dictptr = _PyObject_ComputedDictPointer(self);
-        if (dictptr && *dictptr)
-            Py_VISIT(*dictptr);
+        else {
+            PyObject **dictptr = _PyObject_ComputedDictPointer(self);
+            if (dictptr && *dictptr) {
+                Py_VISIT(*dictptr);
+            }
+        }
     }
 
     if (type->tp_flags & Py_TPFLAGS_HEAPTYPE
@@ -3089,20 +3092,15 @@ type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type)
         }
     }
 
-    if (ctx->add_dict && ctx->base->tp_itemsize) {
-        type->tp_dictoffset = -(long)sizeof(PyObject *);
-        slotoffset += sizeof(PyObject *);
-    }
-
     if (ctx->add_weak) {
         assert(!ctx->base->tp_itemsize);
         type->tp_weaklistoffset = slotoffset;
         slotoffset += sizeof(PyObject *);
     }
-    if (ctx->add_dict && ctx->base->tp_itemsize == 0) {
+    if (ctx->add_dict) {
         assert((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
         type->tp_flags |= Py_TPFLAGS_MANAGED_DICT;
-        type->tp_dictoffset = -slotoffset - (Py_ssize_t)sizeof(PyObject *)*3;
+        type->tp_dictoffset = -1;
     }
 
     type->tp_basicsize = slotoffset;
@@ -6576,15 +6574,20 @@ static int
 type_ready_dict_offset(PyTypeObject *type)
 {
     if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
-        if (type->tp_dictoffset > 0) {
+        if (type->tp_dictoffset > 0 || type->tp_dictoffset < -1) {
             PyErr_Format(PyExc_TypeError,
                         "type %s has the Py_TPFLAGS_MANAGED_DICT flag "
-                        "but tp_dictoffset is positive",
+                        "but tp_dictoffset is set",
                         type->tp_name);
             return -1;
         }
-        Py_ssize_t offset = -type->tp_basicsize - sizeof(PyObject *)*3;
-        type->tp_dictoffset = offset;
+        type->tp_dictoffset = -1;
+    }
+    else if (type->tp_dictoffset < 0) {
+        PyErr_Format(PyExc_TypeError,
+                     "type %s has negative tp_dictoffset",
+                     type->tp_name);
+        return -1;
     }
     return 0;
 }
@@ -6797,23 +6800,20 @@ type_ready_post_checks(PyTypeObject *type)
         return -1;
     }
     if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
-        if (type->tp_itemsize != 0) {
-            PyErr_Format(PyExc_SystemError,
-                        "type %s has the Py_TPFLAGS_MANAGED_DICT flag "
-                        "but is variable sized",
-                        type->tp_name);
-            return -1;
-        }
-        Py_ssize_t size = type->tp_basicsize;
-        if (type->tp_dictoffset != -size - (Py_ssize_t)sizeof(PyObject *)*3) {
+        if (type->tp_dictoffset != -1) {
             PyErr_Format(PyExc_SystemError,
                         "type %s has the Py_TPFLAGS_MANAGED_DICT flag "
                         "but tp_dictoffset is set to incompatible value",
                         type->tp_name);
-            assert(0);
             return -1;
         }
     }
+    else if (type->tp_dictoffset < 0) {
+        PyErr_Format(PyExc_SystemError,
+                     "type %s has negative tp_dictoffset",
+                     type->tp_name);
+        return -1;
+    }
     return 0;
 }
 
diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py
index d03c6379125832..807d84da128225 100755
--- a/Tools/gdb/libpython.py
+++ b/Tools/gdb/libpython.py
@@ -478,14 +478,8 @@ def get_attr_dict(self):
             dictoffset = int_from_int(typeobj.field('tp_dictoffset'))
             if dictoffset != 0:
                 if dictoffset < 0:
-                    type_PyVarObject_ptr = gdb.lookup_type('PyVarObject').pointer()
-                    tsize = int_from_int(self._gdbval.cast(type_PyVarObject_ptr)['ob_size'])
-                    if tsize < 0:
-                        tsize = -tsize
-                    size = _PyObject_VAR_SIZE(typeobj, tsize)
-                    dictoffset += size
-                    assert dictoffset % _sizeof_void_p() == 0
-
+                    assert dictoffset == -1
+                    dictoffset = -3 * _sizeof_void_p()
                 dictptr = self._gdbval.cast(_type_char_ptr()) + dictoffset
                 PyObjectPtrPtr = PyObjectPtr.get_gdb_type().pointer()
                 dictptr = dictptr.cast(PyObjectPtrPtr)

From 595c34a2a18e69f9dfed450fd4398757f060ae81 Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Wed, 10 Aug 2022 16:03:11 +0100
Subject: [PATCH 6/7] Allow negative tp_dictoffset again. It seems harmless.

---
 Lib/test/test_capi.py        | 10 ++++++++++
 Modules/_testcapi/heaptype.c | 27 +++++++++++++++++++++++++++
 Modules/_testcapimodule.c    |  1 +
 Objects/object.c             |  2 +-
 Objects/typeobject.c         | 14 ++++----------
 Tools/gdb/libpython.py       | 14 ++++++++++++--
 6 files changed, 55 insertions(+), 13 deletions(-)

diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index 4596d4d94fe1a4..e4d20355d4303d 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -563,6 +563,16 @@ class C(_testcapi.HeapCTypeWithManagedDict):
         i.spam = i
         del i
 
+    def test_heaptype_with_negative_dict(self):
+        inst = _testcapi.HeapCTypeWithNegativeDict()
+        inst.foo = 42
+        self.assertEqual(inst.foo, 42)
+        self.assertEqual(inst.dictobj, inst.__dict__)
+        self.assertEqual(inst.dictobj, {"foo": 42})
+
+        inst = _testcapi.HeapCTypeWithNegativeDict()
+        self.assertEqual({}, inst.__dict__)
+
     def test_heaptype_with_weakref(self):
         inst = _testcapi.HeapCTypeWithWeakref()
         ref = weakref.ref(inst)
diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c
index b6e47504f53cbf..de990bac51d6b6 100644
--- a/Modules/_testcapi/heaptype.c
+++ b/Modules/_testcapi/heaptype.c
@@ -800,6 +800,27 @@ static PyType_Spec  HeapCTypeWithManagedDict_spec = {
     HeapCTypeWithManagedDict_slots
 };
 
+static struct PyMemberDef heapctypewithnegativedict_members[] = {
+    {"dictobj", T_OBJECT, offsetof(HeapCTypeWithDictObject, dict)},
+    {"__dictoffset__", T_PYSSIZET, -(Py_ssize_t)sizeof(void*), READONLY},
+    {NULL} /* Sentinel */
+};
+
+static PyType_Slot HeapCTypeWithNegativeDict_slots[] = {
+    {Py_tp_members, heapctypewithnegativedict_members},
+    {Py_tp_getset, heapctypewithdict_getsetlist},
+    {Py_tp_dealloc, heapctypewithdict_dealloc},
+    {0, 0},
+};
+
+static PyType_Spec HeapCTypeWithNegativeDict_spec = {
+    "_testcapi.HeapCTypeWithNegativeDict",
+    sizeof(HeapCTypeWithDictObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    HeapCTypeWithNegativeDict_slots
+};
+
 typedef struct {
     PyObject_HEAD
     PyObject *weakreflist;
@@ -975,6 +996,12 @@ _PyTestCapi_Init_Heaptype(PyObject *m) {
     }
     PyModule_AddObject(m, "HeapCTypeWithDict2", HeapCTypeWithDict2);
 
+    PyObject *HeapCTypeWithNegativeDict = PyType_FromSpec(&HeapCTypeWithNegativeDict_spec);
+    if (HeapCTypeWithNegativeDict == NULL) {
+        return -1;
+    }
+    PyModule_AddObject(m, "HeapCTypeWithNegativeDict", HeapCTypeWithNegativeDict);
+
     PyObject *HeapCTypeWithManagedDict = PyType_FromSpec(&HeapCTypeWithManagedDict_spec);
     if (HeapCTypeWithManagedDict == NULL) {
         return -1;
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index fea8ae34849e11..91bdeb8b6464df 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -5421,6 +5421,7 @@ clear_managed_dict(PyObject *self, PyObject *obj)
     Py_RETURN_NONE;
 }
 
+
 static PyObject *
 test_macros(PyObject *self, PyObject *Py_UNUSED(args))
 {
diff --git a/Objects/object.c b/Objects/object.c
index f0e4ec3100ff91..9bbe0eef308fba 100644
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -1061,10 +1061,10 @@ _PyObject_ComputedDictPointer(PyObject *obj)
 
     assert((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
     dictoffset = tp->tp_dictoffset;
-    assert(dictoffset >= 0);
     if (dictoffset == 0)
         return NULL;
     if (dictoffset < 0) {
+        assert(dictoffset != -1);
         Py_ssize_t tsize = Py_SIZE(obj);
         if (tsize < 0) {
             tsize = -tsize;
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index dfcbe61939bfde..aa54b84aad19e1 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -6583,12 +6583,6 @@ type_ready_dict_offset(PyTypeObject *type)
         }
         type->tp_dictoffset = -1;
     }
-    else if (type->tp_dictoffset < 0) {
-        PyErr_Format(PyExc_TypeError,
-                     "type %s has negative tp_dictoffset",
-                     type->tp_name);
-        return -1;
-    }
     return 0;
 }
 
@@ -6809,10 +6803,10 @@ type_ready_post_checks(PyTypeObject *type)
         }
     }
     else if (type->tp_dictoffset < 0) {
-        PyErr_Format(PyExc_SystemError,
-                     "type %s has negative tp_dictoffset",
-                     type->tp_name);
-        return -1;
+        if (type->tp_dictoffset + type->tp_basicsize <= 0) {
+            PyErr_Format(PyExc_SystemError,
+                         "type %s has a tp_dictoffset that is too small");
+        }
     }
     return 0;
 }
diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py
index 807d84da128225..899cb6c7dea0b6 100755
--- a/Tools/gdb/libpython.py
+++ b/Tools/gdb/libpython.py
@@ -478,8 +478,18 @@ def get_attr_dict(self):
             dictoffset = int_from_int(typeobj.field('tp_dictoffset'))
             if dictoffset != 0:
                 if dictoffset < 0:
-                    assert dictoffset == -1
-                    dictoffset = -3 * _sizeof_void_p()
+                    if int_from_int(typeobj.field('tp_flags')) & Py_TPFLAGS_MANAGED_DICT:
+                        assert dictoffset == -1
+                        dictoffset = -3 * _sizeof_void_p()
+                    else:
+                        type_PyVarObject_ptr = gdb.lookup_type('PyVarObject').pointer()
+                        tsize = int_from_int(self._gdbval.cast(type_PyVarObject_ptr)['ob_size'])
+                        if tsize < 0:
+                            tsize = -tsize
+                        size = _PyObject_VAR_SIZE(typeobj, tsize)
+                        dictoffset += size
+                        assert dictoffset % _sizeof_void_p() == 0
+
                 dictptr = self._gdbval.cast(_type_char_ptr()) + dictoffset
                 PyObjectPtrPtr = PyObjectPtr.get_gdb_type().pointer()
                 dictptr = dictptr.cast(PyObjectPtrPtr)

From 319a8ce17740e1376fde801ce504a1ad8f1854a7 Mon Sep 17 00:00:00 2001
From: Mark Shannon <mark@hotpy.org>
Date: Mon, 15 Aug 2022 11:18:22 +0100
Subject: [PATCH 7/7] Make test for tp_dictoffset stricter.

---
 Objects/typeobject.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index aa54b84aad19e1..c7c50ab387593e 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -6802,7 +6802,7 @@ type_ready_post_checks(PyTypeObject *type)
             return -1;
         }
     }
-    else if (type->tp_dictoffset < 0) {
+    else if (type->tp_dictoffset < sizeof(PyObject)) {
         if (type->tp_dictoffset + type->tp_basicsize <= 0) {
             PyErr_Format(PyExc_SystemError,
                          "type %s has a tp_dictoffset that is too small");