From ddf839e96a65fd1afae73dd91812c85e2ddad44e Mon Sep 17 00:00:00 2001
From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com>
Date: Fri, 28 Jun 2024 23:59:13 +0800
Subject: [PATCH 1/9] A StackRef Debugging Mode

---
 .github/workflows/build.yml          |  13 ++++
 .github/workflows/reusable-macos.yml |   9 ++-
 Include/internal/pycore_interp.h     |   4 +
 Include/internal/pycore_stackref.h   |  88 ++++++++++++++++++++-
 Makefile.pre.in                      |   1 +
 Python/bytecodes.c                   |  16 ++--
 Python/ceval.c                       |  12 ++-
 Python/ceval_macros.h                |  15 +++-
 Python/executor_cases.c.h            |  32 ++++----
 Python/generated_cases.c.h           |  16 ++--
 Python/pylifecycle.c                 |  28 +++++++
 Python/pystate.c                     |   4 +
 Python/stackref.c                    | 112 +++++++++++++++++++++++++++
 configure                            |  28 +++++++
 configure.ac                         |  16 ++++
 pyconfig.h.in                        |   4 +
 16 files changed, 359 insertions(+), 39 deletions(-)
 create mode 100644 Python/stackref.c

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 750aa1ed87bca1..8f3e76ac04d35a 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -215,6 +215,19 @@ jobs:
       # Cirrus used for upstream, macos-14 for forks.
       os-matrix: '["ghcr.io/cirruslabs/macos-runner:sonoma", "macos-14"]'
 
+  build_macos_free_threading_with_stackref_debug:
+    name: 'macOS (free-threading) (StackRef Debug)'
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true'
+    uses: ./.github/workflows/reusable-macos.yml
+    with:
+      config_hash: ${{ needs.check_source.outputs.config_hash }}
+      free-threading: true
+      stackref-debug: true
+      # Cirrus and macos-14 are M1.
+      # Cirrus used for upstream, macos-14 for forks.
+      os-matrix: '["ghcr.io/cirruslabs/macos-runner:sonoma", "macos-14"]'
+
   build_ubuntu:
     name: 'Ubuntu'
     needs: check_source
diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml
index f825d1a7b3f69a..632a7eb69d3c9b 100644
--- a/.github/workflows/reusable-macos.yml
+++ b/.github/workflows/reusable-macos.yml
@@ -8,6 +8,10 @@ on:
         required: false
         type: boolean
         default: false
+      stackref-debug:
+        required: false
+        type: boolean
+        default: false
       os-matrix:
         required: false
         type: string
@@ -54,6 +58,7 @@ jobs:
           --config-cache \
           --with-pydebug \
           ${{ inputs.free-threading && '--disable-gil' || '' }} \
+          ${{ inputs.stackref-debug && '--with-pystackrefdebug' || '' }} \
           --prefix=/opt/python-dev \
           --with-openssl="$(brew --prefix openssl@3.0)"
     - name: Build CPython
@@ -61,4 +66,6 @@ jobs:
     - name: Display build info
       run: make pythoninfo
     - name: Tests
-      run: make test
+      # Stackref debug is 2.5x slower than normal CPython,
+      # so we only run it on a restricted subset of tests.
+      run: ${{ inputs.stackref-debug && './python -m test test_asyncio' || 'make test' }}
diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
index 4a83862ac13e26..9831a9c7ed4180 100644
--- a/Include/internal/pycore_interp.h
+++ b/Include/internal/pycore_interp.h
@@ -275,6 +275,10 @@ struct _is {
     /* the initial PyInterpreterState.threads.head */
     _PyThreadStateImpl _initial_thread;
     Py_ssize_t _interactive_src_count;
+
+#if defined(Py_STACKREF_DEBUG) && defined(Py_GIL_DISABLED)
+    struct _Py_stackref_state stackref_state;
+#endif
 };
 
 
diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h
index 32e445dd96f9a1..fabefffa090ba9 100644
--- a/Include/internal/pycore_stackref.h
+++ b/Include/internal/pycore_stackref.h
@@ -52,35 +52,73 @@ typedef union {
     uintptr_t bits;
 } _PyStackRef;
 
+struct _Py_stackref_entry {
+    PyObject *obj;
+    int is_live;
+};
+
+struct _Py_stackref_state {
+    PyMutex lock;
+    size_t n_entries;
+    struct _Py_stackref_entry *entries;
+    size_t next_ref;
+};
+
+typedef enum _PyStackRef_OpKind {
+    BORROW,
+    STEAL,
+    NEW,
+} _PyStackRef_OpKind;
+
+PyObject *_Py_stackref_to_object_transition(_PyStackRef stackref, _PyStackRef_OpKind op);
+_PyStackRef _Py_object_to_stackref_transition(PyObject *obj, char tag, _PyStackRef_OpKind op);
+PyAPI_FUNC(int) _PyStackRef_IsLive(_PyStackRef stackref);
+PyAPI_FUNC(void) _PyStackRef_Close(_PyStackRef stackref);
+PyAPI_FUNC(_PyStackRef) _PyStackRef_Dup(_PyStackRef stackref);
 
 #define Py_TAG_DEFERRED (1)
 
 #define Py_TAG_PTR      (0)
 #define Py_TAG_BITS     (1)
 
+#define RESERVED_BITS 1
+
 #ifdef Py_GIL_DISABLED
     static const _PyStackRef PyStackRef_NULL = { .bits = 0 | Py_TAG_DEFERRED};
 #else
     static const _PyStackRef PyStackRef_NULL = { .bits = 0 };
 #endif
 
-#define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits)
-
 
 #ifdef Py_GIL_DISABLED
-#   define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED })
+#   ifdef Py_STACKREF_DEBUG
+    // From pycore_init_stackrefs in pylifecycle.c
+    static const _PyStackRef PyStackRef_True = { .bits = 1 << RESERVED_BITS };
+#   else
+#       define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED })
+#   endif
 #else
 #   define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) })
 #endif
 
 #ifdef Py_GIL_DISABLED
-#   define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED })
+#   ifdef Py_STACKREF_DEBUG
+    // From pycore_init_stackrefs in pylifecycle.c
+    static const _PyStackRef PyStackRef_False = { .bits = 2 << RESERVED_BITS };
+#   else
+#       define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED })
+#   endif
 #else
 #   define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) })
 #endif
 
 #ifdef Py_GIL_DISABLED
+#   ifdef Py_STACKREF_DEBUG
+    // From pycore_init_stackrefs in pylifecycle.c
+    static const _PyStackRef PyStackRef_None = { .bits = 3 << RESERVED_BITS };
+#   else
 #   define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED })
+#endif
 #else
 #   define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) })
 #endif
@@ -88,7 +126,18 @@ typedef union {
 
 static inline int
 PyStackRef_Is(_PyStackRef a, _PyStackRef b) {
+#if defined(Py_STACKREF_DEBUG) && defined(Py_GIL_DISABLED)
+    // Note: stackrefs are unique. So even immortal objects will produce different stackrefs.
+    return _Py_stackref_to_object_transition(a, BORROW) == _Py_stackref_to_object_transition(b, BORROW);
+#else
     return a.bits == b.bits;
+#endif
+}
+
+static inline int
+PyStackRef_IsNull(_PyStackRef stackref)
+{
+    return PyStackRef_Is(stackref, PyStackRef_NULL);
 }
 
 static inline int
@@ -102,8 +151,13 @@ static inline PyObject *
 PyStackRef_AsPyObjectBorrow(_PyStackRef stackref)
 {
 #ifdef Py_GIL_DISABLED
+#   if defined(Py_STACKREF_DEBUG)
+    return _Py_stackref_to_object_transition(stackref, BORROW);
+#   else
     PyObject *cleared = ((PyObject *)((stackref).bits & (~Py_TAG_BITS)));
     return cleared;
+#   endif
+
 #else
     return ((PyObject *)(stackref).bits);
 #endif
@@ -115,10 +169,14 @@ static inline PyObject *
 PyStackRef_AsPyObjectSteal(_PyStackRef stackref)
 {
 #ifdef Py_GIL_DISABLED
+#   ifdef Py_STACKREF_DEBUG
+    return _Py_stackref_to_object_transition(stackref, STEAL);
+#   else
     if (!PyStackRef_IsNull(stackref) && PyStackRef_IsDeferred(stackref)) {
         return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref));
     }
     return PyStackRef_AsPyObjectBorrow(stackref);
+#   endif
 #else
     return PyStackRef_AsPyObjectBorrow(stackref);
 #endif
@@ -146,7 +204,11 @@ _PyStackRef_FromPyObjectSteal(PyObject *obj)
     // Make sure we don't take an already tagged value.
     assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
     int tag = (obj == NULL || _Py_IsImmortal(obj)) ? (Py_TAG_DEFERRED) : Py_TAG_PTR;
+#   ifdef Py_STACKREF_DEBUG
+    return _Py_object_to_stackref_transition(obj, tag, STEAL);
+#   else
     return ((_PyStackRef){.bits = ((uintptr_t)(obj)) | tag});
+#   endif
 #else
     return ((_PyStackRef){.bits = ((uintptr_t)(obj))});
 #endif
@@ -165,10 +227,18 @@ PyStackRef_FromPyObjectNew(PyObject *obj)
     assert(obj != NULL);
     // TODO (gh-117139): Add deferred objects later.
     if (_Py_IsImmortal(obj)) {
+#   ifdef Py_STACKREF_DEBUG
+        return _Py_object_to_stackref_transition(obj, Py_TAG_DEFERRED, NEW);
+#   else
         return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED };
+#   endif
     }
     else {
+#   ifdef Py_STACKREF_DEBUG
+        return _Py_object_to_stackref_transition(Py_NewRef(obj), Py_TAG_PTR, NEW);
+#   else
         return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR };
+#   endif
     }
 #else
     return ((_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) });
@@ -186,7 +256,11 @@ PyStackRef_FromPyObjectImmortal(PyObject *obj)
     assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
     assert(obj != NULL);
     assert(_Py_IsImmortal(obj));
+#   ifdef Py_STACKREF_DEBUG
+    return _Py_object_to_stackref_transition(obj, Py_TAG_DEFERRED, NEW);
+#   else
     return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED };
+#   endif
 #else
     assert(_Py_IsImmortal(obj));
     return ((_PyStackRef){ .bits = (uintptr_t)(obj) });
@@ -216,6 +290,9 @@ PyStackRef_CLOSE(_PyStackRef stackref)
         return;
     }
     Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref));
+#   ifdef Py_STACKREF_DEBUG
+    _PyStackRef_Close(stackref);
+#   endif
 #else
     Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref));
 #endif
@@ -234,6 +311,9 @@ static inline _PyStackRef
 PyStackRef_DUP(_PyStackRef stackref)
 {
 #ifdef Py_GIL_DISABLED
+#   ifdef Py_STACKREF_DEBUG
+    stackref = _PyStackRef_Dup(stackref);
+#   endif
     if (PyStackRef_IsDeferred(stackref)) {
         assert(PyStackRef_IsNull(stackref) ||
             _Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref)));
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 41904a2183ae70..8f2bdac4f87c31 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -473,6 +473,7 @@ PYTHON_OBJS=	\
 		Python/qsbr.o \
 		Python/bootstrap_hash.o \
 		Python/specialize.o \
+		Python/stackref.o \
 		Python/structmember.o \
 		Python/symtable.o \
 		Python/sysmodule.o \
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 8dfce77dfca297..50978a0dc87694 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -230,7 +230,7 @@ dummy_func(
         }
 
         replicate(8) pure inst(LOAD_FAST, (-- value)) {
-            assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+            assert(!PyStackRef_IsNull(GETLOCAL(oparg)));
             value = PyStackRef_DUP(GETLOCAL(oparg));
         }
 
@@ -673,7 +673,7 @@ dummy_func(
                 err = 1;
             }
             else {
-                err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectSteal(v));
+                err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v));
                 Py_DECREF(slice);
             }
             PyStackRef_CLOSE(v);
@@ -789,7 +789,7 @@ dummy_func(
 
         inst(SET_ADD, (set, unused[oparg-1], v -- set, unused[oparg-1])) {
             int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set),
-                                PyStackRef_AsPyObjectSteal(v));
+                                PyStackRef_AsPyObjectBorrow(v));
             DECREF_INPUTS();
             ERROR_IF(err, error);
         }
@@ -813,7 +813,7 @@ dummy_func(
 
         op(_STORE_SUBSCR, (v, container, sub -- )) {
             /* container[sub] = v */
-            int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(v));
+            int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v));
             DECREF_INPUTS();
             ERROR_IF(err, error);
         }
@@ -1235,7 +1235,7 @@ dummy_func(
         inst(POP_EXCEPT, (exc_value -- )) {
             _PyErr_StackItem *exc_info = tstate->exc_info;
             Py_XSETREF(exc_info->exc_value,
-                PyStackRef_AsPyObjectBorrow(exc_value) == Py_None
+                   PyStackRef_Is(exc_value, PyStackRef_None)
                     ? NULL : PyStackRef_AsPyObjectSteal(exc_value));
         }
 
@@ -1330,9 +1330,9 @@ dummy_func(
                 ERROR_IF(true, error);
             }
             if (PyDict_CheckExact(ns))
-                err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v));
+                err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v));
             else
-                err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v));
+                err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v));
             DECREF_INPUTS();
             ERROR_IF(err, error);
         }
@@ -1450,7 +1450,7 @@ dummy_func(
         op(_STORE_ATTR, (v, owner --)) {
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
             int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner),
-                                       name, PyStackRef_AsPyObjectSteal(v));
+                                       name, PyStackRef_AsPyObjectBorrow(v));
             DECREF_INPUTS();
             ERROR_IF(err, error);
         }
diff --git a/Python/ceval.c b/Python/ceval.c
index 2412256ace8c8d..abf6865074fd71 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -726,6 +726,16 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch)
     }
 }
 
+static int
+_PyEval_StackIsAllLive(_PyStackRef *stack_base, _PyStackRef *stack_pointer)
+{
+    while (stack_pointer > stack_base) {
+        assert(_PyStackRef_IsLive(stack_pointer[-1]));
+        stack_pointer--;
+    }
+    return 1;
+}
+
 /* _PyEval_EvalFrameDefault() is a *big* function,
  * so consume 3 units of C stack */
 #define PY_EVAL_C_STACK_UNITS 2
@@ -1619,7 +1629,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func,
                 goto kw_fail;
             }
 
-            if (PyDict_SetItem(kwdict, keyword, PyStackRef_AsPyObjectSteal(value_stackref)) == -1) {
+            if (PyDict_SetItem(kwdict, keyword, PyStackRef_AsPyObjectBorrow(value_stackref)) == -1) {
                 goto kw_fail;
             }
             PyStackRef_CLOSE(value_stackref);
diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h
index f6d055a1dfaa9f..3757e30a6c811b 100644
--- a/Python/ceval_macros.h
+++ b/Python/ceval_macros.h
@@ -86,6 +86,17 @@
 #define PRE_DISPATCH_GOTO() ((void)0)
 #endif
 
+#if defined(Py_GIL_DISABLED) && defined(Py_DEBUG)
+#define STACKREF_CHECK() \
+    do { \
+        if (frame->f_executable && PyCode_Check(frame->f_executable)) { \
+            _PyEval_StackIsAllLive(_PyFrame_Stackbase(frame), stack_pointer); \
+        }; \
+    } while (0);
+#else
+#define STACKREF_CHECK() ((void)(0))
+#endif
+
 #if LLTRACE
 #define LLTRACE_RESUME_FRAME() \
 do { \
@@ -110,13 +121,15 @@ do { \
     { \
         NEXTOPARG(); \
         PRE_DISPATCH_GOTO(); \
+        STACKREF_CHECK(); \
         DISPATCH_GOTO(); \
     }
 
 #define DISPATCH_SAME_OPARG() \
     { \
         opcode = next_instr->op.code; \
-        PRE_DISPATCH_GOTO(); \
+        PRE_DISPATCH_GOTO();  \
+        STACKREF_CHECK(); \
         DISPATCH_GOTO(); \
     }
 
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 38437c6f2c087c..76b7a9b4b15ae9 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -54,7 +54,7 @@
             _PyStackRef value;
             oparg = 0;
             assert(oparg == CURRENT_OPARG());
-            assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+            assert(!PyStackRef_IsNull(GETLOCAL(oparg)));
             value = PyStackRef_DUP(GETLOCAL(oparg));
             stack_pointer[0] = value;
             stack_pointer += 1;
@@ -66,7 +66,7 @@
             _PyStackRef value;
             oparg = 1;
             assert(oparg == CURRENT_OPARG());
-            assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+            assert(!PyStackRef_IsNull(GETLOCAL(oparg)));
             value = PyStackRef_DUP(GETLOCAL(oparg));
             stack_pointer[0] = value;
             stack_pointer += 1;
@@ -78,7 +78,7 @@
             _PyStackRef value;
             oparg = 2;
             assert(oparg == CURRENT_OPARG());
-            assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+            assert(!PyStackRef_IsNull(GETLOCAL(oparg)));
             value = PyStackRef_DUP(GETLOCAL(oparg));
             stack_pointer[0] = value;
             stack_pointer += 1;
@@ -90,7 +90,7 @@
             _PyStackRef value;
             oparg = 3;
             assert(oparg == CURRENT_OPARG());
-            assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+            assert(!PyStackRef_IsNull(GETLOCAL(oparg)));
             value = PyStackRef_DUP(GETLOCAL(oparg));
             stack_pointer[0] = value;
             stack_pointer += 1;
@@ -102,7 +102,7 @@
             _PyStackRef value;
             oparg = 4;
             assert(oparg == CURRENT_OPARG());
-            assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+            assert(!PyStackRef_IsNull(GETLOCAL(oparg)));
             value = PyStackRef_DUP(GETLOCAL(oparg));
             stack_pointer[0] = value;
             stack_pointer += 1;
@@ -114,7 +114,7 @@
             _PyStackRef value;
             oparg = 5;
             assert(oparg == CURRENT_OPARG());
-            assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+            assert(!PyStackRef_IsNull(GETLOCAL(oparg)));
             value = PyStackRef_DUP(GETLOCAL(oparg));
             stack_pointer[0] = value;
             stack_pointer += 1;
@@ -126,7 +126,7 @@
             _PyStackRef value;
             oparg = 6;
             assert(oparg == CURRENT_OPARG());
-            assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+            assert(!PyStackRef_IsNull(GETLOCAL(oparg)));
             value = PyStackRef_DUP(GETLOCAL(oparg));
             stack_pointer[0] = value;
             stack_pointer += 1;
@@ -138,7 +138,7 @@
             _PyStackRef value;
             oparg = 7;
             assert(oparg == CURRENT_OPARG());
-            assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+            assert(!PyStackRef_IsNull(GETLOCAL(oparg)));
             value = PyStackRef_DUP(GETLOCAL(oparg));
             stack_pointer[0] = value;
             stack_pointer += 1;
@@ -149,7 +149,7 @@
         case _LOAD_FAST: {
             _PyStackRef value;
             oparg = CURRENT_OPARG();
-            assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+            assert(!PyStackRef_IsNull(GETLOCAL(oparg)));
             value = PyStackRef_DUP(GETLOCAL(oparg));
             stack_pointer[0] = value;
             stack_pointer += 1;
@@ -757,7 +757,7 @@
                 err = 1;
             }
             else {
-                err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectSteal(v));
+                err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v));
                 Py_DECREF(slice);
             }
             PyStackRef_CLOSE(v);
@@ -939,7 +939,7 @@
             v = stack_pointer[-1];
             set = stack_pointer[-2 - (oparg-1)];
             int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set),
-                                PyStackRef_AsPyObjectSteal(v));
+                                PyStackRef_AsPyObjectBorrow(v));
             PyStackRef_CLOSE(v);
             if (err) JUMP_TO_ERROR();
             stack_pointer += -1;
@@ -955,7 +955,7 @@
             container = stack_pointer[-2];
             v = stack_pointer[-3];
             /* container[sub] = v */
-            int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(v));
+            int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v));
             PyStackRef_CLOSE(v);
             PyStackRef_CLOSE(container);
             PyStackRef_CLOSE(sub);
@@ -1281,7 +1281,7 @@
             exc_value = stack_pointer[-1];
             _PyErr_StackItem *exc_info = tstate->exc_info;
             Py_XSETREF(exc_info->exc_value,
-                       PyStackRef_AsPyObjectBorrow(exc_value) == Py_None
+                       PyStackRef_Is(exc_value, PyStackRef_None)
                        ? NULL : PyStackRef_AsPyObjectSteal(exc_value));
             stack_pointer += -1;
             assert(WITHIN_STACK_BOUNDS());
@@ -1338,9 +1338,9 @@
                 if (true) JUMP_TO_ERROR();
             }
             if (PyDict_CheckExact(ns))
-            err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v));
+            err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v));
             else
-            err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v));
+            err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v));
             PyStackRef_CLOSE(v);
             if (err) JUMP_TO_ERROR();
             stack_pointer += -1;
@@ -1483,7 +1483,7 @@
             v = stack_pointer[-2];
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
             int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner),
-                                       name, PyStackRef_AsPyObjectSteal(v));
+                                       name, PyStackRef_AsPyObjectBorrow(v));
             PyStackRef_CLOSE(v);
             PyStackRef_CLOSE(owner);
             if (err) JUMP_TO_ERROR();
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 14959fb31be201..32b22aff14a768 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -4667,7 +4667,7 @@
             next_instr += 1;
             INSTRUCTION_STATS(LOAD_FAST);
             _PyStackRef value;
-            assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+            assert(!PyStackRef_IsNull(GETLOCAL(oparg)));
             value = PyStackRef_DUP(GETLOCAL(oparg));
             stack_pointer[0] = value;
             stack_pointer += 1;
@@ -5359,7 +5359,7 @@
             exc_value = stack_pointer[-1];
             _PyErr_StackItem *exc_info = tstate->exc_info;
             Py_XSETREF(exc_info->exc_value,
-                       PyStackRef_AsPyObjectBorrow(exc_value) == Py_None
+                       PyStackRef_Is(exc_value, PyStackRef_None)
                        ? NULL : PyStackRef_AsPyObjectSteal(exc_value));
             stack_pointer += -1;
             assert(WITHIN_STACK_BOUNDS());
@@ -5876,7 +5876,7 @@
             v = stack_pointer[-1];
             set = stack_pointer[-2 - (oparg-1)];
             int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set),
-                                PyStackRef_AsPyObjectSteal(v));
+                                PyStackRef_AsPyObjectBorrow(v));
             PyStackRef_CLOSE(v);
             if (err) goto pop_1_error;
             stack_pointer += -1;
@@ -5977,7 +5977,7 @@
             {
                 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
                 int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner),
-                                       name, PyStackRef_AsPyObjectSteal(v));
+                                       name, PyStackRef_AsPyObjectBorrow(v));
                 PyStackRef_CLOSE(v);
                 PyStackRef_CLOSE(owner);
                 if (err) goto pop_2_error;
@@ -6215,9 +6215,9 @@
                 if (true) goto pop_1_error;
             }
             if (PyDict_CheckExact(ns))
-            err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v));
+            err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v));
             else
-            err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v));
+            err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v));
             PyStackRef_CLOSE(v);
             if (err) goto pop_1_error;
             stack_pointer += -1;
@@ -6244,7 +6244,7 @@
                 err = 1;
             }
             else {
-                err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectSteal(v));
+                err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v));
                 Py_DECREF(slice);
             }
             PyStackRef_CLOSE(v);
@@ -6285,7 +6285,7 @@
             v = stack_pointer[-3];
             {
                 /* container[sub] = v */
-                int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(v));
+                int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v));
                 PyStackRef_CLOSE(v);
                 PyStackRef_CLOSE(container);
                 PyStackRef_CLOSE(sub);
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 39eaa86098b2e5..9b60c89c5f512a 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -837,6 +837,27 @@ pycore_init_builtins(PyThreadState *tstate)
     return _PyStatus_ERR("can't initialize builtins module");
 }
 
+#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
+static PyStatus
+pycore_init_stackrefs(PyInterpreterState *interp)
+{
+    interp->stackref_state.entries = PyMem_Malloc(sizeof(struct _Py_stackref_entry) * 1024);
+    if (interp->stackref_state.entries == NULL) {
+        goto error;
+    }
+    interp->stackref_state.next_ref = 0;
+    interp->stackref_state.n_entries = 1024;
+    // Reserve NULL, True, False, None
+    _Py_object_to_stackref_transition(NULL, Py_TAG_DEFERRED, STEAL);
+    _Py_object_to_stackref_transition(Py_True, Py_TAG_DEFERRED, STEAL);
+    _Py_object_to_stackref_transition(Py_False, Py_TAG_DEFERRED, STEAL);
+    _Py_object_to_stackref_transition(Py_None, Py_TAG_DEFERRED, STEAL);
+    return _PyStatus_OK();
+
+error:
+    return _PyStatus_ERR("can't initialize stackrefs to pyobject mapping");
+}
+#endif
 
 static PyStatus
 pycore_interp_init(PyThreadState *tstate)
@@ -845,6 +866,13 @@ pycore_interp_init(PyThreadState *tstate)
     PyStatus status;
     PyObject *sysmod = NULL;
 
+#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
+    status = pycore_init_stackrefs(interp);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+#endif
+
     // Create singletons before the first PyType_Ready() call, since
     // PyType_Ready() uses singletons like the Unicode empty string (tp_doc)
     // and the empty tuple singletons (tp_bases).
diff --git a/Python/pystate.c b/Python/pystate.c
index 602b13e18c71ae..746a3b92a2d52c 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -903,6 +903,10 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)
         interp->code_watchers[i] = NULL;
     }
     interp->active_code_watchers = 0;
+
+#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
+    PyMem_Free(interp->stackref_state.entries);
+#endif
     // XXX Once we have one allocator per interpreter (i.e.
     // per-interpreter GC) we must ensure that all of the interpreter's
     // objects have been cleaned up at the point.
diff --git a/Python/stackref.c b/Python/stackref.c
new file mode 100644
index 00000000000000..d265c1265899f3
--- /dev/null
+++ b/Python/stackref.c
@@ -0,0 +1,112 @@
+#include "Python.h"
+
+#include "pycore_stackref.h"
+#include "pycore_interp.h"
+#include "pycore_pyatomic_ft_wrappers.h"
+#include "pycore_critical_section.h"
+
+int
+_PyStackRef_IsLive(_PyStackRef stackref)
+{
+#if defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
+    PyInterpreterState *interp = PyInterpreterState_Get();
+    struct _Py_stackref_entry *entry = &interp->stackref_state.entries[stackref.bits >> RESERVED_BITS];
+    return entry->is_live;
+#else
+    return 1;
+#endif
+}
+
+#if defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
+PyObject *
+_Py_stackref_to_object_transition(_PyStackRef stackref, _PyStackRef_OpKind op)
+{
+    PyObject *res;
+    PyInterpreterState *interp = PyInterpreterState_Get();
+    assert(interp != NULL);
+    switch (op) {
+        case STEAL: {
+            PyMutex_Lock(&interp->stackref_state.lock);
+            struct _Py_stackref_entry *entry = &interp->stackref_state.entries[stackref.bits >> RESERVED_BITS];
+            assert(entry->is_live);
+            res = entry->obj;
+            entry->is_live = _Py_IsImmortal(entry->obj);
+            PyMutex_Unlock(&interp->stackref_state.lock);
+            break;
+        }
+        case NEW:
+        case BORROW: {
+            struct _Py_stackref_entry *entry = &interp->stackref_state.entries[stackref.bits >> RESERVED_BITS];
+            assert(entry->is_live);
+            res = entry->obj;
+            break;
+        }
+    }
+    return res;
+}
+
+_PyStackRef
+_Py_object_to_stackref_transition(PyObject *obj, char tag, _PyStackRef_OpKind op)
+{
+    _PyStackRef res;
+    PyInterpreterState *interp = PyInterpreterState_Get();
+    assert(interp != NULL);
+    switch (op) {
+        case STEAL:
+        case NEW:
+        {
+            PyMutex_Lock(&interp->stackref_state.lock);
+            res.bits = (interp->stackref_state.next_ref << RESERVED_BITS) | tag;
+            struct _Py_stackref_entry *entry = &interp->stackref_state.entries[interp->stackref_state.next_ref];
+            entry->is_live = 1;
+            entry->obj = obj;
+            interp->stackref_state.next_ref++;
+            // Out of space, allocate new one.
+            if (interp->stackref_state.next_ref >= interp->stackref_state.n_entries) {
+                size_t old_size = interp->stackref_state.n_entries;
+                interp->stackref_state.n_entries *= 2;
+                struct _Py_stackref_entry *new_mem = PyMem_Malloc(
+                    sizeof(struct _Py_stackref_entry) *
+                    interp->stackref_state.n_entries);
+                if (new_mem == NULL) {
+                    fprintf(stderr, "OOM for PyStackRef\n");
+                    Py_FatalError("Cannot allocate memory for stack references.\n");
+                    return PyStackRef_NULL;
+                }
+                memcpy(new_mem,
+                       interp->stackref_state.entries,
+                       old_size * sizeof(struct _Py_stackref_entry));
+                PyMem_Free(interp->stackref_state.entries);
+                interp->stackref_state.entries = new_mem;
+            }
+            PyMutex_Unlock(&interp->stackref_state.lock);
+            break;
+        }
+        case BORROW:
+            Py_UNREACHABLE();
+    }
+    return res;
+}
+
+void
+_PyStackRef_Close(_PyStackRef stackref)
+{
+    PyInterpreterState *interp = PyInterpreterState_Get();
+    assert(interp != NULL);
+    PyMutex_Lock(&interp->stackref_state.lock);
+    struct _Py_stackref_entry *entry = &interp->stackref_state.entries[stackref.bits >> RESERVED_BITS];
+    assert(entry->is_live);
+    entry->is_live = _Py_IsImmortal(entry->obj);
+    PyMutex_Unlock(&interp->stackref_state.lock);
+}
+
+_PyStackRef
+_PyStackRef_Dup(_PyStackRef stackref)
+{
+    PyInterpreterState *interp = PyInterpreterState_Get();
+    assert(interp != NULL);
+    char tag = stackref.bits & RESERVED_BITS;
+    PyObject *val = _Py_stackref_to_object_transition(stackref, BORROW);
+    return _Py_object_to_stackref_transition(val, tag, NEW);
+}
+#endif
diff --git a/configure b/configure
index 527bb0bfb1fa0e..17fd7f218453a1 100755
--- a/configure
+++ b/configure
@@ -1084,6 +1084,7 @@ enable_shared
 with_static_libpython
 enable_profiling
 enable_gil
+with_pystackrefdebug
 with_pydebug
 with_trace_refs
 enable_pystats
@@ -1862,6 +1863,7 @@ Optional Packages:
   --without-static-libpython
                           do not build libpythonMAJOR.MINOR.a and do not
                           install python.o (default is yes)
+  --with-pystackrefdebug  build with Py_STACKREF_DEBUG defined (default is no)
   --with-pydebug          build with Py_DEBUG defined (default is no)
   --with-trace-refs       enable tracing references for debugging purpose
                           (default is no)
@@ -8121,6 +8123,32 @@ printf "%s\n" "#define Py_GIL_DISABLED 1" >>confdefs.h
   ABIFLAGS="${ABIFLAGS}t"
 fi
 
+# Check for --with-pystackrefdebug
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-pystackrefdebug" >&5
+printf %s "checking for --with-pystackrefdebug... " >&6; }
+
+# Check whether --with-pystackrefdebug was given.
+if test ${with_pystackrefdebug+y}
+then :
+  withval=$with_pystackrefdebug;
+if test "$withval" != no
+then
+
+printf "%s\n" "#define Py_STACKREF_DEBUG 1" >>confdefs.h
+
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+printf "%s\n" "yes" >&6; };
+  Py_STACKREF_DEBUG='true'
+  ABIFLAGS="${ABIFLAGS}d"
+else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }; Py_STACKREF_DEBUG='false'
+fi
+else $as_nop
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+
+
 # Check for --with-pydebug
 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-pydebug" >&5
 printf %s "checking for --with-pydebug... " >&6; }
diff --git a/configure.ac b/configure.ac
index 9b8ab706ae5ba9..89bcfd74679da8 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1700,6 +1700,22 @@ then
   ABIFLAGS="${ABIFLAGS}t"
 fi
 
+# Check for --with-pystackrefdebug
+AC_MSG_CHECKING([for --with-pystackrefdebug])
+AC_ARG_WITH([pystackrefdebug],
+  [AS_HELP_STRING([--with-pystackrefdebug], [build with Py_STACKREF_DEBUG defined (default is no)]) ],
+[
+if test "$withval" != no
+then
+  AC_DEFINE([Py_STACKREF_DEBUG], [1],
+  [Define if you want to build an interpreter with stack reference checks on (WARNING: EXPERIMENTAL).])
+  AC_MSG_RESULT([yes]);
+  Py_STACKREF_DEBUG='true'
+  ABIFLAGS="${ABIFLAGS}d"
+else AC_MSG_RESULT([no]); Py_STACKREF_DEBUG='false'
+fi],
+[AC_MSG_RESULT([no])])
+
 # Check for --with-pydebug
 AC_MSG_CHECKING([for --with-pydebug])
 AC_ARG_WITH([pydebug],
diff --git a/pyconfig.h.in b/pyconfig.h.in
index c279b147db3bdd..17ecc0418156a1 100644
--- a/pyconfig.h.in
+++ b/pyconfig.h.in
@@ -1662,6 +1662,10 @@
 /* Define if rl_startup_hook takes arguments */
 #undef Py_RL_STARTUP_HOOK_TAKES_ARGS
 
+/* Define if you want to build an interpreter with stack reference checks on
+   (WARNING: EXPERIMENTAL). */
+#undef Py_STACKREF_DEBUG
+
 /* Define if you want to enable internal statistics gathering. */
 #undef Py_STATS
 

From 0b0fa3d4431514b0bc25001bbbb9889f65f5101f Mon Sep 17 00:00:00 2001
From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com>
Date: Sat, 29 Jun 2024 00:02:46 +0800
Subject: [PATCH 2/9] Create 2024-06-29-00-02-42.gh-issue-117139.-lKXPB.rst

---
 .../next/Build/2024-06-29-00-02-42.gh-issue-117139.-lKXPB.rst | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 Misc/NEWS.d/next/Build/2024-06-29-00-02-42.gh-issue-117139.-lKXPB.rst

diff --git a/Misc/NEWS.d/next/Build/2024-06-29-00-02-42.gh-issue-117139.-lKXPB.rst b/Misc/NEWS.d/next/Build/2024-06-29-00-02-42.gh-issue-117139.-lKXPB.rst
new file mode 100644
index 00000000000000..26714016643274
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2024-06-29-00-02-42.gh-issue-117139.-lKXPB.rst
@@ -0,0 +1,4 @@
+Added a new ``--with-pystackrefs`` configure build option for CPython. This
+option is fully experimental and may be modified or removed without prior
+notice. Enabling the option turns on internal handle-like debugging for
+CPython's main interpreter loop.

From d2c7bb9edd84c83ea8e51991536eb22f343c2262 Mon Sep 17 00:00:00 2001
From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com>
Date: Sat, 29 Jun 2024 00:07:59 +0800
Subject: [PATCH 3/9] fix CI

---
 .github/workflows/build.yml          | 1 +
 .github/workflows/reusable-macos.yml | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 8f3e76ac04d35a..e6814b83db863e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -564,6 +564,7 @@ jobs:
     - check_generated_files
     - build_macos
     - build_macos_free_threading
+    - build_macos_free_threading_with_stackref_debug
     - build_ubuntu
     - build_ubuntu_free_threading
     - build_ubuntu_ssltests
diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml
index 632a7eb69d3c9b..4910d6ee79c7c2 100644
--- a/.github/workflows/reusable-macos.yml
+++ b/.github/workflows/reusable-macos.yml
@@ -68,4 +68,4 @@ jobs:
     - name: Tests
       # Stackref debug is 2.5x slower than normal CPython,
       # so we only run it on a restricted subset of tests.
-      run: ${{ inputs.stackref-debug && './python -m test test_asyncio' || 'make test' }}
+      run: ${{ inputs.stackref-debug && './python.exe -m test test_asyncio test_typing' || 'make test' }}

From 2c6860af8a2eece04ea00c1dcdfe1ca30e130d73 Mon Sep 17 00:00:00 2001
From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com>
Date: Sat, 29 Jun 2024 00:12:51 +0800
Subject: [PATCH 4/9] Fixup defines

---
 Python/pylifecycle.c | 4 ++--
 Python/pystate.c     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 9b60c89c5f512a..93ae2396bb1999 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -837,7 +837,7 @@ pycore_init_builtins(PyThreadState *tstate)
     return _PyStatus_ERR("can't initialize builtins module");
 }
 
-#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
+#if defined(Py_STACKREF_DEBUG) && defined(Py_GIL_DISABLED)
 static PyStatus
 pycore_init_stackrefs(PyInterpreterState *interp)
 {
@@ -866,7 +866,7 @@ pycore_interp_init(PyThreadState *tstate)
     PyStatus status;
     PyObject *sysmod = NULL;
 
-#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
+#if defined(Py_STACKREF_DEBUG) && defined(Py_GIL_DISABLED)
     status = pycore_init_stackrefs(interp);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
diff --git a/Python/pystate.c b/Python/pystate.c
index 746a3b92a2d52c..bbcdcd3020aa32 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -904,7 +904,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)
     }
     interp->active_code_watchers = 0;
 
-#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
+#if defined(Py_STACKREF_DEBUG) && defined(Py_GIL_DISABLED)
     PyMem_Free(interp->stackref_state.entries);
 #endif
     // XXX Once we have one allocator per interpreter (i.e.

From ba49e5886d8f0a951749c013bf71b09ca3579059 Mon Sep 17 00:00:00 2001
From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com>
Date: Sun, 30 Jun 2024 16:54:53 +0800
Subject: [PATCH 5/9] Use a hashtable instead

---
 .github/workflows/reusable-macos.yml |  4 +-
 Include/internal/pycore_stackref.h   |  4 +-
 Python/ceval.c                       | 10 ----
 Python/pylifecycle.c                 | 18 +++++--
 Python/stackref.c                    | 76 ++++++++++++----------------
 5 files changed, 50 insertions(+), 62 deletions(-)

diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml
index 4910d6ee79c7c2..6611b9c69292ae 100644
--- a/.github/workflows/reusable-macos.yml
+++ b/.github/workflows/reusable-macos.yml
@@ -66,6 +66,6 @@ jobs:
     - name: Display build info
       run: make pythoninfo
     - name: Tests
-      # Stackref debug is 2.5x slower than normal CPython,
+      # Stackref debug is 3.5x slower than normal CPython,
       # so we only run it on a restricted subset of tests.
-      run: ${{ inputs.stackref-debug && './python.exe -m test test_asyncio test_typing' || 'make test' }}
+      run: ${{ inputs.stackref-debug && './python.exe -m test test_typing' || 'make test' }}
diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h
index fabefffa090ba9..1997d116786be8 100644
--- a/Include/internal/pycore_stackref.h
+++ b/Include/internal/pycore_stackref.h
@@ -9,6 +9,7 @@ extern "C" {
 #endif
 
 #include "pycore_object_deferred.h"
+#include "pycore_hashtable.h"
 
 #include <stddef.h>
 
@@ -59,8 +60,7 @@ struct _Py_stackref_entry {
 
 struct _Py_stackref_state {
     PyMutex lock;
-    size_t n_entries;
-    struct _Py_stackref_entry *entries;
+    _Py_hashtable_t *entries;
     size_t next_ref;
 };
 
diff --git a/Python/ceval.c b/Python/ceval.c
index abf6865074fd71..f4b3a417025c14 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -726,16 +726,6 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch)
     }
 }
 
-static int
-_PyEval_StackIsAllLive(_PyStackRef *stack_base, _PyStackRef *stack_pointer)
-{
-    while (stack_pointer > stack_base) {
-        assert(_PyStackRef_IsLive(stack_pointer[-1]));
-        stack_pointer--;
-    }
-    return 1;
-}
-
 /* _PyEval_EvalFrameDefault() is a *big* function,
  * so consume 3 units of C stack */
 #define PY_EVAL_C_STACK_UNITS 2
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 93ae2396bb1999..f789afbae221d0 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -841,14 +841,22 @@ pycore_init_builtins(PyThreadState *tstate)
 static PyStatus
 pycore_init_stackrefs(PyInterpreterState *interp)
 {
-    interp->stackref_state.entries = PyMem_Malloc(sizeof(struct _Py_stackref_entry) * 1024);
+    _Py_hashtable_allocator_t alloc = {
+        .malloc = PyMem_Malloc,
+        .free = PyMem_Free,
+    };
+    interp->stackref_state.entries = _Py_hashtable_new_full(
+        _Py_hashtable_hash_ptr,
+    _Py_hashtable_compare_direct,
+        NULL,
+        NULL,
+        &alloc
+    );
     if (interp->stackref_state.entries == NULL) {
         goto error;
     }
-    interp->stackref_state.next_ref = 0;
-    interp->stackref_state.n_entries = 1024;
-    // Reserve NULL, True, False, None
-    _Py_object_to_stackref_transition(NULL, Py_TAG_DEFERRED, STEAL);
+    interp->stackref_state.next_ref = 1;
+    // Reserve True, False, None
     _Py_object_to_stackref_transition(Py_True, Py_TAG_DEFERRED, STEAL);
     _Py_object_to_stackref_transition(Py_False, Py_TAG_DEFERRED, STEAL);
     _Py_object_to_stackref_transition(Py_None, Py_TAG_DEFERRED, STEAL);
diff --git a/Python/stackref.c b/Python/stackref.c
index d265c1265899f3..b40f5e0e0536a4 100644
--- a/Python/stackref.c
+++ b/Python/stackref.c
@@ -8,40 +8,49 @@
 int
 _PyStackRef_IsLive(_PyStackRef stackref)
 {
-#if defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
-    PyInterpreterState *interp = PyInterpreterState_Get();
-    struct _Py_stackref_entry *entry = &interp->stackref_state.entries[stackref.bits >> RESERVED_BITS];
-    return entry->is_live;
-#else
+    _Py_stackref_to_object_transition(stackref, BORROW);
     return 1;
-#endif
 }
 
+
 #if defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
 PyObject *
 _Py_stackref_to_object_transition(_PyStackRef stackref, _PyStackRef_OpKind op)
 {
-    PyObject *res;
+    PyObject *res = NULL;
     PyInterpreterState *interp = PyInterpreterState_Get();
     assert(interp != NULL);
+    int is_null = 0;
+    void *bits = (void *)(stackref.bits >> RESERVED_BITS);
     switch (op) {
         case STEAL: {
+            // We need to distinguish NULL here because _Py_hashtable_get
+            // returns NULL if it cannot find a value.
+            if (bits == 0) {
+                is_null = 1;
+                break;
+            }
             PyMutex_Lock(&interp->stackref_state.lock);
-            struct _Py_stackref_entry *entry = &interp->stackref_state.entries[stackref.bits >> RESERVED_BITS];
-            assert(entry->is_live);
-            res = entry->obj;
-            entry->is_live = _Py_IsImmortal(entry->obj);
+            res = (PyObject *)_Py_hashtable_get(interp->stackref_state.entries, bits);
+            if (res != NULL && !_Py_IsImmortal(res)) {
+                res = (PyObject *)_Py_hashtable_steal(interp->stackref_state.entries, bits);
+            }
             PyMutex_Unlock(&interp->stackref_state.lock);
             break;
         }
         case NEW:
         case BORROW: {
-            struct _Py_stackref_entry *entry = &interp->stackref_state.entries[stackref.bits >> RESERVED_BITS];
-            assert(entry->is_live);
-            res = entry->obj;
+            if (bits == 0) {
+                is_null = 1;
+                break;
+            }
+            PyMutex_Lock(&interp->stackref_state.lock);
+            res = (PyObject *)_Py_hashtable_get(interp->stackref_state.entries, bits);
+            PyMutex_Unlock(&interp->stackref_state.lock);
             break;
         }
     }
+    assert((res != NULL) ^ is_null);
     return res;
 }
 
@@ -55,30 +64,17 @@ _Py_object_to_stackref_transition(PyObject *obj, char tag, _PyStackRef_OpKind op
         case STEAL:
         case NEW:
         {
+            if (obj == NULL) {
+                return PyStackRef_NULL;
+            }
             PyMutex_Lock(&interp->stackref_state.lock);
-            res.bits = (interp->stackref_state.next_ref << RESERVED_BITS) | tag;
-            struct _Py_stackref_entry *entry = &interp->stackref_state.entries[interp->stackref_state.next_ref];
-            entry->is_live = 1;
-            entry->obj = obj;
-            interp->stackref_state.next_ref++;
-            // Out of space, allocate new one.
-            if (interp->stackref_state.next_ref >= interp->stackref_state.n_entries) {
-                size_t old_size = interp->stackref_state.n_entries;
-                interp->stackref_state.n_entries *= 2;
-                struct _Py_stackref_entry *new_mem = PyMem_Malloc(
-                    sizeof(struct _Py_stackref_entry) *
-                    interp->stackref_state.n_entries);
-                if (new_mem == NULL) {
-                    fprintf(stderr, "OOM for PyStackRef\n");
-                    Py_FatalError("Cannot allocate memory for stack references.\n");
-                    return PyStackRef_NULL;
-                }
-                memcpy(new_mem,
-                       interp->stackref_state.entries,
-                       old_size * sizeof(struct _Py_stackref_entry));
-                PyMem_Free(interp->stackref_state.entries);
-                interp->stackref_state.entries = new_mem;
+            uintptr_t key = interp->stackref_state.next_ref;
+            res.bits = (key << RESERVED_BITS) | tag;
+            int err = _Py_hashtable_set(interp->stackref_state.entries, (void *)key, obj);
+            if (err < 0) {
+                Py_FatalError("Stackref handle allocation failed.\n");
             }
+            interp->stackref_state.next_ref++;
             PyMutex_Unlock(&interp->stackref_state.lock);
             break;
         }
@@ -91,13 +87,7 @@ _Py_object_to_stackref_transition(PyObject *obj, char tag, _PyStackRef_OpKind op
 void
 _PyStackRef_Close(_PyStackRef stackref)
 {
-    PyInterpreterState *interp = PyInterpreterState_Get();
-    assert(interp != NULL);
-    PyMutex_Lock(&interp->stackref_state.lock);
-    struct _Py_stackref_entry *entry = &interp->stackref_state.entries[stackref.bits >> RESERVED_BITS];
-    assert(entry->is_live);
-    entry->is_live = _Py_IsImmortal(entry->obj);
-    PyMutex_Unlock(&interp->stackref_state.lock);
+    _Py_stackref_to_object_transition(stackref, STEAL);
 }
 
 _PyStackRef

From 18d26ce2931c57c1f9c06fe1e1a5e35b18cf2105 Mon Sep 17 00:00:00 2001
From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com>
Date: Sun, 30 Jun 2024 16:55:34 +0800
Subject: [PATCH 6/9] fix unused warning

---
 Python/ceval.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/Python/ceval.c b/Python/ceval.c
index f4b3a417025c14..8b8389a5114965 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -726,6 +726,18 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch)
     }
 }
 
+#if Py_STACKREF_DEBUG
+static int
+_PyEval_StackIsAllLive(_PyStackRef *stack_base, _PyStackRef *stack_pointer)
+{
+    while (stack_pointer > stack_base) {
+        assert(_PyStackRef_IsLive(stack_pointer[-1]));
+        stack_pointer--;
+    }
+    return 1;
+}
+#endif
+
 /* _PyEval_EvalFrameDefault() is a *big* function,
  * so consume 3 units of C stack */
 #define PY_EVAL_C_STACK_UNITS 2

From 5ff2a576e8720b0c92d27b0b31980e2d712e4d6d Mon Sep 17 00:00:00 2001
From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com>
Date: Sun, 30 Jun 2024 17:13:31 +0800
Subject: [PATCH 7/9] fix build

---
 Include/internal/pycore_stackref.h |  4 ++--
 Python/stackref.c                  | 17 +++++++++--------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h
index 1997d116786be8..f5cb316a707e30 100644
--- a/Include/internal/pycore_stackref.h
+++ b/Include/internal/pycore_stackref.h
@@ -70,8 +70,8 @@ typedef enum _PyStackRef_OpKind {
     NEW,
 } _PyStackRef_OpKind;
 
-PyObject *_Py_stackref_to_object_transition(_PyStackRef stackref, _PyStackRef_OpKind op);
-_PyStackRef _Py_object_to_stackref_transition(PyObject *obj, char tag, _PyStackRef_OpKind op);
+PyAPI_FUNC(PyObject *)_Py_stackref_to_object_transition(_PyStackRef stackref, _PyStackRef_OpKind op);
+PyAPI_FUNC(_PyStackRef) _Py_object_to_stackref_transition(PyObject *obj, char tag, _PyStackRef_OpKind op);
 PyAPI_FUNC(int) _PyStackRef_IsLive(_PyStackRef stackref);
 PyAPI_FUNC(void) _PyStackRef_Close(_PyStackRef stackref);
 PyAPI_FUNC(_PyStackRef) _PyStackRef_Dup(_PyStackRef stackref);
diff --git a/Python/stackref.c b/Python/stackref.c
index b40f5e0e0536a4..63d7127b0bfd77 100644
--- a/Python/stackref.c
+++ b/Python/stackref.c
@@ -5,14 +5,6 @@
 #include "pycore_pyatomic_ft_wrappers.h"
 #include "pycore_critical_section.h"
 
-int
-_PyStackRef_IsLive(_PyStackRef stackref)
-{
-    _Py_stackref_to_object_transition(stackref, BORROW);
-    return 1;
-}
-
-
 #if defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
 PyObject *
 _Py_stackref_to_object_transition(_PyStackRef stackref, _PyStackRef_OpKind op)
@@ -100,3 +92,12 @@ _PyStackRef_Dup(_PyStackRef stackref)
     return _Py_object_to_stackref_transition(val, tag, NEW);
 }
 #endif
+
+int
+_PyStackRef_IsLive(_PyStackRef stackref)
+{
+#if Py_STACKREF_DEBUG
+    _Py_stackref_to_object_transition(stackref, BORROW);
+#endif
+    return 1;
+}

From e2563c6844ff4382252733d2bdda0115c43b8c24 Mon Sep 17 00:00:00 2001
From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com>
Date: Sun, 30 Jun 2024 17:16:39 +0800
Subject: [PATCH 8/9] Fix build again

---
 Python/ceval_macros.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h
index 3757e30a6c811b..1f2316a6999b27 100644
--- a/Python/ceval_macros.h
+++ b/Python/ceval_macros.h
@@ -86,7 +86,7 @@
 #define PRE_DISPATCH_GOTO() ((void)0)
 #endif
 
-#if defined(Py_GIL_DISABLED) && defined(Py_DEBUG)
+#ifdef Py_STACKREF_DEBUG
 #define STACKREF_CHECK() \
     do { \
         if (frame->f_executable && PyCode_Check(frame->f_executable)) { \

From a058c345ce90f044ed4bd3691f84c0b78408f6d9 Mon Sep 17 00:00:00 2001
From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com>
Date: Wed, 17 Jul 2024 18:19:23 +0800
Subject: [PATCH 9/9] cleanup

---
 Include/internal/pycore_stackref.h | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h
index 1aa040a9685330..249f3493fcf1fb 100644
--- a/Include/internal/pycore_stackref.h
+++ b/Include/internal/pycore_stackref.h
@@ -53,11 +53,6 @@ typedef union _PyStackRef {
     uintptr_t bits;
 } _PyStackRef;
 
-struct _Py_stackref_entry {
-    PyObject *obj;
-    int is_live;
-};
-
 struct _Py_stackref_state {
     PyMutex lock;
     _Py_hashtable_t *entries;
@@ -118,7 +113,7 @@ PyAPI_FUNC(_PyStackRef) _PyStackRef_Dup(_PyStackRef stackref);
     static const _PyStackRef PyStackRef_None = { .bits = 3 << RESERVED_BITS };
 #   else
 #   define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED })
-#endif
+#   endif
 #else
 #   define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) })
 #endif
@@ -128,7 +123,7 @@ PyAPI_FUNC(_PyStackRef) _PyStackRef_Dup(_PyStackRef stackref);
 #ifdef Py_GIL_DISABLED
 static inline int
 PyStackRef_Is(_PyStackRef a, _PyStackRef b) {
-#   if defined(Py_STACKREF_DEBUG)
+#   ifdef Py_STACKREF_DEBUG
     // Note: stackrefs are unique. So even immortal objects will produce different stackrefs.
     return _Py_stackref_to_object_transition(a, BORROW) == _Py_stackref_to_object_transition(b, BORROW);
 #   else
@@ -136,7 +131,7 @@ PyStackRef_Is(_PyStackRef a, _PyStackRef b) {
 #   endif
 }
 #else
-#define PyStackRef_Is(a, b) ((a).bits == (b).bits)
+#   define PyStackRef_Is(a, b) ((a).bits == (b).bits)
 #endif
 
 static inline int
@@ -154,7 +149,7 @@ PyStackRef_IsNull(_PyStackRef stackref)
 static inline PyObject *
 PyStackRef_AsPyObjectBorrow(_PyStackRef stackref)
 {
-#   if defined(Py_STACKREF_DEBUG)
+#   ifdef Py_STACKREF_DEBUG
     return _Py_stackref_to_object_transition(stackref, BORROW);
 #   else
     PyObject *cleared = ((PyObject *)((stackref).bits & (~Py_TAG_BITS)));
@@ -171,14 +166,14 @@ PyStackRef_AsPyObjectBorrow(_PyStackRef stackref)
 static inline PyObject *
 PyStackRef_AsPyObjectSteal(_PyStackRef stackref)
 {
-#ifdef Py_STACKREF_DEBUG
+#   ifdef Py_STACKREF_DEBUG
     return _Py_stackref_to_object_transition(stackref, STEAL);
-#else
+#   else
     if (!PyStackRef_IsNull(stackref) && PyStackRef_IsDeferred(stackref)) {
         return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref));
     }
     return PyStackRef_AsPyObjectBorrow(stackref);
-#endif
+#   endif
 }
 #else
 #   define PyStackRef_AsPyObjectSteal(stackref) PyStackRef_AsPyObjectBorrow(stackref)
@@ -301,7 +296,6 @@ PyStackRef_CLOSE(_PyStackRef stackref)
 static inline _PyStackRef
 PyStackRef_DUP(_PyStackRef stackref)
 {
-#ifdef Py_GIL_DISABLED
 #   ifdef Py_STACKREF_DEBUG
     stackref = _PyStackRef_Dup(stackref);
 #   endif