Compute f_back when on thread stack, only filling in value when frame…

… object 8000 outlives stack invocation.
python · markshannon · Jul 26, 2021 · Jun 21, 2021 · Jun 22, 2021 · Jun 22, 2021
commit bd95c3297b5017ef3ee2d7072a0588af05de8a16
@@ -78,7 +78,6 @@ struct _ts {
     PyInterpreterState *interp;
 
     /* Borrowed reference to the current frame (it can be NULL) */
-    PyFrameObject *pyframe;
     struct _py_frame *frame;
     int recursion_depth;
     int recursion_headroom; /* Allow 50 more calls to handle any errors. */

diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
@@ -107,6 +107,7 @@ static inline void _Py_LeaveRecursiveCall_inline(void)  {
 
 #define Py_LeaveRecursiveCall() _Py_LeaveRecursiveCall_inline()
 
+struct _py_frame *_PyEval_GetFrame(void);
 
 #ifdef __cplusplus
 }

diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h
@@ -84,7 +84,7 @@ _PyFrame_GetFrameObject(_PyFrame *frame)
 }
 
 int
-_PyFrame_FastToLocalsWithError(_PyFrame *frame, int cleared);
+_PyFrame_FastToLocalsWithError(_PyFrame *frame);
 
 void
 _PyFrame_LocalsToFast(_PyFrame *frame, int clear);

diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c
@@ -7,6 +7,7 @@
 #include "pycore_atomic.h"        // _Py_atomic_int
 #include "pycore_call.h"          // _PyObject_Call()
 #include "pycore_ceval.h"         // _PyEval_SignalReceived()
+#include "pycore_frame.h"
 #include "pycore_moduleobject.h"  // _PyModule_GetState()
 #include "pycore_pyerrors.h"      // _PyErr_SetString()
 #include "pycore_pylifecycle.h"   // NSIG
@@ -1786,11 +1787,7 @@ _PyErr_CheckSignalsTstate(PyThreadState *tstate)
      */
     _Py_atomic_store(&is_tripped, 0);
 
-    PyObject *frame = (PyObject *)tstate->pyframe;
    if (!frame) {
-        frame = Py_None;
-    }
-
+    _PyFrame *frame = tstate->frame;
     signal_state_t *state = &signal_global_state;
     for (int i = 1; i < NSIG; i++) {
         if (!_Py_atomic_load_relaxed(&Handlers[i].tripped)) {
@@ -1821,8 +1818,8 @@ _PyErr_CheckSignalsTstate(PyThreadState *tstate)
             PyErr_WriteUnraisable(Py_None);
             continue;
         }
-
-        PyObject *arglist = Py_BuildValue("(iO)", i, frame);
+        PyObject * f = frame == NULL ? Py_None : (PyObject *)_PyFrame_GetFrameObject(frame);
+        PyObject *arglist = Py_BuildValue("(iO)", i, f);
         PyObject *result;
         if (arglist) {
             result = _PyObject_Call(tstate, func, arglist, NULL);

@@ -14,7 +14,6 @@
 #define OFF(x) offsetof(PyFrameObject, x)
 
 static PyMemberDef frame_memberlist[] = {
-    {"f_back",          T_OBJECT,       OFF(f_back),      READONLY},
     {"f_trace_lines",   T_BOOL,         OFF(f_trace_lines), 0},
     {"f_trace_opcodes", T_BOOL,         OFF(f_trace_opcodes), 0},
     {NULL}      /* Sentinel */
@@ -102,6 +101,15 @@ frame_getcode(PyFrameObject *f, void *closure)
     return (PyObject *)PyFrame_GetCode(f);
 }
 
+static PyObject *
+frame_getback(PyFrameObject *f, void *closure)
+{
+    PyObject *res = (PyObject *)PyFrame_GetBack(f);
+    if (res == NULL) {
+        Py_RETURN_NONE;
+    }
+}
+
 /* Given the index of the effective opcode,
    scan back to construct the oparg with EXTENDED_ARG */
 static unsigned int
@@ -579,6 +587,7 @@ frame_settrace(PyFrameObject *f, PyObject* v, void *closure)
 
 
 static PyGetSetDef frame_getsetlist[] = {
+    {"f_back",          (getter)frame_getback, NULL, NULL},
     {"f_locals",        (getter)frame_getlocals, NULL, NULL},
     {"f_lineno",        (getter)frame_getlineno,
                     (setter)frame_setlineno, NULL},
@@ -847,6 +856,9 @@ _PyFrame_TakeLocals(PyFrameObject *f)
     assert(f->f_frame->stackdepth == 0);
     Py_ssize_t size = ((char*)f->f_frame->stack)-((char *)f->f_localsptr);
     PyObject **copy = PyMem_Malloc(size);
+    if (f->f_frame->previous != NULL) {
+        f->f_back = (PyFrameObject *)Py_NewRef(f->f_frame->previous->frame_obj);
+    }
     if (copy == NULL) {
         for (int i = 0; i < f->f_frame->code->co_nlocalsplus; i++) {
             PyObject *o = f->f_localsptr[i];
@@ -873,7 +885,7 @@ _PyFrame_New_NoTrack(PyThreadState *tstate, _PyFrame *frame, int owns)
     if (f == NULL) {
         return NULL;
     }
-    f->f_back = (PyFrameObject*)Py_XNewRef(tstate->pyframe);
+    f->f_back = NULL;
     f->f_trace = NULL;
     f->f_frame->stackdepth = 0;
     f->f_trace_lines = 1;
@@ -928,7 +940,7 @@ _PyFrame_OpAlreadyRan(_PyFrame *frame, int opcode, int oparg)
 }
 
 int
-_PyFrame_FastToLocalsWithError(_PyFrame *frame, int cleared) {
+_PyFrame_FastToLocalsWithError(_PyFrame *frame) {
     /* Merge fast locals into f->f_locals */
     PyObject *locals;
     PyObject **fast;
@@ -958,7 +970,7 @@ _PyFrame_FastToLocalsWithError(_PyFrame *frame, int cleared) {
 
         PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
         PyObject *value = fast[i];
-        if (!cleared) {
+        if (frame->f_state != FRAME_CLEARED) {
             if (kind & CO_FAST_FREE) {
                 // The cell was set when the frame was created from
                 // the function's closure.
@@ -1012,7 +1024,7 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f)
         PyErr_BadInternalCall();
         return -1;
     }
-    return _PyFrame_FastToLocalsWithError(f->f_frame, f->f_frame->f_state == FRAME_CLEARED);
+    return _PyFrame_FastToLocalsWithError(f->f_frame);
 }
 
 void
@@ -1153,6 +1165,9 @@ PyFrame_GetBack(PyFrameObject *frame)
 {
     assert(frame != NULL);
     PyFrameObject *back = frame->f_back;
+    if (back == NULL && frame->f_frame->previous != NULL) {
+        back = frame->f_frame->previous->frame_obj;
+    }
     Py_XINCREF(back);
     return back;
 }

@@ -186,9 +186,8 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
 
     /* Generators always return to their most recent caller, not
      * necessarily their creator. */
-    Py_XINCREF(tstate->pyframe);
     assert(f->f_back == NULL);
-    f->f_back = tstate->pyframe;
+    f->f_frame->previous = tstate->frame;
 
     gen->gi_exc_state.previous_item = tstate->exc_info;
     tstate->exc_info = &gen->gi_exc_state;
@@ -202,11 +201,12 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
     tstate->exc_info = gen->gi_exc_state.previous_item;
     gen->gi_exc_state.previous_item = NULL;
 
+    tstate->frame = f->f_frame->previous;
     /* Don't keep the reference to f_back any longer than necessary.  It
      * may keep a chain of frames alive or it could create a reference
      * cycle. */
-    assert(f->f_back == tstate->pyframe);
-    Py_CLEAR(f->f_back);
+    f->f_frame->previous = NULL;
+    assert(f->f_back == NULL);
 
     /* If the generator just returned (as opposed to yielding), signal
      * that the generator is exhausted. */
@@ -423,22 +423,22 @@ _gen_throw(PyGenObject *gen, int close_on_genexit,
         if (PyGen_CheckExact(yf) || PyCoro_CheckExact(yf)) {
             /* `yf` is a generator or a coroutine. */
             PyThreadState *tstate = _PyThreadState_GET();
-            PyFrameObject *f = tstate->pyframe;
+            _PyFrame *f = tstate->frame;
 
             /* Since we are fast-tracking things by skipping the eval loop,
                we need to update the current frame so the stack trace
                will be reported correctly to the user. */
             /* XXX We should probably be updating the current frame
                somewhere in ceval.c. */
-            tstate->pyframe = gen->gi_frame;
+            tstate->frame = gen->gi_frame->f_frame;
             /* Close the generator that we are currently iterating with
                'yield from' or awaiting on with 'await'. */
             PyFrameState state = gen->gi_frame->f_frame->f_state;
             gen->gi_frame->f_frame->f_state = FRAME_EXECUTING;
             ret = _gen_throw((PyGenObject *)yf, close_on_genexit,
                              typ, val, tb);
             gen->gi_frame->f_frame->f_state = state;
-            tstate->pyframe = f;
+            tstate->frame = f;
         } else {
             /* `yf` is an iterator or a coroutine-like object. */
             PyObject *meth;
@@ -1153,32 +1153,32 @@ PyTypeObject _PyCoroWrapper_Type = {
 static PyObject *
 compute_cr_origin(int origin_depth)
 {
-    PyFrameObject *frame = PyEval_GetFrame();
+    _PyFrame *frame = _PyEval_GetFrame();
     /* First count how many frames we have */
     int frame_count = 0;
     for (; frame && frame_count < origin_depth; ++frame_count) {
-        frame = frame->f_back;
+        frame = frame->previous;
     }
 
     /* Now collect them */
     PyObject *cr_origin = PyTuple_New(frame_count);
     if (cr_origin == NULL) {
         return NULL;
     }
-    frame = PyEval_GetFrame();
+    frame = _PyEval_GetFrame();
     for (int i = 0; i < frame_count; ++i) {
-        PyCodeObject *code = PyFrame_GetCode(frame);
+        PyCodeObject *code = frame->code;
         PyObject *frameinfo = Py_BuildValue("OiO",
                                             code->co_filename,
-                                            PyFrame_GetLineNumber(frame),
+                                            PyCode_Addr2Line(frame->code, frame->lasti*2),
                                             code->co_name);
         Py_DECREF(code);
         if (!frameinfo) {
             Py_DECREF(cr_origin);
             return NULL;
         }
         PyTuple_SET_ITEM(cr_origin, i, frameinfo);
-        frame = frame->f_back;
+        frame = frame->previous;
     }
 
     return cr_origin;

@@ -1450,8 +1450,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *fo, int throwflag
     tstate->cframe = &cframe;
 
     /* push frame */
-    tstate->pyframe = fo;
     frame = fo->f_frame;
+    tstate->frame = frame;
     co = frame->code;
 
     if (cframe.use_tracing) {
@@ -3507,7 +3507,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *fo, int throwflag
         case TARGET(IMPORT_STAR): {
             PyObject *from = POP(), *locals;
             int err;
-            if (_PyFrame_FastToLocalsWithError(frame, 0) < 0) {
+            if (_PyFrame_FastToLocalsWithError(frame) < 0) {
                 Py_DECREF(from);
                 goto error;
             }
@@ -4404,7 +4404,7 @@ MISS_WITH_CACHE(LOAD_GLOBAL)
     if (PyDTrace_FUNCTION_RETURN_ENABLED())
         dtrace_function_return(frame);
     _Py_LeaveRecursiveCall(tstate);
-    tstate->pyframe = fo->f_back;
+    tstate->frame = frame->previous;
     return _Py_CheckFunctionResult(tstate, NULL, retval, __func__);
 }
 
@@ -4967,9 +4967,7 @@ make_coro(PyThreadState *tstate, PyFrameConstructor *con,
     PyObject *gen;
     int is_coro = ((PyCodeObject *)con->fc_code)->co_flags & CO_COROUTINE;
 
-    /* Don't need to keep the reference to f_back, it will be set
-        * when the generator is resumed. */
-    Py_CLEAR(f->f_back);
+    assert(f->f_back == NULL);
 
     /* Create a new generator that owns the ready to run frame
         * and return that as the value. */
@@ -5676,19 +5674,29 @@ _PyEval_GetAsyncGenFinalizer(void)
     return tstate->async_gen_finalizer;
 }
 
+_PyFrame *
+_PyEval_GetFrame(void)
+{
+    PyThreadState *tstate = _PyThreadState_GET();
+    return tstate->frame;
+}
+
 PyFrameObject *
 PyEval_GetFrame(void)
 {
     PyThreadState *tstate = _PyThreadState_GET();
-    return tstate->pyframe;
+    if (tstate->frame == NULL) {
+        return NULL;
+    }
+    return _PyFrame_GetFrameObject(tstate->frame);
 }
 
 PyObject *
 _PyEval_GetBuiltins(PyThreadState *tstate)
 {
-    PyFrameObject *frame = tstate->pyframe;
+    _PyFrame *frame = tstate->frame;
     if (frame != NULL) {
-        return frame->f_frame->builtins;
+        return frame->builtins;
     }
     return tstate->interp->builtins;
 }
@@ -5719,17 +5727,17 @@ PyObject *
 PyEval_GetLocals(void)
 {
     PyThreadState *tstate = _PyThreadState_GET();
-    PyFrameObject *current_frame = tstate->pyframe;
+     _PyFrame *current_frame = tstate->frame;
     if (current_frame == NULL) {
         _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist");
         return NULL;
     }
 
-    if (PyFrame_FastToLocalsWithError(current_frame) < 0) {
+    if (_PyFrame_FastToLocalsWithError(current_frame) < 0) {
         return NULL;
     }
 
-    PyObject *locals = current_frame->f_frame->locals;
+    PyObject *locals = current_frame->locals;
     assert(locals != NULL);
     return locals;
 }
@@ -5738,22 +5746,22 @@ PyObject *
 PyEval_GetGlobals(void)
 {
     PyThreadState *tstate = _PyThreadState_GET();
-    PyFrameObject *current_frame = tstate->pyframe;
+    _PyFrame *current_frame = tstate->frame;
     if (current_frame == NULL) {
         return NULL;
     }
-    return current_frame->f_frame->globals;
+    return current_frame->globals;
 }
 
 int
 PyEval_MergeCompilerFlags(PyCompilerFlags *cf)
 {
     PyThreadState *tstate = _PyThreadState_GET();
-    PyFrameObject *current_frame = tstate->pyframe;
+    _PyFrame *current_frame = tstate->frame;
     int result = cf->cf_flags != 0;
 
     if (current_frame != NULL) {
-        const int codeflags = current_frame->f_frame->code->co_flags;
+        const int codeflags = current_frame->code->co_flags;
         const int compilerflags = codeflags & PyCF_MASK;
         if (compilerflags) {
             result = 1;

diff --git a/Python/errors.c b/Python/errors.c
@@ -1406,7 +1406,7 @@ _PyErr_WriteUnraisableMsg(const char *err_msg_str, PyObject *obj)
     }
 
     if (exc_tb == NULL) {
-        PyFrameObject *frame = tstate->pyframe;
+        PyFrameObject *frame = PyThreadState_GetFrame(tstate);
         if (frame != NULL) {
             exc_tb = _PyTraceBack_FromFrame(NULL, frame);
             if (exc_tb == NULL) {

diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
@@ -2003,7 +2003,7 @@ Py_EndInterpreter(PyThreadState *tstate)
     if (tstate != _PyThreadState_GET()) {
         Py_FatalError("thread is not current");
     }
-    if (tstate->pyframe != NULL) {
+    if (tstate->frame != NULL) {
         Py_FatalError("thread still has a frame");
     }
     interp->finalizing = 1;
-Original file line number
+Diff line change
@@ Expand Up / @@ -107,6 +107,7 @@ static inline void _Py_LeaveRecursiveCall_inline(void) { @@
     #define Py_LeaveRecursiveCall() _Py_LeaveRecursiveCall_inline()
+    struct _py_frame *_PyEval_GetFrame(void);
     #ifdef __cplusplus
     }
@@ Expand Down @@