python · ncoghlan · Mar 19, 2022 · Mar 19, 2022 · Mar 19, 2022 · Mar 19, 2022
diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst
@@ -954,13 +954,13 @@ Porting to Python 3.11
   * ``f_lineno``: use :c:func:`PyFrame_GetLineNumber`
   * ``f_locals``: use ``PyObject_GetAttrString((PyObject*)frame, "f_locals")``.
   * ``f_stackdepth``: removed.
-  * ``f_state``: no public API (renamed to ``f_frame.f_state``).
+  * ``f_state``: no public API.
   * ``f_trace``: no public API.
   * ``f_trace_lines``: use ``PyObject_GetAttrString((PyObject*)frame, "f_trace_lines")``
     (it also be modified).
   * ``f_trace_opcodes``: use ``PyObject_GetAttrString((PyObject*)frame, "f_trace_opcodes")``
     (it also be modified).
-  * ``f_localsplus``: no public API (renamed to ``f_frame.localsplus``).
+  * ``f_localsplus``: no public API.
   * ``f_valuestack``: removed.
 
   The Python frame object is now created lazily. A side effect is that the

diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h
@@ -15,7 +15,7 @@ PyAPI_FUNC(PyObject *) _PyEval_GetBuiltinId(_Py_Identifier *);
    flag was set, else return 0. */
 PyAPI_FUNC(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf);
 
-PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc);
+PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _Py_frame *f, int exc);
 
 PyAPI_FUNC(void) _PyEval_SetSwitchInterval(unsigned long microseconds);
 PyAPI_FUNC(unsigned long) _PyEval_GetSwitchInterval(void);

@@ -48,7 +48,7 @@ typedef struct _PyCFrame {
      */
     int use_tracing;
     /* Pointer to the currently executing frame (it can be NULL) */
-    struct _PyInterpreterFrame *current_frame;
+    struct _Py_frame *current_frame;
     struct _PyCFrame *previous;
 } _PyCFrame;
 
@@ -260,7 +260,7 @@ PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void);
 
 /* Frame evaluation API */
 
-typedef PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, struct _PyInterpreterFrame *, int);
+typedef PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, struct _Py_frame *, int);
 
 PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc(
     PyInterpreterState *interp);

diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
@@ -60,7 +60,7 @@ extern PyObject* _PyEval_BuiltinsFromGlobals(
 
 
 static inline PyObject*
-_PyEval_EvalFrame(PyThreadState *tstate, struct _PyInterpreterFrame *frame, int throwflag)
+_PyEval_EvalFrame(PyThreadState *tstate, struct _Py_frame *frame, int throwflag)
 {
     if (tstate->interp->eval_frame == NULL) {
         return _PyEval_EvalFrameDefault(tstate, frame, throwflag);
@@ -129,7 +129,7 @@ static inline void _Py_LeaveRecursiveCall_inline(void)  {
 
 #define Py_LeaveRecursiveCall() _Py_LeaveRecursiveCall_inline()
 
-extern struct _PyInterpreterFrame* _PyEval_GetFrame(void);
+extern struct _Py_frame* _PyEval_GetFrame(void);
 
 extern PyObject* _Py_MakeCoro(PyFunctionObject *func);
 

diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h
@@ -6,10 +6,83 @@ extern "C" {
 
 #include <stdbool.h>
 
+/* Starting in CPython 3.11, CPython separates the frame state between the
+ * full frame objects exposed by the Python and C runtime state introspection
+ * APIs, and internal lighter weight frame data structs, which are simple C
+ * structures owned by either the interpreter eval loop (while executing
+ * ordinary functions), by a generator or coroutine object (for frames that
+ * are able to be suspended), or by their corresponding full frame object (if
+ * a state instrospection API has been invoked and the full frame object has
+ * taken responsibility for the lifecycle of the frame data storage).
+ *
+ * This split storage eliminates a lot of allocation and deallocation of full
+ * Python objects during code execution, providing a significant speed gain
+ * over the previous approach of using full Python objects for both
+ * introspection and code execution.
+ *
+ * Struct names:
+ *
+ *   * PyFrameObject: the full Python frame object
+ *   * _Py_frame: the lightweight frame data struct
+ *     * _PyInterpreterFrame is an interim alias for this (see comment below)
+ *   * _PyCFrame: a struct that lives on the C stack and allows Python level
+ *     recursive evaluation to be decoupled from recursive C level invocation
+ *     of the bytecode eval loop
+ *       * See pystate.h for more details on this struct
+ *
+ * Field naming conventions:
+ *
+ *   * full frame object fields have an "f_*" prefix
+ *   * new frame data struct fields have no prefix
+ *   * Several frame data struct fields have the "f_*" prefix as a result of
+ *     trying to keep diffs as small as was feasible when splitting the original
+ *     frame struct definition in two. The following are all frame data struct
+ *     fields, NOT full frame object fields:
+ *     * f_func
+ *     * f_globals
+ *     * f_builtins
+ *     * f_locals
+ *     * f_code
+ *     * f_lasti
+ *     * f_state
+ *   * Renaming those fields was considered but ultimately deemed too disruptive
+ *     to key third party projects that were trying to keep up with the Python
+ *     3.11 code evaluation changes during the alpha release cycle
+ *     (see bpo-44800 for details)
+ *
+ * Naming conventions for local variables, function parameters and fields in other structs:
+ *
+ *   * "frame", and "f" may refer to either full frame objects or frame data structs
+ *     * the field naming convention usually makes the type unambiguous in code reviews
+ *   * the following alternative names are used when more clarity is needed:
+ *     * full frame objects: "frame_obj" (and variants like "frameobj" or "fobj")
+ *     * frame data structs: "fdata"
+ *   * the "iframe" name is still used in the generator & coroutine structs. It
+ *     comes from the period where frame data structs were called "interpreter frames"
+ *     (which implied a larger distinction between full frame objects and their
+ *     associated lightweight frame data structs than is actually the case).
+ *   * "current frame" should NOT be abbreviated as "cframe", as the latter now
+ *     typically refers to _PyCFrame structs
+ *
+ * Function/macro parameter types:
+ *
+ *   * "PyFrame_*" functions and other public C API functions that relate to
+ *     frames accept full frame objects
+ *   * "_PyFrame_*" functions and other private C API functions that relate to
+ *     frames accept either full frame objects or frame data structs. Check
+ *     the specific function signatures for details.
+ *
+ * Function return types:
+ *
+ *   * Public C API functions will only ever return full frame objects
+ *   * Private C API functions with an underscore prefix may return frame
+ *     data structs instead. Check the specific function signatures for details.
+ */
+
 struct _frame {
     PyObject_HEAD
     PyFrameObject *f_back;      /* previous frame, or NULL */
-    struct _PyInterpreterFrame *f_frame; /* points to the frame data */
+    struct _Py_frame *f_frame; /* points to the frame data */
     PyObject *f_trace;          /* Trace function */
     int f_lineno;               /* Current line number. Only valid if non-zero */
     char f_trace_lines;         /* Emit per-line trace events? */
@@ -44,63 +117,78 @@ typedef signed char PyFrameState;
     unless it's -1 in which case next_instr should be first_instr.
 */
 
-typedef struct _PyInterpreterFrame {
+typedef struct _Py_frame {
     PyFunctionObject *f_func; /* Strong reference */
     PyObject *f_globals; /* Borrowed reference */
     PyObject *f_builtins; /* Borrowed reference */
     PyObject *f_locals; /* Strong reference, may be NULL */
     PyCodeObject *f_code; /* Strong reference */
     PyFrameObject *frame_obj; /* Strong reference, may be NULL */
-    struct _PyInterpreterFrame *previous;
+    struct _Py_frame *previous;
     int f_lasti;       /* Last instruction if called */
     int stacktop;     /* Offset of TOS from localsplus  */
     PyFrameState f_state;  /* What state the frame is in */
     bool is_entry;  // Whether this is the "root" frame for the current _PyCFrame.
     bool is_generator;
     PyObject *localsplus[1];
-} _PyInterpreterFrame;
+} _Py_frame;
+
+/* Interim compatibility for the internal frame API as shipped in 3.11a6
+ * Some projects (Cython, gevent, greenlet) are known to access the private
+ * frame API in CPython. This API has already changed twice for 3.11 (first
+ * with the structural split, then with the full object structure becoming
+ * opaque and internal struct gaining the `_Py` prefix). The recommended
+ * resolution is expected to change again once a proper public API for the
+ * required frame operations is defined (as discussed in
+ * https://github.com/faster-cpython/ideas/issues/309).
+ *
+ * This interim compatibility workaround enables the bpo-44800 struct name
+ * change for CPython maintainability without forcing yet another interim
+ * code update on the affected projects.
+ */
+typedef _Py_frame _PyInterpreterFrame;
 
-static inline int _PyFrame_IsRunnable(_PyInterpreterFrame *f) {
+static inline int _PyFrame_IsRunnable(_Py_frame *f) {
     return f->f_state < FRAME_EXECUTING;
 }
 
-static inline int _PyFrame_IsExecuting(_PyInterpreterFrame *f) {
+static inline int _PyFrame_IsExecuting(_Py_frame *f) {
     return f->f_state == FRAME_EXECUTING;
 }
 
-static inline int _PyFrameHasCompleted(_PyInterpreterFrame *f) {
+static inline int _PyFrameHasCompleted(_Py_frame *f) {
     return f->f_state > FRAME_EXECUTING;
 }
 
-static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f) {
+static inline PyObject **_PyFrame_Stackbase(_Py_frame *f) {
     return f->localsplus + f->f_code->co_nlocalsplus;
 }
 
-static inline PyObject *_PyFrame_StackPeek(_PyInterpreterFrame *f) {
+static inline PyObject *_PyFrame_StackPeek(_Py_frame *f) {
     assert(f->stacktop > f->f_code->co_nlocalsplus);
     assert(f->localsplus[f->stacktop-1] != NULL);
     return f->localsplus[f->stacktop-1];
 }
 
-static inline PyObject *_PyFrame_StackPop(_PyInterpreterFrame *f) {
+static inline PyObject *_PyFrame_StackPop(_Py_frame *f) {
     assert(f->stacktop > f->f_code->co_nlocalsplus);
     f->stacktop--;
     return f->localsplus[f->stacktop];
 }
 
-static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value) {
+static inline void _PyFrame_StackPush(_Py_frame *f, PyObject *value) {
     f->localsplus[f->stacktop] = value;
     f->stacktop++;
 }
 
-#define FRAME_SPECIALS_SIZE ((sizeof(_PyInterpreterFrame)-1)/sizeof(PyObject *))
+#define FRAME_SPECIALS_SIZE ((sizeof(_Py_frame)-1)/sizeof(PyObject *))
 
-void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest);
+void _PyFrame_Copy(_Py_frame *src, _Py_frame *dest);
 
 /* Consumes reference to func */
 static inline void
 _PyFrame_InitializeSpecials(
-    _PyInterpreterFrame *frame, PyFunctionObject *func,
+    _Py_frame *frame, PyFunctionObject *func,
     PyObject *locals, int nlocalsplus)
 {
     frame->f_func = func;
@@ -120,33 +208,33 @@ _PyFrame_InitializeSpecials(
  * that precedes this frame.
  */
 static inline PyObject**
-_PyFrame_GetLocalsArray(_PyInterpreterFrame *frame)
+_PyFrame_GetLocalsArray(_Py_frame *frame)
 {
     return frame->localsplus;
 }
 
 static inline PyObject**
-_PyFrame_GetStackPointer(_PyInterpreterFrame *frame)
+_PyFrame_GetStackPointer(_Py_frame *frame)
 {
     return frame->localsplus+frame->stacktop;
 }
 
 static inline void
-_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer)
+_PyFrame_SetStackPointer(_Py_frame *frame, PyObject **stack_pointer)
 {
     frame->stacktop = (int)(stack_pointer - frame->localsplus);
 }
 
 /* For use by _PyFrame_GetFrameObject
   Do not call directly. */
 PyFrameObject *
-_PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame);
+_PyFrame_MakeAndSetFrameObject(_Py_frame *frame);
 
 /* Gets the PyFrameObject for this frame, lazily
  * creating it if necessary.
  * Returns a borrowed referennce */
 static inline PyFrameObject *
-_PyFrame_GetFrameObject(_PyInterpreterFrame *frame)
+_PyFrame_GetFrameObject(_Py_frame *frame)
 {
     PyFrameObject *res =  frame->frame_obj;
     if (res != NULL) {
@@ -156,7 +244,7 @@ _PyFrame_GetFrameObject(_PyInterpreterFrame *frame)
 }
 
 /* Clears all references in the frame.
- * If take is non-zero, then the _PyInterpreterFrame frame
+ * If take is non-zero, then the _Py_frame frame
  * may be transferred to the frame object it references
  * instead of being cleared. Either way
  * the caller no longer owns the references
@@ -165,21 +253,21 @@ _PyFrame_GetFrameObject(_PyInterpreterFrame *frame)
  * frames like the ones in generators and coroutines.
  */
 void
-_PyFrame_Clear(_PyInterpreterFrame * frame);
+_PyFrame_Clear(_Py_frame * frame);
 
 int
-_PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg);
+_PyFrame_Traverse(_Py_frame *frame, visitproc visit, void *arg);
 
 int
-_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame);
+_PyFrame_FastToLocalsWithError(_Py_frame *frame);
 
 void
-_PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear);
+_PyFrame_LocalsToFast(_Py_frame *frame, int clear);
 
-extern _PyInterpreterFrame *
+extern _Py_frame *
 _PyThreadState_BumpFramePointerSlow(PyThreadState *tstate, size_t size);
 
-static inline _PyInterpreterFrame *
+static inline _Py_frame *
 _PyThreadState_BumpFramePointer(PyThreadState *tstate, size_t size)
 {
     PyObject **base = tstate->datastack_top;
@@ -188,16 +276,16 @@ _PyThreadState_BumpFramePointer(PyThreadState *tstate, size_t size)
         assert(tstate->datastack_limit);
         if (top < tstate->datastack_limit) {
             tstate->datastack_top = top;
-            return (_PyInterpreterFrame *)base;
+            return (_Py_frame *)base;
         }
     }
     return _PyThreadState_BumpFramePointerSlow(tstate, size);
 }
 
-void _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame);
+void _PyThreadState_PopFrame(PyThreadState *tstate, _Py_frame *frame);
 
 /* Consume reference to func */
-_PyInterpreterFrame *
+_Py_frame *
 _PyFrame_Push(PyThreadState *tstate, PyFunctionObject *func);
 
 #ifdef __cplusplus

diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-03-19-21-47-36.bpo-44800.XnGKoD.rst b/Misc/NEWS.d/next/Core and Builtins/2022-03-19-21-47-36.bpo-44800.XnGKoD.rst
@@ -0,0 +1,5 @@
+Renamed ``_PyInterpreterFrame`` as ``_Py_frame`` to emphasise its close
+association with ``PyFrameObject`` (they're conceptually the same thing, but
+split into a Python object struct and a C data struct as a performance
+optimisation). ``_PyInterpreterFrame`` is retained as a compatibility alias
+to avoid any disruption to the Python 3.11 performance improvement work.
diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c
@@ -305,7 +305,7 @@ hashtable_compare_traceback(const void *key1, const void *key2)
 
 
 static void
-tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame)
+tracemalloc_get_frame(_Py_frame *pyframe, frame_t *frame)
 {
     frame->filename = &_Py_STR(anon_unknown);
     int lineno = PyCode_Addr2Line(pyframe->f_code, pyframe->f_lasti*sizeof(_Py_CODEUNIT));
@@ -399,7 +399,7 @@ traceback_get_frames(traceback_t *traceback)
         return;
     }
 
-    _PyInterpreterFrame *pyframe = tstate->cframe->current_frame;
+    _Py_frame *pyframe = tstate->cframe->current_frame;
     for (; pyframe != NULL;) {
         if (traceback->nframe < _Py_tracemalloc_config.max_nframe) {
             tracemalloc_get_frame(pyframe, &traceback->frames[traceback->nframe]);
@@ -410,7 +410,7 @@ traceback_get_frames(traceback_t *traceback)
             traceback->total_nframe++;
         }
 
-        _PyInterpreterFrame *back = pyframe->previous;
+        _Py_frame *back = pyframe->previous;
         pyframe = back;
     }
 }

diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c
@@ -1839,7 +1839,7 @@ _is_running(PyInterpreterState *interp)
     }
 
     assert(!PyErr_Occurred());
-    _PyInterpreterFrame *frame = tstate->cframe->current_frame;
+    _Py_frame *frame = tstate->cframe->current_frame;
     if (frame == NULL) {
         return 0;
     }

diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c
@@ -8,7 +8,7 @@
 #include "pycore_call.h"          // _PyObject_Call()
 #include "pycore_ceval.h"         // _PyEval_SignalReceived()
 #include "pycore_fileutils.h"     // _Py_BEGIN_SUPPRESS_IPH
-#include "pycore_frame.h"         // _PyInterpreterFrame
+#include "pycore_frame.h"         // _Py_frame
 #include "pycore_moduleobject.h"  // _PyModule_GetState()
 #include "pycore_pyerrors.h"      // _PyErr_SetString()
 #include "pycore_pylifecycle.h"   // NSIG
@@ -1817,7 +1817,7 @@ _PyErr_CheckSignalsTstate(PyThreadState *tstate)
      */
     _Py_atomic_store(&is_tripped, 0);
 
-    _PyInterpreterFrame *frame = tstate->cframe->current_frame;
+    _Py_frame *frame = tstate->cframe->current_frame;
     signal_state_t *state = &signal_global_state;
     for (int i = 1; i < NSIG; i++) {
         if (!_Py_atomic_load_relaxed(&Handlers[i].tripped)) {