python · ericsnowcurrently · Jan 12, 2022 · Nov 18, 2021 · Nov 18, 2021 · Nov 17, 2021
@@ -2,6 +2,9 @@
 #  error "this header file must not be included directly"
 #endif
 
+#include <stdbool.h>
+
+
 PyAPI_FUNC(int) _PyInterpreterState_RequiresIDRef(PyInterpreterState *);
 PyAPI_FUNC(void) _PyInterpreterState_RequireIDRef(PyInterpreterState *, int);
 
@@ -82,6 +85,7 @@ struct _ts {
        In order to be effective, this must be set to 0 during or right
        after allocation. */
     int _initialized;
+    bool _static;
 
     int recursion_remaining;
     int recursion_limit;
@@ -176,8 +180,24 @@ struct _ts {
     PyObject **datastack_limit;
     /* XXX signal handlers should also be here */
 
+    struct {
+        // See _PyRuntimeState._preallocated for an explanation.
+        // XXX Move exc_state and root_cframe down here.
+        // XXX Allocate the initial datastack_chunk here..
+        bool _dummy;  // We'll remove this as soon as we pre-allocate something.
+    } _preallocated;
 };
 
+#ifdef Py_BUILD_CORE
+#define _PyThreadState_INIT \
+    { \
+        ._static = 1, \
+    }
+#endif
+
+
+/* other API */
+
 // Alias for backward compatibility with Python 3.8
 #define _PyInterpreterState_Get PyInterpreterState_Get
 

@@ -42,7 +42,7 @@ extern "C" {
 // All others must be per-interpreter.
 
 #define _Py_GLOBAL_OBJECT(NAME) \
-    _PyRuntime.global_objects.NAME
+    _PyRuntime._preallocated.global_objects.NAME
 #define _Py_SINGLETON(NAME) \
     _Py_GLOBAL_OBJECT(singletons.NAME)
 
@@ -326,10 +326,6 @@ struct _Py_global_objects {
     }, \
 }
 
-static inline void
-_Py_global_objects_reset(struct _Py_global_objects *objects)
-{
-}
 
 #ifdef __cplusplus
 }

@@ -8,6 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include <stdbool.h>
+
 #include "pycore_atomic.h"        // _Py_atomic_address
 #include "pycore_ast_state.h"     // struct ast_state
 #include "pycore_bytesobject.h"   // struct _Py_bytes_state
@@ -104,6 +106,7 @@ struct _is {
        after allocation. */
     int _initialized;
     int finalizing;
+    bool _static;
 
     struct _ceval_state ceval;
     struct _gc_runtime_state gc;
@@ -168,8 +171,25 @@ struct _is {
 
     struct ast_state ast;
     struct type_cache type_cache;
+
+    struct {
+        // See _PyRuntimeState._preallocated for an explanation.
+        PyThreadState tstate;
+        // XXX Pre-allocate as many objects from above as possible here.
+    } _preallocated;
 };
 
+#define _PyInterpreterState_INIT \
+    { \
+        ._static = 1, \
+        ._preallocated = { \
+            .tstate = _PyThreadState_INIT, \
+        }, \
+    }
+
+
+/* other API */
+
 extern void _PyInterpreterState_ClearModules(PyInterpreterState *interp);
 extern void _PyInterpreterState_Clear(PyThreadState *tstate);
 

@@ -11,8 +11,10 @@ extern "C" {
 #include "pycore_atomic.h"          /* _Py_atomic_address */
 #include "pycore_gil.h"             // struct _gil_runtime_state
 #include "pycore_global_objects.h"  // struct _Py_global_objects
+#include "pycore_interp.h"          // struct _is
 #include "pycore_unicodeobject.h"   // struct _Py_unicode_runtime_ids
 
+
 /* ceval state */
 
 struct _ceval_runtime_state {
@@ -118,25 +120,32 @@ typedef struct pyruntimestate {
 
     struct _Py_unicode_runtime_ids unicode_ids;
 
-    struct _Py_global_objects global_objects;
-    // If anything gets added after global_objects then
-    // _PyRuntimeState_reset() needs to get updated to clear it.
+    struct {
+        // The fields here are values that would otherwise have been
+        // malloc'ed during runtime init in pystate.c and pylifecycle.c.
+        // This allows us to avoid allocation costs during startup and
+        // helps simplify the startup code.
+        struct _Py_global_objects global_objects;
+        struct _is interpreters_main;
+        // The only other values to possibly include here are
+        // mutexes (PyThread_type_lock).  Currently we don't pre-allocate them
+        // because on Windows we only get a pointer type.
+        // All other pointers in PyThreadState are either
+        // populated lazily or change but default to NULL.
+    } _preallocated;
 } _PyRuntimeState;
 
 #define _PyRuntimeState_INIT \
     { \
-        .global_objects = _Py_global_objects_INIT, \
+        ._preallocated = { \
+            .global_objects = _Py_global_objects_INIT, \
+            .interpreters_main = _PyInterpreterState_INIT, \
+        }, \
     }
 /* Note: _PyRuntimeState_INIT sets other fields to 0/NULL */
 
-static inline void
-_PyRuntimeState_reset(_PyRuntimeState *runtime)
-{
-    /* Make it match _PyRuntimeState_INIT. */
-    memset(runtime, 0, (size_t)&runtime->global_objects - (size_t)runtime);
-    _Py_global_objects_reset(&runtime->global_objects);
-}
 
+/* other API */
 
 PyAPI_DATA(_PyRuntimeState) _PyRuntime;
 

diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-12-01-11-54-27.bpo-45953.2znR0E.rst b/Misc/NEWS.d/next/Core and Builtins/2021-12-01-11-54-27.bpo-45953.2znR0E.rst
@@ -0,0 +1,4 @@
+The main interpreter in _PyRuntimeState.interpreters is now statically
+allocated (as part of _PyRuntime).  Likewise for the initial thread state of
+each interpreter.  This means less allocation during runtime init, as well
+as better memory locality for these key state objects.
@@ -46,6 +46,10 @@ static PyThreadState *_PyGILState_GetThisThreadState(struct _gilstate_runtime_st
 static void _PyThreadState_Delete(PyThreadState *tstate, int check_current);
 
 
+/* We use "initial" if the runtime gets re-used
+   (e.g. Py_Finalize() followed by Py_Initialize(). */
+static _PyRuntimeState initial = _PyRuntimeState_INIT;
+
 static int
 alloc_for_runtime(PyThread_type_lock *plock1, PyThread_type_lock *plock2,
                   PyThread_type_lock *plock3)
@@ -91,9 +95,12 @@ init_runtime(_PyRuntimeState *runtime,
              PyThread_type_lock xidregistry_mutex)
 {
     if (runtime->_initialized) {
-        _PyRuntimeState_reset(runtime);
-        assert(!runtime->initialized);
+        Py_FatalError("runtime already initialized");
     }
+    assert(!runtime->preinitializing &&
+           !runtime->preinitialized &&
+           !runtime->core_initialized &&
+           !runtime->initialized);
 
     runtime->open_code_hook = open_code_hook;
     runtime->open_code_userdata = open_code_userdata;
@@ -144,6 +151,11 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime)
         return _PyStatus_NO_MEMORY();
     }
 
+    if (runtime->_initialized) {
+        // Py_Initialize() must be running again.
+        // Reset to _PyRuntimeState_INIT.
+        memcpy(runtime, &initial, sizeof(*runtime));
+    }
     init_runtime(runtime, open_code_hook, open_code_userdata, audit_hook_head,
                  unicode_next_index, lock1, lock2, lock3);
 
@@ -250,13 +262,15 @@ alloc_interpreter(void)
 static void
 free_interpreter(PyInterpreterState *interp)
 {
-    PyMem_RawFree(interp);
+    if (!interp->_static) {
+        PyMem_RawFree(interp);
+    }
 }
 
 /* Get the interpreter state to a minimal consistent state.
    Further init happens in pylifecycle.c before it can be used.
    All fields not initialized here are expected to be zeroed out,
-   e.g. by PyMem_RawCalloc() or memset().
+   e.g. by PyMem_RawCalloc() or memset(), or otherwise pre-initialized.
    The runtime state is not manipulated.  Instead it is assumed that
    the interpreter is getting added to the runtime.
   */
@@ -338,10 +352,7 @@ PyInterpreterState_New(void)
         assert(interpreters->main == NULL);
         assert(id == 0);
 
-        interp = alloc_interpreter();
-        if (interp == NULL) {
-            goto error;
-        }
+        interp = &runtime->_preallocated.interpreters_main;
         assert(interp->id == 0);
         assert(interp->next == NULL);
 
@@ -355,6 +366,9 @@ PyInterpreterState_New(void)
         if (interp == NULL) {
             goto error;
         }
+        // Set to _PyInterpreterState_INIT.
+        memcpy(interp, &initial._preallocated.interpreters_main,
+               sizeof(*interp));
 
         if (id < 0) {
             /* overflow or Py_Initialize() not called yet! */
@@ -735,13 +749,15 @@ alloc_threadstate(void)
 static void
 free_threadstate(PyThreadState *tstate)
 {
-    PyMem_RawFree(tstate);
+    if (!tstate->_static) {
+        PyMem_RawFree(tstate);
+    }
 }
 
 /* Get the thread state to a minimal consistent state.
    Further init happens in pylifecycle.c before it can be used.
    All fields not initialized here are expected to be zeroed out,
-   e.g. by PyMem_RawCalloc() or memset().
+   e.g. by PyMem_RawCalloc() or memset(), or otherwise pre-initialized.
    The interpreter state is not manipulated.  Instead it is assumed that
    the thread is getting added to the interpreter.
   */
@@ -816,10 +832,7 @@ new_threadstate(PyInterpreterState *interp)
         // It's the interpreter's initial thread state.
         assert(id == 1);
 
-        tstate = alloc_threadstate();
-        if (tstate == NULL) {
-            goto error;
-        }
+        tstate = &interp->_preallocated.tstate;
     }
     else {
         // Every valid interpreter must have at least one thread.
@@ -830,6 +843,10 @@ new_threadstate(PyInterpreterState *interp)
         if (tstate == NULL) {
             goto error;
         }
+        // Set to _PyThreadState_INIT.
+        memcpy(tstate,
+               &initial._preallocated.interpreters_main._preallocated.tstate,
+               sizeof(*tstate));
     }
     interp->threads.head = tstate;
 
@@ -1170,7 +1187,7 @@ _PyThreadState_DeleteExcept(_PyRuntimeState *runtime, PyThreadState *tstate)
     for (p = list; p; p = next) {
         next = p->next;
         PyThreadState_Clear(p);
-        PyMem_RawFree(p);
+        free_threadstate(p);
     }
 }