bpo-42923: Dump extension modules on fatal error

The Py_FatalError() function and the faulthandler module now dump the list of extension modules on a fatal error. Add _Py_DumpExtensionModules() and _PyModule_IsExtension() internal functions.
python · vstinner · Jan 18, 2021 · Jan 13, 2021 · Jan 18, 2021 · 77396298fe2bd667b685be65ed20b6ea97433bd8
commit 77396298fe2bd667b685be65ed20b6ea97433bd8
diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h
@@ -84,6 +84,8 @@ PyAPI_FUNC(PyObject *) _PyErr_FormatFromCauseTstate(
 
 PyAPI_FUNC(int) _PyErr_CheckSignalsTstate(PyThreadState *tstate);
 
+PyAPI_FUNC(void) _Py_DumpExtensionModules(int fd, PyInterpreterState *interp);
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/Include/moduleobject.h b/Include/moduleobject.h
@@ -84,6 +84,12 @@ typedef struct PyModuleDef{
   freefunc m_free;
 } PyModuleDef;
 
+
+// Internal C API
+#ifdef Py_BUILD_CORE
+extern int _PyModule_IsExtension(PyObject *obj);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
@@ -556,6 +556,16 @@ def test_fatal_error(self):
         self.assertIn('Fatal Python error: test_fatal_error: MESSAGE\n',
                       err)
 
+        match = re.search('^Extension modules:(.*)$', err, re.MULTILINE)
+        if not match:
+            self.fail(f"Cannot find 'Extension modules:' in {err!r}")
+        modules = set(match.group(1).strip().split(', '))
+        # Test _PyModule_IsExtension(): the list doesn't have to
+        # be exhaustive.
+        for name in ('sys', 'builtins', '_imp', '_thread', '_weakref',
+                     '_io', 'marshal', '_signal', '_abc', '_testcapi'):
+            self.assertIn(name, modules)
+
 
 class TestPendingCalls(unittest.TestCase):
 

diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py
@@ -2,6 +2,7 @@
 import datetime
 import faulthandler
 import os
+import re
 import signal
 import subprocess
 import sys
@@ -329,6 +330,24 @@ def test_disable(self):
                      "%r is present in %r" % (not_expected, stderr))
         self.assertNotEqual(exitcode, 0)
 
+    @skip_segfault_on_android
+    def test_dump_ext_modules(self):
+        code = """
+            import faulthandler
+            faulthandler.enable()
+            faulthandler._sigsegv()
+            """
+        stderr, exitcode = self.get_output(code)
+        stderr = '\n'.join(stderr)
+        match = re.search('^Extension modules:(.*)$', stderr, re.MULTILINE)
+        if not match:
+            self.fail(f"Cannot find 'Extension modules:' in {stderr!r}")
+        modules = set(match.group(1).strip().split(', '))
+        # Only check for a few extensions, the list doesn't have to be
+        # exhaustive.
+        for ext in ('sys', 'builtins', '_io', 'faulthandler'):
+            self.assertIn(ext, modules)
+
     def test_is_enabled(self):
         orig_stderr = sys.stderr
         try:

diff --git a/Misc/NEWS.d/next/Library/2021-01-13-12-15-13.bpo-42923.zBiNls.rst b/Misc/NEWS.d/next/Library/2021-01-13-12-15-13.bpo-42923.zBiNls.rst
@@ -0,0 +1,2 @@
+The :c:func:`Py_FatalError` function and the :mod:`faulthandler` module now
+dump the list of extension modules on a fatal error.
diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c
@@ -1,5 +1,6 @@
 #include "Python.h"
 #include "pycore_initconfig.h"    // _PyStatus_ERR
+#include "pycore_pyerrors.h"      // _Py_DumpExtensionModules
 #include "pycore_traceback.h"     // _Py_DumpTracebackThreads
 #include <signal.h>
 #include <object.h>
@@ -349,6 +350,8 @@ faulthandler_fatal_error(int signum)
     faulthandler_dump_traceback(fd, fatal_error.all_threads,
                                 fatal_error.interp);
 
+    _Py_DumpExtensionModules(fd, fatal_error.interp);
+
     errno = save_errno;
 #ifdef MS_WINDOWS
     if (signum == SIGSEGV) {

diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c
@@ -35,6 +35,19 @@ PyTypeObject PyModuleDef_Type = {
 };
 
 
+int
+_PyModule_IsExtension(PyObject *obj)
+{
+    if (!PyModule_Check(obj)) {
+        return 0;
+    }
+    PyModuleObject *module = (PyModuleObject*)obj;
+
+    struct PyModuleDef *def = module->md_def;
+    return (def != NULL && def->m_methods != NULL);
+}
+
+
 PyObject*
 PyModuleDef_Init(struct PyModuleDef* def)
 {

diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
@@ -2496,6 +2496,45 @@ fatal_error_exit(int status)
 }
 
 
+// Dump the list of extension modules of sys.modules into fd file descriptor.
+// This function is called by a signal handler in faulthandler: avoid memory
+// allocations and keep the implementation simple. For example, the list
+// is not sorted on purpose.
+void
+_Py_DumpExtensionModules(int fd, PyInterpreterState *interp)
+{
+    if (interp == NULL) {
+        return;
+    }
+    PyObject *modules = interp->modules;
+    if (!PyDict_Check(modules)) {
+        return;
+    }
+
+    PUTS(fd, "\nExtension modules: ");
+
+    Py_ssize_t pos = 0;
+    PyObject *key, *value;
+    int comma = 0;
+    while (PyDict_Next(modules, &pos, &key, &value)) {
+        if (!PyUnicode_Check(key)) {
+            continue;
+        }
+        if (!_PyModule_IsExtension(value)) {
+            continue;
+        }
+
+        if (comma) {
+            PUTS(fd, ", ");
+        }
+        comma = 1;
+
+        _Py_DumpASCII(fd, key);
+    }
+    PUTS(fd, "\n");
+}
+
+
 static void _Py_NO_RETURN
 fatal_error(int fd, int header, const char *prefix, const char *msg,
             int status)
@@ -2557,6 +2596,8 @@ fatal_error(int fd, int header, const char *prefix, const char *msg,
         _Py_FatalError_DumpTracebacks(fd, interp, tss_tstate);
     }
 
+    _Py_DumpExtensionModules(fd, interp);
+
     /* The main purpose of faulthandler is to display the traceback.
        This function already did its best to display a traceback.
        Disable faulthandler to prevent writing a second traceback
-Original file line number
+Diff line change
@@ Expand Up / @@ -84,6 +84,8 @@ PyAPI_FUNC(PyObject *) _PyErr_FormatFromCauseTstate( @@
     PyAPI_FUNC(int) _PyErr_CheckSignalsTstate(PyThreadState *tstate);
+    PyAPI_FUNC(void) _Py_DumpExtensionModules(int fd, PyInterpreterState *interp);
     #ifdef __cplusplus
     }
     #endif
@@ Expand Down @@
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		The :c:func:`Py_FatalError` function and the :mod:`faulthandler` module now
		dump the list of extension modules on a fatal error.