From bac3fcba5b2d83aa294267a456ccc36d86151dd4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 28 May 2025 20:11:09 +0300 Subject: [PATCH 001/116] gh-108512: Add and use new replacements for PySys_GetObject() (GH-111035) Add functions PySys_GetAttr(), PySys_GetAttrString(), PySys_GetOptionalAttr() and PySys_GetOptionalAttrString(). --- Doc/c-api/init_config.rst | 2 +- Doc/c-api/sys.rst | 51 ++++++++++++- Doc/data/stable_abi.dat | 4 + Doc/whatsnew/3.15.rst | 5 +- Include/internal/pycore_sysmodule.h | 5 -- Include/sysmodule.h | 6 ++ Lib/test/test_capi/test_sys.py | 64 +++++++++++++++- Lib/test/test_stable_abi_ctypes.py | 4 + ...-10-18-14-36-35.gh-issue-108512.fMZLfr.rst | 2 + Misc/stable_abi.toml | 8 ++ Modules/_cursesmodule.c | 3 +- Modules/_lsprof.c | 6 +- Modules/_pickle.c | 2 +- Modules/_testlimitedcapi/sys.c | 73 +++++++++++++++++++ Modules/_threadmodule.c | 3 +- Modules/_tkinter.c | 5 +- Modules/faulthandler.c | 3 +- Modules/main.c | 13 ++-- Modules/syslogmodule.c | 3 +- Objects/moduleobject.c | 3 +- PC/python3dll.c | 4 + Python/_warnings.c | 3 +- Python/bltinmodule.c | 11 ++- Python/ceval.c | 2 +- Python/errors.c | 5 +- Python/import.c | 12 +-- Python/initconfig.c | 8 +- Python/intrinsics.c | 3 +- Python/pylifecycle.c | 8 +- Python/pythonrun.c | 18 ++--- Python/sysmodule.c | 36 ++++----- Python/traceback.c | 5 +- 32 files changed, 287 insertions(+), 93 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2023-10-18-14-36-35.gh-issue-108512.fMZLfr.rst diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index e1931655618b1c..4fd10224262488 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -2111,7 +2111,7 @@ initialization:: /* Specify sys.path explicitly */ /* If you want to modify the default set of paths, finish - initialization first and then use PySys_GetObject("path") */ + initialization first and then use PySys_GetAttrString("path") */ config.module_search_paths_set = 1; status = PyWideStringList_Append(&config.module_search_paths, L"/path/to/stdlib"); diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index b3c89800e386ff..b34936dd55e94c 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -258,10 +258,57 @@ These are utility functions that make functionality from the :mod:`sys` module accessible to C code. They all work with the current interpreter thread's :mod:`sys` module's dict, which is contained in the internal thread state structure. +.. c:function:: PyObject *PySys_GetAttr(PyObject *name) + + Get the attribute *name* of the :mod:`sys` module. + Return a :term:`strong reference`. + Raise :exc:`RuntimeError` and return ``NULL`` if it does not exist or + if the :mod:`sys` module cannot be found. + + If the non-existing object should not be treated as a failure, you can use + :c:func:`PySys_GetOptionalAttr` instead. + + .. versionadded:: next + +.. c:function:: PyObject *PySys_GetAttrString(const char *name) + + This is the same as :c:func:`PySys_GetAttr`, but *name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + If the non-existing object should not be treated as a failure, you can use + :c:func:`PySys_GetOptionalAttrString` instead. + + .. versionadded:: next + +.. c:function:: int PySys_GetOptionalAttr(PyObject *name, PyObject **result) + + Variant of :c:func:`PySys_GetAttr` which doesn't raise + exception if the object does not exist. + + * Set *\*result* to a new :term:`strong reference` to the object and + return ``1`` if the object exists. + * Set *\*result* to ``NULL`` and return ``0`` without setting an exception + if the object does not exist. + * Set an exception, set *\*result* to ``NULL``, and return ``-1``, + if an error occurred. + + .. versionadded:: next + +.. c:function:: int PySys_GetOptionalAttrString(const char *name, PyObject **result) + + This is the same as :c:func:`PySys_GetOptionalAttr`, but *name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. versionadded:: next + .. c:function:: PyObject *PySys_GetObject(const char *name) - Return the object *name* from the :mod:`sys` module or ``NULL`` if it does - not exist, without setting an exception. + Similar to :c:func:`PySys_GetAttrString`, but return a :term:`borrowed + reference` and return ``NULL`` *without* setting exception on failure. + + Preserves exception that was set before the call. .. c:function:: int PySys_SetObject(const char *name, PyObject *v) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index e71a40e55e918c..0d0dfb3843260e 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -628,7 +628,11 @@ func,PySys_Audit,3.13,, func,PySys_AuditTuple,3.13,, func,PySys_FormatStderr,3.2,, func,PySys_FormatStdout,3.2,, +func,PySys_GetAttr,3.15,, +func,PySys_GetAttrString,3.15,, func,PySys_GetObject,3.2,, +func,PySys_GetOptionalAttr,3.15,, +func,PySys_GetOptionalAttrString,3.15,, func,PySys_GetXOptions,3.7,, func,PySys_ResetWarnOptions,3.2,, func,PySys_SetArgv,3.2,, diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 4d1a27354fc620..87cca4eeff385a 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -213,7 +213,10 @@ C API changes New features ------------ -* TODO +* Add :c:func:`PySys_GetAttr`, :c:func:`PySys_GetAttrString`, + :c:func:`PySys_GetOptionalAttr`, and :c:func:`PySys_GetOptionalAttrString` + functions as replacements for :c:func:`PySys_GetObject`. + (Contributed by Serhiy Storchaka in :gh:`108512`.) Porting to Python 3.15 ---------------------- diff --git a/Include/internal/pycore_sysmodule.h b/Include/internal/pycore_sysmodule.h index 008a2da0d04fa7..347b0a7a790c06 100644 --- a/Include/internal/pycore_sysmodule.h +++ b/Include/internal/pycore_sysmodule.h @@ -8,11 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -PyAPI_FUNC(int) _PySys_GetOptionalAttr(PyObject *, PyObject **); -PyAPI_FUNC(int) _PySys_GetOptionalAttrString(const char *, PyObject **); -PyAPI_FUNC(PyObject *) _PySys_GetRequiredAttr(PyObject *); -PyAPI_FUNC(PyObject *) _PySys_GetRequiredAttrString(const char *); - // Export for '_pickle' shared extension PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *); diff --git a/Include/sysmodule.h b/Include/sysmodule.h index c1d5f610fe08a5..2f362791797ded 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -4,6 +4,12 @@ extern "C" { #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030f0000 +PyAPI_FUNC(PyObject *) PySys_GetAttr(PyObject *); +PyAPI_FUNC(PyObject *) PySys_GetAttrString(const char *); +PyAPI_FUNC(int) PySys_GetOptionalAttr(PyObject *, PyObject **); +PyAPI_FUNC(int) PySys_GetOptionalAttrString(const char *, PyObject **); +#endif PyAPI_FUNC(PyObject *) PySys_GetObject(const char *); PyAPI_FUNC(int) PySys_SetObject(const char *, PyObject *); diff --git a/Lib/test/test_capi/test_sys.py b/Lib/test/test_capi/test_sys.py index d3a9b378e7769a..3793ce2461effd 100644 --- a/Lib/test/test_capi/test_sys.py +++ b/Lib/test/test_capi/test_sys.py @@ -19,6 +19,68 @@ class CAPITest(unittest.TestCase): maxDiff = None + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_sys_getattr(self): + # Test PySys_GetAttr() + sys_getattr = _testlimitedcapi.sys_getattr + + self.assertIs(sys_getattr('stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(sys_getattr('\U0001f40d'), 42) + + with self.assertRaisesRegex(RuntimeError, r'lost sys\.nonexistent'): + sys_getattr('nonexistent') + with self.assertRaisesRegex(RuntimeError, r'lost sys\.\U0001f40d'): + sys_getattr('\U0001f40d') + self.assertRaises(TypeError, sys_getattr, 1) + self.assertRaises(TypeError, sys_getattr, []) + # CRASHES sys_getattr(NULL) + + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_sys_getattrstring(self): + # Test PySys_GetAttrString() + getattrstring = _testlimitedcapi.sys_getattrstring + + self.assertIs(getattrstring(b'stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(getattrstring('\U0001f40d'.encode()), 42) + + with self.assertRaisesRegex(RuntimeError, r'lost sys\.nonexistent'): + getattrstring(b'nonexistent') + with self.assertRaisesRegex(RuntimeError, r'lost sys\.\U0001f40d'): + getattrstring('\U0001f40d'.encode()) + self.assertRaises(UnicodeDecodeError, getattrstring, b'\xff') + # CRASHES getattrstring(NULL) + + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_sys_getoptionalattr(self): + # Test PySys_GetOptionalAttr() + getoptionalattr = _testlimitedcapi.sys_getoptionalattr + + self.assertIs(getoptionalattr('stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(getoptionalattr('\U0001f40d'), 42) + + self.assertIs(getoptionalattr('nonexistent'), AttributeError) + self.assertIs(getoptionalattr('\U0001f40d'), AttributeError) + self.assertRaises(TypeError, getoptionalattr, 1) + self.assertRaises(TypeError, getoptionalattr, []) + # CRASHES getoptionalattr(NULL) + + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_sys_getoptionalattrstring(self): + # Test PySys_GetOptionalAttrString() + getoptionalattrstring = _testlimitedcapi.sys_getoptionalattrstring + + self.assertIs(getoptionalattrstring(b'stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(getoptionalattrstring('\U0001f40d'.encode()), 42) + + self.assertIs(getoptionalattrstring(b'nonexistent'), AttributeError) + self.assertIs(getoptionalattrstring('\U0001f40d'.encode()), AttributeError) + self.assertRaises(UnicodeDecodeError, getoptionalattrstring, b'\xff') + # CRASHES getoptionalattrstring(NULL) + @support.cpython_only @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_sys_getobject(self): @@ -29,7 +91,7 @@ def test_sys_getobject(self): with support.swap_attr(sys, '\U0001f40d', 42): self.assertEqual(getobject('\U0001f40d'.encode()), 42) - self.assertIs(getobject(b'nonexisting'), AttributeError) + self.assertIs(getobject(b'nonexistent'), AttributeError) with support.catch_unraisable_exception() as cm: self.assertIs(getobject(b'\xff'), AttributeError) self.assertEqual(cm.unraisable.exc_type, UnicodeDecodeError) diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 1e6f69d49e9335..5a6ba9de337904 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -658,7 +658,11 @@ def test_windows_feature_macros(self): "PySys_AuditTuple", "PySys_FormatStderr", "PySys_FormatStdout", + "PySys_GetAttr", + "PySys_GetAttrString", "PySys_GetObject", + "PySys_GetOptionalAttr", + "PySys_GetOptionalAttrString", "PySys_GetXOptions", "PySys_HasWarnOptions", "PySys_ResetWarnOptions", diff --git a/Misc/NEWS.d/next/C_API/2023-10-18-14-36-35.gh-issue-108512.fMZLfr.rst b/Misc/NEWS.d/next/C_API/2023-10-18-14-36-35.gh-issue-108512.fMZLfr.rst new file mode 100644 index 00000000000000..279e588f3adcb7 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2023-10-18-14-36-35.gh-issue-108512.fMZLfr.rst @@ -0,0 +1,2 @@ +Add functions :c:func:`PySys_GetAttr`, :c:func:`PySys_GetAttrString`, +:c:func:`PySys_GetOptionalAttr` and :c:func:`PySys_GetOptionalAttrString`. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index d38919a8ea9b62..1f323cc03973e5 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2575,3 +2575,11 @@ added = '3.14' [function.Py_PACK_VERSION] added = '3.14' +[function.PySys_GetAttr] + added = '3.15' +[function.PySys_GetAttrString] + added = '3.15' +[function.PySys_GetOptionalAttr] + added = '3.15' +[function.PySys_GetOptionalAttrString] + added = '3.15' diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 290ae4e55cd7a7..d7788ef7a585b0 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -108,7 +108,6 @@ static const char PyCursesVersion[] = "2.2"; #include "pycore_capsule.h" // _PyCapsule_SetTraverse() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_structseq.h" // _PyStructSequence_NewType() -#include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString() #include "pycore_fileutils.h" // _Py_set_inheritable #ifdef __hpux @@ -3847,7 +3846,7 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) if (fd == -1) { PyObject* sys_stdout; - if (_PySys_GetOptionalAttrString("stdout", &sys_stdout) < 0) { + if (PySys_GetOptionalAttrString("stdout", &sys_stdout) < 0) { return NULL; } diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 626c176715bdac..bbad5eb69032da 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -782,7 +782,7 @@ _lsprof_Profiler_enable_impl(ProfilerObject *self, int subcalls, return NULL; } - PyObject* monitoring = PyImport_ImportModuleAttrString("sys", "monitoring"); + PyObject* monitoring = PySys_GetAttrString("monitoring"); if (!monitoring) { return NULL; } @@ -864,7 +864,7 @@ _lsprof_Profiler_disable_impl(ProfilerObject *self) } if (self->flags & POF_ENABLED) { PyObject* result = NULL; - PyObject* monitoring = PyImport_ImportModuleAttrString("sys", "monitoring"); + PyObject* monitoring = PySys_GetAttrString("monitoring"); if (!monitoring) { return NULL; @@ -983,7 +983,7 @@ profiler_init_impl(ProfilerObject *self, PyObject *timer, double timeunit, Py_XSETREF(self->externalTimer, Py_XNewRef(timer)); self->tool_id = PY_MONITORING_PROFILER_ID; - PyObject* monitoring = PyImport_ImportModuleAttrString("sys", "monitoring"); + PyObject* monitoring = PySys_GetAttrString("monitoring"); if (!monitoring) { return -1; } diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 29ef0cb0c2e088..86d8b38620cb7f 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -1915,7 +1915,7 @@ whichmodule(PickleState *st, PyObject *global, PyObject *global_name, PyObject * __module__ can be None. If it is so, then search sys.modules for the module of global. */ Py_CLEAR(module_name); - modules = _PySys_GetRequiredAttr(&_Py_ID(modules)); + modules = PySys_GetAttr(&_Py_ID(modules)); if (modules == NULL) { return NULL; } diff --git a/Modules/_testlimitedcapi/sys.c b/Modules/_testlimitedcapi/sys.c index 7d8b7a8569e515..cec7f8ab612019 100644 --- a/Modules/_testlimitedcapi/sys.c +++ b/Modules/_testlimitedcapi/sys.c @@ -1,7 +1,76 @@ +#include "pyconfig.h" // Py_GIL_DISABLED +// Need limited C API version 3.15 for PySys_GetAttr() etc +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x030f0000 +#endif #include "parts.h" #include "util.h" +static PyObject * +sys_getattr(PyObject *Py_UNUSED(module), PyObject *name) +{ + NULLABLE(name); + return PySys_GetAttr(name); +} + +static PyObject * +sys_getattrstring(PyObject *Py_UNUSED(module), PyObject *arg) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_Parse(arg, "z#", &name, &size)) { + return NULL; + } + return PySys_GetAttrString(name); +} + +static PyObject * +sys_getoptionalattr(PyObject *Py_UNUSED(module), PyObject *name) +{ + PyObject *value = UNINITIALIZED_PTR; + NULLABLE(name); + + switch (PySys_GetOptionalAttr(name, &value)) { + case -1: + assert(value == NULL); + assert(PyErr_Occurred()); + return NULL; + case 0: + assert(value == NULL); + return Py_NewRef(PyExc_AttributeError); + case 1: + return value; + default: + Py_FatalError("PySys_GetOptionalAttr() returned invalid code"); + } +} + +static PyObject * +sys_getoptionalattrstring(PyObject *Py_UNUSED(module), PyObject *arg) +{ + PyObject *value = UNINITIALIZED_PTR; + const char *name; + Py_ssize_t size; + if (!PyArg_Parse(arg, "z#", &name, &size)) { + return NULL; + } + + switch (PySys_GetOptionalAttrString(name, &value)) { + case -1: + assert(value == NULL); + assert(PyErr_Occurred()); + return NULL; + case 0: + assert(value == NULL); + return Py_NewRef(PyExc_AttributeError); + case 1: + return value; + default: + Py_FatalError("PySys_GetOptionalAttrString() returned invalid code"); + } +} + static PyObject * sys_getobject(PyObject *Py_UNUSED(module), PyObject *arg) { @@ -39,6 +108,10 @@ sys_getxoptions(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(ignored)) static PyMethodDef test_methods[] = { + {"sys_getattr", sys_getattr, METH_O}, + {"sys_getattrstring", sys_getattrstring, METH_O}, + {"sys_getoptionalattr", sys_getoptionalattr, METH_O}, + {"sys_getoptionalattrstring", sys_getoptionalattrstring, METH_O}, {"sys_getobject", sys_getobject, METH_O}, {"sys_setobject", sys_setobject, METH_VARARGS}, {"sys_getxoptions", sys_getxoptions, METH_NOARGS}, diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index cc83be4b5ff311..150a266b521736 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -10,7 +10,6 @@ #include "pycore_object_deferred.h" // _PyObject_SetDeferredRefcount() #include "pycore_pylifecycle.h" #include "pycore_pystate.h" // _PyThreadState_SetCurrent() -#include "pycore_sysmodule.h" // _PySys_GetOptionalAttr() #include "pycore_time.h" // _PyTime_FromSeconds() #include "pycore_weakref.h" // _PyWeakref_GET_REF() @@ -2290,7 +2289,7 @@ thread_excepthook(PyObject *module, PyObject *args) PyObject *thread = PyStructSequence_GET_ITEM(args, 3); PyObject *file; - if (_PySys_GetOptionalAttr( &_Py_ID(stderr), &file) < 0) { + if (PySys_GetOptionalAttr( &_Py_ID(stderr), &file) < 0) { return NULL; } if (file == NULL || file == Py_None) { diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 77695401919cb7..875840bd6a6364 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -31,7 +31,6 @@ Copyright (C) 1994 Steen Lumholt. #endif #include "pycore_long.h" // _PyLong_IsNegative() -#include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString() #include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8String #ifdef MS_WINDOWS @@ -146,7 +145,7 @@ _get_tcl_lib_path(void) int stat_return_value; PyObject *prefix; - (void) _PySys_GetOptionalAttrString("base_prefix", &prefix); + (void) PySys_GetOptionalAttrString("base_prefix", &prefix); if (prefix == NULL) { return NULL; } @@ -3547,7 +3546,7 @@ PyInit__tkinter(void) /* This helps the dynamic loader; in Unicode aware Tcl versions it also helps Tcl find its encodings. */ - (void) _PySys_GetOptionalAttrString("executable", &uexe); + (void) PySys_GetOptionalAttrString("executable", &uexe); if (uexe && PyUnicode_Check(uexe)) { // sys.executable can be None cexe = PyUnicode_EncodeFSDefault(uexe); Py_DECREF(uexe); diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index c94f4f66366170..73bea8172c7253 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -6,7 +6,6 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_runtime.h" // _Py_ID() #include "pycore_signal.h" // Py_NSIG -#include "pycore_sysmodule.h" // _PySys_GetRequiredAttr() #include "pycore_time.h" // _PyTime_FromSecondsObject() #include "pycore_traceback.h" // _Py_DumpTracebackThreads #ifdef HAVE_UNISTD_H @@ -98,7 +97,7 @@ faulthandler_get_fileno(PyObject **file_ptr) PyObject *file = *file_ptr; if (file == NULL || file == Py_None) { - file = _PySys_GetRequiredAttr(&_Py_ID(stderr)); + file = PySys_GetAttr(&_Py_ID(stderr)); if (file == NULL) { return -1; } diff --git a/Modules/main.c b/Modules/main.c index 2d7ed25f5f9790..74e48c94732565 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -497,16 +497,13 @@ pymain_run_startup(PyConfig *config, int *exitcode) static int pymain_run_interactive_hook(int *exitcode) { - PyObject *hook = PyImport_ImportModuleAttrString("sys", - "__interactivehook__"); - if (hook == NULL) { - if (PyErr_ExceptionMatches(PyExc_AttributeError)) { - // no sys.__interactivehook__ attribute - PyErr_Clear(); - return 0; - } + PyObject *hook; + if (PySys_GetOptionalAttrString("__interactivehook__", &hook) < 0) { goto error; } + if (hook == NULL) { + return 0; + } if (PySys_Audit("cpython.run_interactivehook", "O", hook) < 0) { goto error; diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index 9c54af51402a27..ab20fff1509dfe 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -56,7 +56,6 @@ Revision history: #include "Python.h" #include "osdefs.h" // SEP -#include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString() #include @@ -92,7 +91,7 @@ syslog_get_argv(void) Py_ssize_t slash; PyObject *argv; - if (_PySys_GetOptionalAttrString("argv", &argv) <= 0) { + if (PySys_GetOptionalAttrString("argv", &argv) <= 0) { return NULL; } diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index f363ef173cbd46..ba86b41e945e9d 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -12,7 +12,6 @@ #include "pycore_object.h" // _PyType_AllocNoTrack #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString() #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() #include "osdefs.h" // MAXPATHLEN @@ -1058,7 +1057,7 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) int is_possibly_shadowing_stdlib = 0; if (is_possibly_shadowing) { PyObject *stdlib_modules; - if (_PySys_GetOptionalAttrString("stdlib_module_names", &stdlib_modules) < 0) { + if (PySys_GetOptionalAttrString("stdlib_module_names", &stdlib_modules) < 0) { goto done; } if (stdlib_modules && PyAnySet_Check(stdlib_modules)) { diff --git a/PC/python3dll.c b/PC/python3dll.c index f0c578e11c643b..8ec791f8280f13 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -595,7 +595,11 @@ EXPORT_FUNC(PySys_Audit) EXPORT_FUNC(PySys_AuditTuple) EXPORT_FUNC(PySys_FormatStderr) EXPORT_FUNC(PySys_FormatStdout) +EXPORT_FUNC(PySys_GetAttr) +EXPORT_FUNC(PySys_GetAttrString) EXPORT_FUNC(PySys_GetObject) +EXPORT_FUNC(PySys_GetOptionalAttr) +EXPORT_FUNC(PySys_GetOptionalAttrString) EXPORT_FUNC(PySys_GetXOptions) EXPORT_FUNC(PySys_HasWarnOptions) EXPORT_FUNC(PySys_ResetWarnOptions) diff --git a/Python/_warnings.c b/Python/_warnings.c index 39bf1b225ccb0c..12e6172b0cf828 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -6,7 +6,6 @@ #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_sysmodule.h" // _PySys_GetOptionalAttr() #include "pycore_traceback.h" // _Py_DisplaySourceLine() #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() @@ -678,7 +677,7 @@ show_warning(PyThreadState *tstate, PyObject *filename, int lineno, goto error; } - if (_PySys_GetOptionalAttr(&_Py_ID(stderr), &f_stderr) <= 0) { + if (PySys_GetOptionalAttr(&_Py_ID(stderr), &f_stderr) <= 0) { fprintf(stderr, "lost sys.stderr\n"); goto error; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 3d0295ee3883f2..e08c63924ca16d 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -14,7 +14,6 @@ #include "pycore_pyerrors.h" // _PyErr_NoMemory() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_pythonrun.h" // _Py_SourceAsString() -#include "pycore_sysmodule.h" // _PySys_GetRequiredAttr() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_cell.h" // PyCell_GetRef() @@ -465,7 +464,7 @@ builtin_callable(PyObject *module, PyObject *obj) static PyObject * builtin_breakpoint(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *keywords) { - PyObject *hook = _PySys_GetRequiredAttrString("breakpointhook"); + PyObject *hook = PySys_GetAttrString("breakpointhook"); if (hook == NULL) { return NULL; } @@ -2164,7 +2163,7 @@ builtin_print_impl(PyObject *module, PyObject * const *args, int i, err; if (file == Py_None) { - file = _PySys_GetRequiredAttr(&_Py_ID(stdout)); + file = PySys_GetAttr(&_Py_ID(stdout)); if (file == NULL) { return NULL; } @@ -2270,7 +2269,7 @@ builtin_input_impl(PyObject *module, PyObject *prompt) int tty; /* Check that stdin/out/err are intact */ - fin = _PySys_GetRequiredAttr(&_Py_ID(stdin)); + fin = PySys_GetAttr(&_Py_ID(stdin)); if (fin == NULL) { goto error; } @@ -2278,7 +2277,7 @@ builtin_input_impl(PyObject *module, PyObject *prompt) PyErr_SetString(PyExc_RuntimeError, "lost sys.stdin"); goto error; } - fout = _PySys_GetRequiredAttr(&_Py_ID(stdout)); + fout = PySys_GetAttr(&_Py_ID(stdout)); if (fout == NULL) { goto error; } @@ -2286,7 +2285,7 @@ builtin_input_impl(PyObject *module, PyObject *prompt) PyErr_SetString(PyExc_RuntimeError, "lost sys.stdout"); goto error; } - ferr = _PySys_GetRequiredAttr(&_Py_ID(stderr)); + ferr = PySys_GetAttr(&_Py_ID(stderr)); if (ferr == NULL) { goto error; } diff --git a/Python/ceval.c b/Python/ceval.c index 4efa4b697fb59a..7aec196cb85704 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2979,7 +2979,7 @@ _PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) int is_possibly_shadowing_stdlib = 0; if (is_possibly_shadowing) { PyObject *stdlib_modules; - if (_PySys_GetOptionalAttrString("stdlib_module_names", &stdlib_modules) < 0) { + if (PySys_GetOptionalAttrString("stdlib_module_names", &stdlib_modules) < 0) { goto done; } if (stdlib_modules && PyAnySet_Check(stdlib_modules)) { diff --git a/Python/errors.c b/Python/errors.c index 81f267b043afaf..a3122f76bdd87d 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -10,7 +10,6 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_runtime.h" // _Py_ID() #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() -#include "pycore_sysmodule.h" // _PySys_GetOptionalAttr() #include "pycore_traceback.h" // _PyTraceBack_FromFrame() #include "pycore_unicodeobject.h" // _PyUnicode_Equal() @@ -1570,7 +1569,7 @@ write_unraisable_exc(PyThreadState *tstate, PyObject *exc_type, PyObject *obj) { PyObject *file; - if (_PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { return -1; } if (file == NULL || file == Py_None) { @@ -1677,7 +1676,7 @@ format_unraisable_v(const char *format, va_list va, PyObject *obj) } PyObject *hook; - if (_PySys_GetOptionalAttr(&_Py_ID(unraisablehook), &hook) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(unraisablehook), &hook) < 0) { Py_DECREF(hook_args); err_msg_str = NULL; obj = NULL; diff --git a/Python/import.c b/Python/import.c index e7be1b90751a6c..98557991378e05 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3369,11 +3369,11 @@ PyObject * PyImport_GetImporter(PyObject *path) { PyThreadState *tstate = _PyThreadState_GET(); - PyObject *path_importer_cache = _PySys_GetRequiredAttrString("path_importer_cache"); + PyObject *path_importer_cache = PySys_GetAttrString("path_importer_cache"); if (path_importer_cache == NULL) { return NULL; } - PyObject *path_hooks = _PySys_GetRequiredAttrString("path_hooks"); + PyObject *path_hooks = PySys_GetAttrString("path_hooks"); if (path_hooks == NULL) { Py_DECREF(path_importer_cache); return NULL; @@ -3682,14 +3682,14 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) PyTime_t t1 = 0, accumulated_copy = accumulated; PyObject *sys_path, *sys_meta_path, *sys_path_hooks; - if (_PySys_GetOptionalAttrString("path", &sys_path) < 0) { + if (PySys_GetOptionalAttrString("path", &sys_path) < 0) { return NULL; } - if (_PySys_GetOptionalAttrString("meta_path", &sys_meta_path) < 0) { + if (PySys_GetOptionalAttrString("meta_path", &sys_meta_path) < 0) { Py_XDECREF(sys_path); return NULL; } - if (_PySys_GetOptionalAttrString("path_hooks", &sys_path_hooks) < 0) { + if (PySys_GetOptionalAttrString("path_hooks", &sys_path_hooks) < 0) { Py_XDECREF(sys_meta_path); Py_XDECREF(sys_path); return NULL; @@ -4127,7 +4127,7 @@ _PyImport_FiniCore(PyInterpreterState *interp) static int init_zipimport(PyThreadState *tstate, int verbose) { - PyObject *path_hooks = _PySys_GetRequiredAttrString("path_hooks"); + PyObject *path_hooks = PySys_GetAttrString("path_hooks"); if (path_hooks == NULL) { return -1; } diff --git a/Python/initconfig.c b/Python/initconfig.c index 25e30aa648e8aa..71d7cfed5c44c1 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -3647,7 +3647,7 @@ _Py_DumpPathConfig(PyThreadState *tstate) #define DUMP_SYS(NAME) \ do { \ PySys_FormatStderr(" sys.%s = ", #NAME); \ - if (_PySys_GetOptionalAttrString(#NAME, &obj) < 0) { \ + if (PySys_GetOptionalAttrString(#NAME, &obj) < 0) { \ PyErr_Clear(); \ } \ if (obj != NULL) { \ @@ -3671,7 +3671,7 @@ _Py_DumpPathConfig(PyThreadState *tstate) #undef DUMP_SYS PyObject *sys_path; - (void) _PySys_GetOptionalAttrString("path", &sys_path); + (void) PySys_GetOptionalAttrString("path", &sys_path); if (sys_path != NULL && PyList_Check(sys_path)) { PySys_WriteStderr(" sys.path = [\n"); Py_ssize_t len = PyList_GET_SIZE(sys_path); @@ -4294,7 +4294,7 @@ _PyConfig_CreateXOptionsDict(const PyConfig *config) static int config_get_sys_write_bytecode(const PyConfig *config, int *value) { - PyObject *attr = _PySys_GetRequiredAttrString("dont_write_bytecode"); + PyObject *attr = PySys_GetAttrString("dont_write_bytecode"); if (attr == NULL) { return -1; } @@ -4315,7 +4315,7 @@ config_get(const PyConfig *config, const PyConfigSpec *spec, { if (use_sys) { if (spec->sys.attr != NULL) { - return _PySys_GetRequiredAttrString(spec->sys.attr); + return PySys_GetAttrString(spec->sys.attr); } if (strcmp(spec->name, "write_bytecode") == 0) { diff --git a/Python/intrinsics.c b/Python/intrinsics.c index ff44ba0ee64fa9..8ea920e690cd0d 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -9,7 +9,6 @@ #include "pycore_intrinsics.h" // INTRINSIC_PRINT #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_runtime.h" // _Py_ID() -#include "pycore_sysmodule.h" // _PySys_GetRequiredAttr() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_typevarobject.h" // _Py_make_typevar() #include "pycore_unicodeobject.h" // _PyUnicode_FromASCII() @@ -27,7 +26,7 @@ no_intrinsic1(PyThreadState* tstate, PyObject *unused) static PyObject * print_expr(PyThreadState* Py_UNUSED(ignored), PyObject *value) { - PyObject *hook = _PySys_GetRequiredAttr(&_Py_ID(displayhook)); + PyObject *hook = PySys_GetAttr(&_Py_ID(displayhook)); if (hook == NULL) { return NULL; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8394245d373030..724fda63511282 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1283,7 +1283,7 @@ init_interp_main(PyThreadState *tstate) if (is_main_interp) { /* Initialize warnings. */ PyObject *warnoptions; - if (_PySys_GetOptionalAttrString("warnoptions", &warnoptions) < 0) { + if (PySys_GetOptionalAttrString("warnoptions", &warnoptions) < 0) { return _PyStatus_ERR("can't initialize warnings"); } if (warnoptions != NULL && PyList_Check(warnoptions) && @@ -1806,7 +1806,7 @@ flush_std_files(void) PyObject *file; int status = 0; - if (_PySys_GetOptionalAttr(&_Py_ID(stdout), &file) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(stdout), &file) < 0) { status = -1; } else if (file != NULL && file != Py_None && !file_is_closed(file)) { @@ -1819,7 +1819,7 @@ flush_std_files(void) } Py_XDECREF(file); - if (_PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { PyErr_Clear(); status = -1; } @@ -3046,7 +3046,7 @@ _Py_FatalError_PrintExc(PyThreadState *tstate) } PyObject *ferr; - if (_PySys_GetOptionalAttr(&_Py_ID(stderr), &ferr) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(stderr), &ferr) < 0) { _PyErr_Clear(tstate); } if (ferr == NULL || ferr == Py_None) { diff --git a/Python/pythonrun.c b/Python/pythonrun.c index f67b72aa91f671..8f1c78bf831863 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -114,7 +114,7 @@ _PyRun_InteractiveLoopObject(FILE *fp, PyObject *filename, PyCompilerFlags *flag } PyObject *v; - if (_PySys_GetOptionalAttr(&_Py_ID(ps1), &v) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(ps1), &v) < 0) { PyErr_Print(); return -1; } @@ -128,7 +128,7 @@ _PyRun_InteractiveLoopObject(FILE *fp, PyObject *filename, PyCompilerFlags *flag } } Py_XDECREF(v); - if (_PySys_GetOptionalAttr(&_Py_ID(ps2), &v) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(ps2), &v) < 0) { PyErr_Print(); return -1; } @@ -206,7 +206,7 @@ pyrun_one_parse_ast(FILE *fp, PyObject *filename, PyObject *encoding_obj = NULL; const char *encoding = NULL; if (fp == stdin) { - if (_PySys_GetOptionalAttr(&_Py_ID(stdin), &attr) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(stdin), &attr) < 0) { PyErr_Clear(); } else if (attr != NULL && attr != Py_None) { @@ -226,7 +226,7 @@ pyrun_one_parse_ast(FILE *fp, PyObject *filename, // Get sys.ps1 (as UTF-8) PyObject *ps1_obj = NULL; const char *ps1 = ""; - if (_PySys_GetOptionalAttr(&_Py_ID(ps1), &attr) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(ps1), &attr) < 0) { PyErr_Clear(); } else if (attr != NULL) { @@ -247,7 +247,7 @@ pyrun_one_parse_ast(FILE *fp, PyObject *filename, // Get sys.ps2 (as UTF-8) PyObject *ps2_obj = NULL; const char *ps2 = ""; - if (_PySys_GetOptionalAttr(&_Py_ID(ps2), &attr) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(ps2), &attr) < 0) { PyErr_Clear(); } else if (attr != NULL) { @@ -658,7 +658,7 @@ _Py_HandleSystemExitAndKeyboardInterrupt(int *exitcode_p) } PyObject *sys_stderr; - if (_PySys_GetOptionalAttr(&_Py_ID(stderr), &sys_stderr) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(stderr), &sys_stderr) < 0) { PyErr_Clear(); } else if (sys_stderr != NULL && sys_stderr != Py_None) { @@ -722,7 +722,7 @@ _PyErr_PrintEx(PyThreadState *tstate, int set_sys_last_vars) _PyErr_Clear(tstate); } } - if (_PySys_GetOptionalAttr(&_Py_ID(excepthook), &hook) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(excepthook), &hook) < 0) { PyErr_Clear(); } if (_PySys_Audit(tstate, "sys.excepthook", "OOOO", hook ? hook : Py_None, @@ -1197,7 +1197,7 @@ void PyErr_Display(PyObject *unused, PyObject *value, PyObject *tb) { PyObject *file; - if (_PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { PyObject *exc = PyErr_GetRaisedException(); _PyObject_Dump(value); fprintf(stderr, "lost sys.stderr\n"); @@ -1321,7 +1321,7 @@ static void flush_io_stream(PyThreadState *tstate, PyObject *name) { PyObject *f; - if (_PySys_GetOptionalAttr(name, &f) < 0) { + if (PySys_GetOptionalAttr(name, &f) < 0) { PyErr_Clear(); } if (f != NULL) { diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 4ed045e3297bbc..e5ae841d195d4f 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -76,12 +76,12 @@ module sys PyObject * -_PySys_GetRequiredAttr(PyObject *name) +PySys_GetAttr(PyObject *name) { if (!PyUnicode_Check(name)) { PyErr_Format(PyExc_TypeError, - "attribute name must be string, not '%.200s'", - Py_TYPE(name)->tp_name); + "attribute name must be string, not '%T'", + name); return NULL; } PyThreadState *tstate = _PyThreadState_GET(); @@ -98,7 +98,7 @@ _PySys_GetRequiredAttr(PyObject *name) } PyObject * -_PySys_GetRequiredAttrString(const char *name) +PySys_GetAttrString(const char *name) { PyThreadState *tstate = _PyThreadState_GET(); PyObject *sysdict = tstate->interp->sysdict; @@ -114,12 +114,12 @@ _PySys_GetRequiredAttrString(const char *name) } int -_PySys_GetOptionalAttr(PyObject *name, PyObject **value) +PySys_GetOptionalAttr(PyObject *name, PyObject **value) { if (!PyUnicode_Check(name)) { PyErr_Format(PyExc_TypeError, - "attribute name must be string, not '%.200s'", - Py_TYPE(name)->tp_name); + "attribute name must be string, not '%T'", + name); *value = NULL; return -1; } @@ -133,7 +133,7 @@ _PySys_GetOptionalAttr(PyObject *name, PyObject **value) } int -_PySys_GetOptionalAttrString(const char *name, PyObject **value) +PySys_GetOptionalAttrString(const char *name, PyObject **value) { PyThreadState *tstate = _PyThreadState_GET(); PyObject *sysdict = tstate->interp->sysdict; @@ -773,7 +773,7 @@ sys_displayhook(PyObject *module, PyObject *o) } if (PyObject_SetAttr(builtins, _Py_LATIN1_CHR('_'), Py_None) != 0) return NULL; - outf = _PySys_GetRequiredAttr(&_Py_ID(stdout)); + outf = PySys_GetAttr(&_Py_ID(stdout)); if (outf == NULL) { return NULL; } @@ -3005,7 +3005,7 @@ static PyObject * get_warnoptions(PyThreadState *tstate) { PyObject *warnoptions; - if (_PySys_GetOptionalAttr(&_Py_ID(warnoptions), &warnoptions) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(warnoptions), &warnoptions) < 0) { return NULL; } if (warnoptions == NULL || !PyList_Check(warnoptions)) { @@ -3042,7 +3042,7 @@ PySys_ResetWarnOptions(void) } PyObject *warnoptions; - if (_PySys_GetOptionalAttr(&_Py_ID(warnoptions), &warnoptions) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(warnoptions), &warnoptions) < 0) { PyErr_Clear(); return; } @@ -3106,7 +3106,7 @@ PyAPI_FUNC(int) PySys_HasWarnOptions(void) { PyObject *warnoptions; - if (_PySys_GetOptionalAttr(&_Py_ID(warnoptions), &warnoptions) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(warnoptions), &warnoptions) < 0) { PyErr_Clear(); return 0; } @@ -3120,7 +3120,7 @@ static PyObject * get_xoptions(PyThreadState *tstate) { PyObject *xoptions; - if (_PySys_GetOptionalAttr(&_Py_ID(_xoptions), &xoptions) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(_xoptions), &xoptions) < 0) { return NULL; } if (xoptions == NULL || !PyDict_Check(xoptions)) { @@ -3373,7 +3373,7 @@ sys_set_flag(PyObject *flags, Py_ssize_t pos, PyObject *value) int _PySys_SetFlagObj(Py_ssize_t pos, PyObject *value) { - PyObject *flags = _PySys_GetRequiredAttrString("flags"); + PyObject *flags = PySys_GetAttrString("flags"); if (flags == NULL) { return -1; } @@ -3935,7 +3935,7 @@ _PySys_UpdateConfig(PyThreadState *tstate) #undef COPY_WSTR // sys.flags - PyObject *flags = _PySys_GetRequiredAttrString("flags"); + PyObject *flags = PySys_GetAttrString("flags"); if (flags == NULL) { return -1; } @@ -4251,7 +4251,7 @@ PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) } PyObject *sys_path; - if (_PySys_GetOptionalAttr(&_Py_ID(path), &sys_path) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(path), &sys_path) < 0) { Py_FatalError("can't get sys.path"); } else if (sys_path != NULL) { @@ -4347,7 +4347,7 @@ sys_write(PyObject *key, FILE *fp, const char *format, va_list va) PyObject *exc = _PyErr_GetRaisedException(tstate); written = PyOS_vsnprintf(buffer, sizeof(buffer), format, va); - file = _PySys_GetRequiredAttr(key); + file = PySys_GetAttr(key); if (sys_pyfile_write(buffer, file) != 0) { _PyErr_Clear(tstate); fputs(buffer, fp); @@ -4391,7 +4391,7 @@ sys_format(PyObject *key, FILE *fp, const char *format, va_list va) PyObject *exc = _PyErr_GetRaisedException(tstate); message = PyUnicode_FromFormatV(format, va); if (message != NULL) { - file = _PySys_GetRequiredAttr(key); + file = PySys_GetAttr(key); if (sys_pyfile_write_unicode(message, file) != 0) { _PyErr_Clear(tstate); utf8 = PyUnicode_AsUTF8(message); diff --git a/Python/traceback.c b/Python/traceback.c index c06cb1a59089e2..4f674eaf55715b 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -9,7 +9,6 @@ #include "pycore_interpframe.h" // _PyFrame_GetCode() #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_sysmodule.h" // _PySys_GetOptionalAttr() #include "pycore_traceback.h" // EXCEPTION_TB_HEADER #include "frameobject.h" // PyFrame_New() @@ -399,7 +398,7 @@ _Py_FindSourceFile(PyObject *filename, char* namebuf, size_t namelen, PyObject * taillen = strlen(tail); PyThreadState *tstate = _PyThreadState_GET(); - if (_PySys_GetOptionalAttr(&_Py_ID(path), &syspath) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(path), &syspath) < 0) { PyErr_Clear(); goto error; } @@ -777,7 +776,7 @@ _PyTraceBack_Print(PyObject *v, const char *header, PyObject *f) PyErr_BadInternalCall(); return -1; } - if (_PySys_GetOptionalAttrString("tracebacklimit", &limitv) < 0) { + if (PySys_GetOptionalAttrString("tracebacklimit", &limitv) < 0) { return -1; } else if (limitv != NULL && PyLong_Check(limitv)) { From f6324bc7eedc615c3c961fe368a8c56697d42936 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 28 May 2025 20:14:56 +0300 Subject: [PATCH 002/116] gh-108885: Imporove tests for doctest (GH-134832) Test the error and failure report in more detail. --- .../test_doctest/sample_doctest_errors.py | 46 ++ Lib/test/test_doctest/test_doctest.py | 485 +++++++++++++++++- Lib/test/test_doctest/test_doctest_errors.txt | 14 + Lib/test/test_doctest/test_doctest_skip.txt | 2 + Lib/test/test_doctest/test_doctest_skip2.txt | 6 + 5 files changed, 545 insertions(+), 8 deletions(-) create mode 100644 Lib/test/test_doctest/sample_doctest_errors.py create mode 100644 Lib/test/test_doctest/test_doctest_errors.txt create mode 100644 Lib/test/test_doctest/test_doctest_skip2.txt diff --git a/Lib/test/test_doctest/sample_doctest_errors.py b/Lib/test/test_doctest/sample_doctest_errors.py new file mode 100644 index 00000000000000..4a6f07af2d4e5a --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest_errors.py @@ -0,0 +1,46 @@ +"""This is a sample module used for testing doctest. + +This module includes various scenarios involving errors. + +>>> 2 + 2 +5 +>>> 1/0 +1 +""" + +def g(): + [][0] # line 12 + +def errors(): + """ + >>> 2 + 2 + 5 + >>> 1/0 + 1 + >>> def f(): + ... 2 + '2' + ... + >>> f() + 1 + >>> g() + 1 + """ + +def syntax_error(): + """ + >>> 2+*3 + 5 + """ + +__test__ = { + 'bad': """ + >>> 2 + 2 + 5 + >>> 1/0 + 1 + """, +} + +def test_suite(): + import doctest + return doctest.DocTestSuite() diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index a4a49298bab3be..c5b247797c321d 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -2267,13 +2267,21 @@ def test_DocTestSuite(): >>> import unittest >>> import test.test_doctest.sample_doctest >>> suite = doctest.DocTestSuite(test.test_doctest.sample_doctest) - >>> suite.run(unittest.TestResult()) + >>> result = suite.run(unittest.TestResult()) + >>> result + >>> for tst, _ in result.failures: + ... print(tst) + bad (test.test_doctest.sample_doctest.__test__) + foo (test.test_doctest.sample_doctest) + test_silly_setup (test.test_doctest.sample_doctest) + y_is_one (test.test_doctest.sample_doctest) We can also supply the module by name: >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest') - >>> suite.run(unittest.TestResult()) + >>> result = suite.run(unittest.TestResult()) + >>> result The module need not contain any doctest examples: @@ -2297,6 +2305,14 @@ def test_DocTestSuite(): >>> len(result.skipped) 2 + >>> for tst, _ in result.skipped: + ... print(tst) + double_skip (test.test_doctest.sample_doctest_skip) + single_skip (test.test_doctest.sample_doctest_skip) + >>> for tst, _ in result.failures: + ... print(tst) + no_skip_fail (test.test_doctest.sample_doctest_skip) + partial_skip_fail (test.test_doctest.sample_doctest_skip) We can use the current module: @@ -2383,7 +2399,174 @@ def test_DocTestSuite(): modified the test globals, which are a copy of the sample_doctest module dictionary. The test globals are automatically cleared for us after a test. - """ + """ + +def test_DocTestSuite_errors(): + """Tests for error reporting in DocTestSuite. + + >>> import unittest + >>> import test.test_doctest.sample_doctest_errors as mod + >>> suite = doctest.DocTestSuite(mod) + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File ... + raise self.failureException(self.format_failure(new.getvalue())) + AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors + File "...sample_doctest_errors.py", line 0, in sample_doctest_errors + + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 5, in test.test_doctest.sample_doctest_errors + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 7, in test.test_doctest.sample_doctest_errors + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + + >>> print(result.failures[1][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File ... + raise self.failureException(self.format_failure(new.getvalue())) + AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.__test__.bad + File "...sample_doctest_errors.py", line unknown line number, in bad + + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + + >>> print(result.failures[2][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File ... + raise self.failureException(self.format_failure(new.getvalue())) + AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.errors + File "...sample_doctest_errors.py", line 14, in errors + + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 16, in test.test_doctest.sample_doctest_errors.errors + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 18, in test.test_doctest.sample_doctest_errors.errors + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 23, in test.test_doctest.sample_doctest_errors.errors + Failed example: + f() + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "", line 1, in + f() + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 25, in test.test_doctest.sample_doctest_errors.errors + Failed example: + g() + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "", line 1, in + g() + ~^^ + File "...sample_doctest_errors.py", line 12, in g + [][0] # line 12 + ~~^^^ + IndexError: list index out of range + + + >>> print(result.failures[3][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File ... + raise self.failureException(self.format_failure(new.getvalue())) + AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.syntax_error + File "...sample_doctest_errors.py", line 29, in syntax_error + + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 31, in test.test_doctest.sample_doctest_errors.syntax_error + Failed example: + 2+*3 + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^ + File "", line 1 + 2+*3 + ^ + SyntaxError: invalid syntax + + + """ def test_DocFileSuite(): """We can test tests found in text files using a DocFileSuite. @@ -2455,12 +2638,16 @@ def test_DocFileSuite(): >>> suite = doctest.DocFileSuite('test_doctest.txt', ... 'test_doctest4.txt', - ... 'test_doctest_skip.txt') + ... 'test_doctest_skip.txt', + ... 'test_doctest_skip2.txt') >>> result = suite.run(unittest.TestResult()) >>> result - - >>> len(result.skipped) - 1 + + >>> len(result.skipped) + 1 + >>> for tst, _ in result.skipped: # doctest: +ELLIPSIS + ... print('=', tst) + = ...test_doctest_skip.txt You can specify initial global variables: @@ -2542,8 +2729,82 @@ def test_DocFileSuite(): ... encoding='utf-8') >>> suite.run(unittest.TestResult()) + """ - """ +def test_DocFileSuite_errors(): + """Tests for error reporting in DocTestSuite. + + >>> import unittest + >>> suite = doctest.DocFileSuite('test_doctest_errors.txt') + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File ... + raise self.failureException(self.format_failure(new.getvalue())) + AssertionError: Failed doctest test for test_doctest_errors.txt + File "...test_doctest_errors.txt", line 0 + + ---------------------------------------------------------------------- + File "...test_doctest_errors.txt", line 4, in test_doctest_errors.txt + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ---------------------------------------------------------------------- + File "...test_doctest_errors.txt", line 6, in test_doctest_errors.txt + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ---------------------------------------------------------------------- + File "...test_doctest_errors.txt", line 11, in test_doctest_errors.txt + Failed example: + f() + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "", line 1, in + f() + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + ---------------------------------------------------------------------- + File "...test_doctest_errors.txt", line 13, in test_doctest_errors.txt + Failed example: + 2+*3 + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^ + File "", line 1 + 2+*3 + ^ + SyntaxError: invalid syntax + + + """ def test_trailing_space_in_test(): """ @@ -2612,6 +2873,8 @@ def test_unittest_reportflags(): ... optionflags=doctest.DONT_ACCEPT_BLANKLINE) >>> import unittest >>> result = suite.run(unittest.TestResult()) + >>> result + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS Traceback ... Failed example: @@ -2629,6 +2892,8 @@ def test_unittest_reportflags(): Now, when we run the test: >>> result = suite.run(unittest.TestResult()) + >>> result + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS Traceback ... Failed example: @@ -2650,6 +2915,8 @@ def test_unittest_reportflags(): Then the default eporting options are ignored: >>> result = suite.run(unittest.TestResult()) + >>> result + *NOTE*: These doctest are intentionally not placed in raw string to depict the trailing whitespace using `\x20` in the diff below. @@ -2860,6 +3127,73 @@ def test_testfile(): r""" >>> _colorize.COLORIZE = save_colorize """ +def test_testfile_errors(): r""" +Tests for error reporting in the testfile() function. + + >>> doctest.testfile('test_doctest_errors.txt', verbose=False) # doctest: +ELLIPSIS + ********************************************************************** + File "...test_doctest_errors.txt", line 4, in test_doctest_errors.txt + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ********************************************************************** + File "...test_doctest_errors.txt", line 6, in test_doctest_errors.txt + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ********************************************************************** + File "...test_doctest_errors.txt", line 11, in test_doctest_errors.txt + Failed example: + f() + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "", line 1, in + f() + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + ********************************************************************** + File "...test_doctest_errors.txt", line 13, in test_doctest_errors.txt + Failed example: + 2+*3 + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^ + File "", line 1 + 2+*3 + ^ + SyntaxError: invalid syntax + ********************************************************************** + 1 item had failures: + 4 of 5 in test_doctest_errors.txt + ***Test Failed*** 4 failures. + TestResults(failed=4, attempted=5) +""" + class TestImporter(importlib.abc.MetaPathFinder): def find_spec(self, fullname, path, target=None): @@ -2990,6 +3324,141 @@ def test_testmod(): r""" TestResults(failed=0, attempted=0) """ +def test_testmod_errors(): r""" +Tests for error reporting in the testmod() function. + + >>> import test.test_doctest.sample_doctest_errors as mod + >>> doctest.testmod(mod, verbose=False) # doctest: +ELLIPSIS + ********************************************************************** + File "...sample_doctest_errors.py", line 5, in test.test_doctest.sample_doctest_errors + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ********************************************************************** + File "...sample_doctest_errors.py", line 7, in test.test_doctest.sample_doctest_errors + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ********************************************************************** + File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ********************************************************************** + File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ********************************************************************** + File "...sample_doctest_errors.py", line 16, in test.test_doctest.sample_doctest_errors.errors + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ********************************************************************** + File "...sample_doctest_errors.py", line 18, in test.test_doctest.sample_doctest_errors.errors + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ********************************************************************** + File "...sample_doctest_errors.py", line 23, in test.test_doctest.sample_doctest_errors.errors + Failed example: + f() + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "", line 1, in + f() + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + ********************************************************************** + File "...sample_doctest_errors.py", line 25, in test.test_doctest.sample_doctest_errors.errors + Failed example: + g() + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "", line 1, in + g() + ~^^ + File "...sample_doctest_errors.py", line 12, in g + [][0] # line 12 + ~~^^^ + IndexError: list index out of range + ********************************************************************** + File "...sample_doctest_errors.py", line 31, in test.test_doctest.sample_doctest_errors.syntax_error + Failed example: + 2+*3 + Exception raised: + Traceback (most recent call last): + File ... + exec(compile(example.source, filename, "single", + ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + compileflags, True), test.globs) + ^^^^^^^^^^^^^^^^^^^ + File "", line 1 + 2+*3 + ^ + SyntaxError: invalid syntax + ********************************************************************** + 4 items had failures: + 2 of 2 in test.test_doctest.sample_doctest_errors + 2 of 2 in test.test_doctest.sample_doctest_errors.__test__.bad + 4 of 5 in test.test_doctest.sample_doctest_errors.errors + 1 of 1 in test.test_doctest.sample_doctest_errors.syntax_error + ***Test Failed*** 9 failures. + TestResults(failed=9, attempted=10) +""" + try: os.fsencode("foo-bär@baz.py") supports_unicode = True diff --git a/Lib/test/test_doctest/test_doctest_errors.txt b/Lib/test/test_doctest/test_doctest_errors.txt new file mode 100644 index 00000000000000..93c3c106e60b32 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest_errors.txt @@ -0,0 +1,14 @@ +This is a sample doctest in a text file, in which all examples fail +or raise an exception. + + >>> 2 + 2 + 5 + >>> 1/0 + 1 + >>> def f(): + ... 2 + '2' + ... + >>> f() + 1 + >>> 2+*3 + 5 diff --git a/Lib/test/test_doctest/test_doctest_skip.txt b/Lib/test/test_doctest/test_doctest_skip.txt index f340e2b8141253..06c23d06e606a3 100644 --- a/Lib/test/test_doctest/test_doctest_skip.txt +++ b/Lib/test/test_doctest/test_doctest_skip.txt @@ -2,3 +2,5 @@ This is a sample doctest in a text file, in which all examples are skipped. >>> 2 + 2 # doctest: +SKIP 5 + >>> 2 + 2 # doctest: +SKIP + 4 diff --git a/Lib/test/test_doctest/test_doctest_skip2.txt b/Lib/test/test_doctest/test_doctest_skip2.txt new file mode 100644 index 00000000000000..85e4938c346a09 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest_skip2.txt @@ -0,0 +1,6 @@ +This is a sample doctest in a text file, in which some examples are skipped. + + >>> 2 + 2 # doctest: +SKIP + 5 + >>> 2 + 2 + 4 From 9fbd66a93d526c49fac8e1427c25e8f7f4154e29 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 28 May 2025 19:03:41 +0100 Subject: [PATCH 003/116] GH-133912: Fix `PyObject_GenericSetDict` to handle inline values (GH-134725) --- Include/internal/pycore_object.h | 2 + Lib/test/test_capi/test_type.py | 10 ++++ ...-05-26-15-55-50.gh-issue-133912.-xAguL.rst | 2 + Modules/_testcapimodule.c | 49 +++++++++++++++++ Objects/object.c | 31 +---------- Objects/typeobject.c | 54 +++++++++++-------- 6 files changed, 95 insertions(+), 53 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-26-15-55-50.gh-issue-133912.-xAguL.rst diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 3fc1e7d7dcbd15..50225623fe52db 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -1010,6 +1010,8 @@ enum _PyAnnotateFormat { _Py_ANNOTATE_FORMAT_STRING = 4, }; +int _PyObject_SetDict(PyObject *obj, PyObject *value); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_capi/test_type.py b/Lib/test/test_capi/test_type.py index 3c9974c7387388..15fb4a93e2ad74 100644 --- a/Lib/test/test_capi/test_type.py +++ b/Lib/test/test_capi/test_type.py @@ -264,3 +264,13 @@ def test_manual_heap_type(self): ManualHeapType = _testcapi.ManualHeapType for i in range(100): self.assertIsInstance(ManualHeapType(), ManualHeapType) + + def test_extension_managed_dict_type(self): + ManagedDictType = _testcapi.ManagedDictType + obj = ManagedDictType() + obj.foo = 42 + self.assertEqual(obj.foo, 42) + self.assertEqual(obj.__dict__, {'foo': 42}) + obj.__dict__ = {'bar': 3} + self.assertEqual(obj.__dict__, {'bar': 3}) + self.assertEqual(obj.bar, 3) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-26-15-55-50.gh-issue-133912.-xAguL.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-26-15-55-50.gh-issue-133912.-xAguL.rst new file mode 100644 index 00000000000000..2118f3d0c350ec --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-26-15-55-50.gh-issue-133912.-xAguL.rst @@ -0,0 +1,2 @@ +Fix the C API function ``PyObject_GenericSetDict`` to handle extension +classes with inline values. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3aa6e4c9e43a26..281c5b41137ac2 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3175,6 +3175,48 @@ create_manual_heap_type(void) return (PyObject *)type; } +typedef struct { + PyObject_VAR_HEAD +} ManagedDictObject; + +int ManagedDict_traverse(PyObject *self, visitproc visit, void *arg) { + PyObject_VisitManagedDict(self, visit, arg); + Py_VISIT(Py_TYPE(self)); + return 0; +} + +int ManagedDict_clear(PyObject *self) { + PyObject_ClearManagedDict(self); + return 0; +} + +static PyGetSetDef ManagedDict_getset[] = { + {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {NULL, NULL, NULL, NULL, NULL}, +}; + +static PyType_Slot ManagedDict_slots[] = { + {Py_tp_new, (void *)PyType_GenericNew}, + {Py_tp_getset, (void *)ManagedDict_getset}, + {Py_tp_traverse, (void *)ManagedDict_traverse}, + {Py_tp_clear, (void *)ManagedDict_clear}, + {0} +}; + +static PyType_Spec ManagedDict_spec = { + "_testcapi.ManagedDictType", + sizeof(ManagedDictObject), + 0, // itemsize + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_MANAGED_DICT | Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_HAVE_GC, + ManagedDict_slots +}; + +static PyObject * +create_managed_dict_type(void) +{ + return PyType_FromSpec(&ManagedDict_spec); +} + static struct PyModuleDef _testcapimodule = { PyModuleDef_HEAD_INIT, .m_name = "_testcapi", @@ -3315,6 +3357,13 @@ PyInit__testcapi(void) return NULL; } + PyObject *managed_dict_type = create_managed_dict_type(); + if (managed_dict_type == NULL) { + return NULL; + } + if (PyModule_Add(m, "ManagedDictType", managed_dict_type) < 0) { + return NULL; + } /* Include tests from the _testcapi/ directory */ if (_PyTestCapi_Init_Vectorcall(m) < 0) { diff --git a/Objects/object.c b/Objects/object.c index 68c8bfeae33e33..9fe61ba7f1593a 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2016,40 +2016,11 @@ PyObject_GenericSetAttr(PyObject *obj, PyObject *name, PyObject *value) int PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) { - PyObject **dictptr = _PyObject_GetDictPtr(obj); - if (dictptr == NULL) { - if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_INLINE_VALUES) && - _PyObject_GetManagedDict(obj) == NULL - ) { - /* Was unable to convert to dict */ - PyErr_NoMemory(); - } - else { - PyErr_SetString(PyExc_AttributeError, - "This object has no __dict__"); - } - return -1; - } if (value == NULL) { PyErr_SetString(PyExc_TypeError, "cannot delete __dict__"); return -1; } - if (!PyDict_Check(value)) { - PyErr_Format(PyExc_TypeError, - "__dict__ must be set to a dictionary, " - "not a '%.200s'", Py_TYPE(value)->tp_name); - return -1; - } - Py_BEGIN_CRITICAL_SECTION(obj); - PyObject *olddict = *dictptr; - FT_ATOMIC_STORE_PTR_RELEASE(*dictptr, Py_NewRef(value)); -#ifdef Py_GIL_DISABLED - _PyObject_XDecRefDelayed(olddict); -#else - Py_XDECREF(olddict); -#endif - Py_END_CRITICAL_SECTION(); - return 0; + return _PyObject_SetDict(obj, value); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 07f01e3bcb229c..db923c164774b7 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3948,10 +3948,39 @@ subtype_dict(PyObject *obj, void *context) return PyObject_GenericGetDict(obj, context); } +int +_PyObject_SetDict(PyObject *obj, PyObject *value) +{ + if (value != NULL && !PyDict_Check(value)) { + PyErr_Format(PyExc_TypeError, + "__dict__ must be set to a dictionary, " + "not a '%.200s'", Py_TYPE(value)->tp_name); + return -1; + } + if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + return _PyObject_SetManagedDict(obj, value); + } + PyObject **dictptr = _PyObject_ComputedDictPointer(obj); + if (dictptr == NULL) { + PyErr_SetString(PyExc_AttributeError, + "This object has no __dict__"); + return -1; + } + Py_BEGIN_CRITICAL_SECTION(obj); + PyObject *olddict = *dictptr; + FT_ATOMIC_STORE_PTR_RELEASE(*dictptr, Py_NewRef(value)); +#ifdef Py_GIL_DISABLED + _PyObject_XDecRefDelayed(olddict); +#else + Py_XDECREF(olddict); +#endif + Py_END_CRITICAL_SECTION(); + return 0; +} + static int subtype_setdict(PyObject *obj, PyObject *value, void *context) { - PyObject **dictptr; PyTypeObject *base; base = get_builtin_base_with_dict(Py_TYPE(obj)); @@ -3969,28 +3998,7 @@ subtype_setdict(PyObject *obj, PyObject *value, void *context) } return func(descr, obj, value); } - /* Almost like PyObject_GenericSetDict, but allow __dict__ to be deleted. */ - if (value != NULL && !PyDict_Check(value)) { - PyErr_Format(PyExc_TypeError, - "__dict__ must be set to a dictionary, " - "not a '%.200s'", Py_TYPE(value)->tp_name); - return -1; - } - - if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - return _PyObject_SetManagedDict(obj, value); - } - else { - dictptr = _PyObject_ComputedDictPointer(obj); - if (dictptr == NULL) { - PyErr_SetString(PyExc_AttributeError, - "This object has no __dict__"); - return -1; - } - Py_CLEAR(*dictptr); - *dictptr = Py_XNewRef(value); - } - return 0; + return _PyObject_SetDict(obj, value); } static PyObject * From e496444fdfb2a64b4c7c2075fc53cc69f7c01677 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 28 May 2025 21:27:24 +0100 Subject: [PATCH 004/116] Add a tool to benchmark external inspection (#134810) --- .../benchmark_external_inspection.py | 459 ++++++++++++++++++ 1 file changed, 459 insertions(+) create mode 100644 Tools/inspection/benchmark_external_inspection.py diff --git a/Tools/inspection/benchmark_external_inspection.py b/Tools/inspection/benchmark_external_inspection.py new file mode 100644 index 00000000000000..62182194c1ab2a --- /dev/null +++ b/Tools/inspection/benchmark_external_inspection.py @@ -0,0 +1,459 @@ +import _remote_debugging +import time +import subprocess +import sys +import contextlib +import tempfile +import os +import argparse +from _colorize import get_colors, can_colorize + +CODE = '''\ +import time +import os +import sys +import math + +def slow_fibonacci(n): + """Intentionally slow recursive fibonacci - should show up prominently in profiler""" + if n <= 1: + return n + return slow_fibonacci(n-1) + slow_fibonacci(n-2) + +def medium_computation(): + """Medium complexity function""" + result = 0 + for i in range(1000): + result += math.sqrt(i) * math.sin(i) + return result + +def fast_loop(): + """Fast simple loop""" + total = 0 + for i in range(100): + total += i + return total + +def string_operations(): + """String manipulation that should be visible in profiler""" + text = "hello world " * 100 + words = text.split() + return " ".join(reversed(words)) + +def nested_calls(): + """Nested function calls to test call stack depth""" + def level1(): + def level2(): + def level3(): + return medium_computation() + return level3() + return level2() + return level1() + +def main_loop(): + """Main computation loop with different execution paths""" + iteration = 0 + + while True: + iteration += 1 + + # Different execution paths with different frequencies + if iteration % 50 == 0: + # Expensive operation - should show high per-call time + result = slow_fibonacci(20) + + elif iteration % 10 == 0: + # Medium operation + result = nested_calls() + + elif iteration % 5 == 0: + # String operations + result = string_operations() + + else: + # Fast operation - most common + result = fast_loop() + + # Small delay to make sampling more interesting + time.sleep(0.001) + +if __name__ == "__main__": + main_loop() +''' + +DEEP_STATIC_CODE = """\ +import time +def factorial(n): + if n <= 1: + time.sleep(10000) + return 1 + return n * factorial(n-1) + +factorial(900) +""" + +CODE_WITH_TONS_OF_THREADS = '''\ +import time +import threading +import random +import math + +def cpu_intensive_work(): + """Do some CPU intensive calculations""" + result = 0 + for _ in range(10000): + result += math.sin(random.random()) * math.cos(random.random()) + return result + +def io_intensive_work(): + """Simulate IO intensive work with sleeps""" + time.sleep(0.1) + +def mixed_workload(): + """Mix of CPU and IO work""" + while True: + if random.random() < 0.3: + cpu_intensive_work() + else: + io_intensive_work() + +def create_threads(n): + """Create n threads doing mixed workloads""" + threads = [] + for _ in range(n): + t = threading.Thread(target=mixed_workload, daemon=True) + t.start() + threads.append(t) + return threads + +# Start with 5 threads +active_threads = create_threads(5) +thread_count = 5 + +# Main thread manages threads and does work +while True: + # Randomly add or remove threads + if random.random() < 0.1: # 10% chance each iteration + if random.random() < 0.5 and thread_count < 100: + # Add 1-5 new threads + new_count = random.randint(1, 5) + new_threads = create_threads(new_count) + active_threads.extend(new_threads) + thread_count += new_count + elif thread_count > 10: + # Remove 1-3 threads + remove_count = random.randint(1, 5) + # The threads will terminate naturally since they're daemons + active_threads = active_threads[remove_count:] + thread_count -= remove_count + + cpu_intensive_work() + time.sleep(0.05) +''' + +CODE_EXAMPLES = { + "basic": { + "code": CODE, + "description": "Mixed workload with fibonacci, computations, and string operations", + }, + "deep_static": { + "code": DEEP_STATIC_CODE, + "description": "Deep recursive call stack with 900+ frames (factorial)", + }, + "threads": { + "code": CODE_WITH_TONS_OF_THREADS, + "description": "Tons of threads doing mixed CPU/IO work", + }, +} + + +def benchmark(unwinder, duration_seconds=10): + """Benchmark mode - measure raw sampling speed for specified duration""" + sample_count = 0 + fail_count = 0 + total_work_time = 0.0 + start_time = time.perf_counter() + end_time = start_time + duration_seconds + + colors = get_colors(can_colorize()) + + print( + f"{colors.BOLD_BLUE}Benchmarking sampling speed for {duration_seconds} seconds...{colors.RESET}" + ) + + try: + while time.perf_counter() < end_time: + work_start = time.perf_counter() + try: + stack_trace = unwinder.get_stack_trace() + if stack_trace: + sample_count += 1 + except (OSError, RuntimeError, UnicodeDecodeError) as e: + fail_count += 1 + + work_end = time.perf_counter() + total_work_time += work_end - work_start + + total_attempts = sample_count + fail_count + if total_attempts % 10000 == 0: + avg_work_time_us = (total_work_time / total_attempts) * 1e6 + work_rate = ( + total_attempts / total_work_time if total_work_time > 0 else 0 + ) + success_rate = (sample_count / total_attempts) * 100 + + # Color code the success rate + if success_rate >= 95: + success_color = colors.GREEN + elif success_rate >= 80: + success_color = colors.YELLOW + else: + success_color = colors.RED + + print( + f"{colors.CYAN}Attempts:{colors.RESET} {total_attempts} | " + f"{colors.CYAN}Success:{colors.RESET} {success_color}{success_rate:.1f}%{colors.RESET} | " + f"{colors.CYAN}Rate:{colors.RESET} {colors.MAGENTA}{work_rate:.1f}Hz{colors.RESET} | " + f"{colors.CYAN}Avg:{colors.RESET} {colors.YELLOW}{avg_work_time_us:.2f}µs{colors.RESET}" + ) + except KeyboardInterrupt: + print(f"\n{colors.YELLOW}Benchmark interrupted by user{colors.RESET}") + + actual_end_time = time.perf_counter() + wall_time = actual_end_time - start_time + total_attempts = sample_count + fail_count + + # Return final statistics + return { + "wall_time": wall_time, + "total_attempts": total_attempts, + "sample_count": sample_count, + "fail_count": fail_count, + "success_rate": ( + (sample_count / total_attempts) * 100 if total_attempts > 0 else 0 + ), + "total_work_time": total_work_time, + "avg_work_time_us": ( + (total_work_time / total_attempts) * 1e6 if total_attempts > 0 else 0 + ), + "work_rate_hz": total_attempts / total_work_time if total_work_time > 0 else 0, + "samples_per_sec": sample_count / wall_time if wall_time > 0 else 0, + } + + +def print_benchmark_results(results): + """Print comprehensive benchmark results""" + colors = get_colors(can_colorize()) + + print(f"\n{colors.BOLD_GREEN}{'='*60}{colors.RESET}") + print(f"{colors.BOLD_GREEN}get_stack_trace() Benchmark Results{colors.RESET}") + print(f"{colors.BOLD_GREEN}{'='*60}{colors.RESET}") + + # Basic statistics + print(f"\n{colors.BOLD_CYAN}Basic Statistics:{colors.RESET}") + print( + f" {colors.CYAN}Wall time:{colors.RESET} {colors.YELLOW}{results['wall_time']:.3f}{colors.RESET} seconds" + ) + print( + f" {colors.CYAN}Total attempts:{colors.RESET} {colors.MAGENTA}{results['total_attempts']:,}{colors.RESET}" + ) + print( + f" {colors.CYAN}Successful samples:{colors.RESET} {colors.GREEN}{results['sample_count']:,}{colors.RESET}" + ) + print( + f" {colors.CYAN}Failed samples:{colors.RESET} {colors.RED}{results['fail_count']:,}{colors.RESET}" + ) + + # Color code the success rate + success_rate = results["success_rate"] + if success_rate >= 95: + success_color = colors.BOLD_GREEN + elif success_rate >= 80: + success_color = colors.BOLD_YELLOW + else: + success_color = colors.BOLD_RED + + print( + f" {colors.CYAN}Success rate:{colors.RESET} {success_color}{success_rate:.2f}%{colors.RESET}" + ) + + # Performance metrics + print(f"\n{colors.BOLD_CYAN}Performance Metrics:{colors.RESET}") + print( + f" {colors.CYAN}Average call time:{colors.RESET} {colors.YELLOW}{results['avg_work_time_us']:.2f}{colors.RESET} µs" + ) + print( + f" {colors.CYAN}Work rate:{colors.RESET} {colors.MAGENTA}{results['work_rate_hz']:.1f}{colors.RESET} calls/sec" + ) + print( + f" {colors.CYAN}Sample rate:{colors.RESET} {colors.MAGENTA}{results['samples_per_sec']:.1f}{colors.RESET} samples/sec" + ) + print( + f" {colors.CYAN}Total work time:{colors.RESET} {colors.YELLOW}{results['total_work_time']:.3f}{colors.RESET} seconds" + ) + + # Color code work efficiency + efficiency = (results["total_work_time"] / results["wall_time"]) * 100 + if efficiency >= 80: + efficiency_color = colors.GREEN + elif efficiency >= 50: + efficiency_color = colors.YELLOW + else: + efficiency_color = colors.RED + + print( + f" {colors.CYAN}Work efficiency:{colors.RESET} {efficiency_color}{efficiency:.1f}%{colors.RESET}" + ) + + +def parse_arguments(): + """Parse command line arguments""" + # Build the code examples description + examples_desc = "\n".join( + [f" {name}: {info['description']}" for name, info in CODE_EXAMPLES.items()] + ) + + parser = argparse.ArgumentParser( + description="Benchmark get_stack_trace() performance", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=f""" +Examples: + %(prog)s # Run basic benchmark for 10 seconds (default) + %(prog)s --duration 30 # Run basic benchmark for 30 seconds + %(prog)s -d 60 # Run basic benchmark for 60 seconds + %(prog)s --code deep_static # Run deep static call stack benchmark + %(prog)s --code deep_static -d 30 # Run deep static benchmark for 30 seconds + +Available code examples: +{examples_desc} + """, + color=True, + ) + + parser.add_argument( + "--duration", + "-d", + type=int, + default=10, + help="Benchmark duration in seconds (default: 10)", + ) + + parser.add_argument( + "--code", + "-c", + choices=list(CODE_EXAMPLES.keys()), + default="basic", + help="Code example to benchmark (default: basic)", + ) + + return parser.parse_args() + + +def create_target_process(temp_file, code_example="basic"): + """Create and start the target process for benchmarking""" + example_info = CODE_EXAMPLES.get(code_example, {"code": CODE}) + selected_code = example_info["code"] + temp_file.write(selected_code) + temp_file.flush() + + process = subprocess.Popen( + [sys.executable, temp_file.name], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + # Give it time to start + time.sleep(1.0) + + # Check if it's still running + if process.poll() is not None: + stdout, stderr = process.communicate() + raise RuntimeError( + f"Target process exited unexpectedly:\nSTDOUT: {stdout.decode()}\nSTDERR: {stderr.decode()}" + ) + + return process, temp_file.name + + +def cleanup_process(process, temp_file_path): + """Clean up the target process and temporary file""" + with contextlib.suppress(Exception): + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5.0) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def main(): + """Main benchmark function""" + colors = get_colors(can_colorize()) + args = parse_arguments() + + print(f"{colors.BOLD_MAGENTA}External Inspection Benchmark Tool{colors.RESET}") + print(f"{colors.BOLD_MAGENTA}{'=' * 34}{colors.RESET}") + + example_info = CODE_EXAMPLES.get(args.code, {"description": "Unknown"}) + print( + f"\n{colors.CYAN}Code Example:{colors.RESET} {colors.GREEN}{args.code}{colors.RESET}" + ) + print(f"{colors.CYAN}Description:{colors.RESET} {example_info['description']}") + print( + f"{colors.CYAN}Benchmark Duration:{colors.RESET} {colors.YELLOW}{args.duration}{colors.RESET} seconds" + ) + + process = None + temp_file_path = None + + try: + # Create target process + print(f"\n{colors.BLUE}Creating and starting target process...{colors.RESET}") + with tempfile.NamedTemporaryFile(mode="w", suffix=".py") as temp_file: + process, temp_file_path = create_target_process(temp_file, args.code) + print( + f"{colors.GREEN}Target process started with PID: {colors.BOLD_WHITE}{process.pid}{colors.RESET}" + ) + + # Run benchmark with specified duration + with process: + # Create unwinder and run benchmark + print(f"{colors.BLUE}Initializing unwinder...{colors.RESET}") + try: + unwinder = _remote_debugging.RemoteUnwinder( + process.pid, all_threads=True + ) + results = benchmark(unwinder, duration_seconds=args.duration) + finally: + cleanup_process(process, temp_file_path) + + # Print results + print_benchmark_results(results) + + except PermissionError as e: + print( + f"{colors.BOLD_RED}Error: Insufficient permissions to read stack trace: {e}{colors.RESET}" + ) + print( + f"{colors.YELLOW}Try running with appropriate privileges (e.g., sudo){colors.RESET}" + ) + return 1 + except Exception as e: + print(f"{colors.BOLD_RED}Error during benchmarking: {e}{colors.RESET}") + if process: + with contextlib.suppress(Exception): + stdout, stderr = process.communicate(timeout=1) + if stdout: + print( + f"{colors.CYAN}Process STDOUT:{colors.RESET} {stdout.decode()}" + ) + if stderr: + print( + f"{colors.RED}Process STDERR:{colors.RESET} {stderr.decode()}" + ) + raise + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From e9d845b41dca9ad84b76ef777d05e647a4b4d8cd Mon Sep 17 00:00:00 2001 From: Emma Smith Date: Wed, 28 May 2025 14:18:34 -0700 Subject: [PATCH 005/116] gh-134262: Add retries to downloads in PCbuild\get_external.py (GH-134820) --- PCbuild/get_external.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index 4ecc8925349c93..99aff63882f5ba 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -9,6 +9,25 @@ from urllib.request import urlretrieve +def retrieve_with_retries(download_location, output_path, reporthook, + max_retries=7): + """Download a file with exponential backoff retry and save to disk.""" + for attempt in range(max_retries): + try: + resp = urlretrieve( + download_location, + output_path, + reporthook=reporthook, + ) + except ConnectionError as ex: + if attempt == max_retries: + msg = f"Download from {download_location} failed." + raise OSError(msg) from ex + time.sleep(2.25**attempt) + else: + return resp + + def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): repo = f'cpython-{"bin" if binary else "source"}-deps' url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip' @@ -16,10 +35,10 @@ def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): if verbose: reporthook = print zip_dir.mkdir(parents=True, exist_ok=True) - filename, headers = urlretrieve( + filename, _headers = retrieve_with_retries( url, zip_dir / f'{commit_hash}.zip', - reporthook=reporthook, + reporthook ) return filename From e64395e8eb8d3a9e35e3e534e87d427ff27ab0a5 Mon Sep 17 00:00:00 2001 From: Emma Smith Date: Wed, 28 May 2025 16:15:39 -0700 Subject: [PATCH 006/116] gh-134262: Fix off by one errors in download retry functions (GH-134867) --- PCbuild/get_external.py | 2 +- Tools/build/generate_sbom.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index 99aff63882f5ba..8c1155c74a642c 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -12,7 +12,7 @@ def retrieve_with_retries(download_location, output_path, reporthook, max_retries=7): """Download a file with exponential backoff retry and save to disk.""" - for attempt in range(max_retries): + for attempt in range(max_retries + 1): try: resp = urlretrieve( download_location, diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index ecb7b54f6d8a13..df52f8de762a01 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -169,7 +169,7 @@ def download_with_retries(download_location: str, base_delay: float = 2.25, max_jitter: float = 1.0) -> typing.Any: """Download a file with exponential backoff retry.""" - for attempt in range(max_retries): + for attempt in range(max_retries + 1): try: resp = urllib.request.urlopen(download_location) except urllib.error.URLError as ex: From b783e1791b2076ef560104f8207156d75859c0a6 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Thu, 29 May 2025 14:16:20 +0100 Subject: [PATCH 007/116] gh-69011: : clarify & deduplicate `ctypes.create_*_buffer` docs (GH-132858) This adds a warning about the possibly-missing NUL terminator, but in a way that doesn't make it sound like a bug/wart. --- Doc/library/ctypes.rst | 52 +++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 5b733d5321e907..8e74c6c9dee744 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -2031,35 +2031,55 @@ Utility functions pointer. -.. function:: create_string_buffer(init_or_size, size=None) +.. function:: create_string_buffer(init, size=None) + create_string_buffer(size) This function creates a mutable character buffer. The returned object is a ctypes array of :class:`c_char`. - *init_or_size* must be an integer which specifies the size of the array, or a - bytes object which will be used to initialize the array items. + If *size* is given (and not ``None``), it must be an :class:`int`. + It specifies the size of the returned array. - If a bytes object is specified as first argument, the buffer is made one item - larger than its length so that the last element in the array is a NUL - termination character. An integer can be passed as second argument which allows - specifying the size of the array if the length of the bytes should not be used. + If the *init* argument is given, it must be :class:`bytes`. It is used + to initialize the array items. Bytes not initialized this way are + set to zero (NUL). + + If *size* is not given (or if it is ``None``), the buffer is made one element + larger than *init*, effectively adding a NUL terminator. + + If both arguments are given, *size* must not be less than ``len(init)``. + + .. warning:: + + If *size* is equal to ``len(init)``, a NUL terminator is + not added. Do not treat such a buffer as a C string. + + For example:: + + >>> bytes(create_string_buffer(2)) + b'\x00\x00' + >>> bytes(create_string_buffer(b'ab')) + b'ab\x00' + >>> bytes(create_string_buffer(b'ab', 2)) + b'ab' + >>> bytes(create_string_buffer(b'ab', 4)) + b'ab\x00\x00' + >>> bytes(create_string_buffer(b'abcdef', 2)) + Traceback (most recent call last): + ... + ValueError: byte string too long .. audit-event:: ctypes.create_string_buffer init,size ctypes.create_string_buffer -.. function:: create_unicode_buffer(init_or_size, size=None) +.. function:: create_unicode_buffer(init, size=None) + create_unicode_buffer(size) This function creates a mutable unicode character buffer. The returned object is a ctypes array of :class:`c_wchar`. - *init_or_size* must be an integer which specifies the size of the array, or a - string which will be used to initialize the array items. - - If a string is specified as first argument, the buffer is made one item - larger than the length of the string so that the last element in the array is a - NUL termination character. An integer can be passed as second argument which - allows specifying the size of the array if the length of the string should not - be used. + The function takes the same arguments as :func:`~create_string_buffer` except + *init* must be a string and *size* counts :class:`c_wchar`. .. audit-event:: ctypes.create_unicode_buffer init,size ctypes.create_unicode_buffer From cafbcd666a7488e9f72553708350b7091dc913fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 29 May 2025 15:28:57 +0200 Subject: [PATCH 008/116] gh-133866: remove deprecated and undocumented function `ctypes.SetPointerType` (GH-133869) --- Doc/whatsnew/3.15.rst | 8 ++++++++ Lib/ctypes/__init__.py | 6 ------ Lib/test/test_ctypes/test_incomplete.py | 9 +++------ .../2025-05-11-10-01-48.gh-issue-133866.g3dHP_.rst | 3 +++ 4 files changed, 14 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-11-10-01-48.gh-issue-133866.g3dHP_.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 87cca4eeff385a..ced9c63071a53c 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -138,6 +138,14 @@ Deprecated Removed ======= +ctypes +------ + +* Removed the undocumented function :func:`!ctypes.SetPointerType`, + which has been deprecated since Python 3.13. + (Contributed by Bénédikt Tran in :gh:`133866`.) + + http.server ----------- diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 823a3692fd1bbf..d6d07a13f756e2 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -379,12 +379,6 @@ def create_unicode_buffer(init, size=None): return buf raise TypeError(init) - -def SetPointerType(pointer, cls): - import warnings - warnings._deprecated("ctypes.SetPointerType", remove=(3, 15)) - pointer.set_type(cls) - def ARRAY(typ, len): return typ * len diff --git a/Lib/test/test_ctypes/test_incomplete.py b/Lib/test/test_ctypes/test_incomplete.py index fefdfe9102e668..2f344611995b2c 100644 --- a/Lib/test/test_ctypes/test_incomplete.py +++ b/Lib/test/test_ctypes/test_incomplete.py @@ -21,9 +21,7 @@ class cell(Structure): _fields_ = [("name", c_char_p), ("next", lpcell)] - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - ctypes.SetPointerType(lpcell, cell) + lpcell.set_type(cell) self.assertIs(POINTER(cell), lpcell) @@ -50,10 +48,9 @@ class cell(Structure): _fields_ = [("name", c_char_p), ("next", lpcell)] - with self.assertWarns(DeprecationWarning): - ctypes.SetPointerType(lpcell, cell) - + lpcell.set_type(cell) self.assertIs(POINTER(cell), lpcell) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-05-11-10-01-48.gh-issue-133866.g3dHP_.rst b/Misc/NEWS.d/next/Library/2025-05-11-10-01-48.gh-issue-133866.g3dHP_.rst new file mode 100644 index 00000000000000..00f13c9a305eb5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-11-10-01-48.gh-issue-133866.g3dHP_.rst @@ -0,0 +1,3 @@ +Remove the undocumented function :func:`!ctypes.SetPointerType`, +which has been deprecated since Python 3.13. +Patch by Bénédikt Tran. From 4109a9c6b33faa0032ffc95d96cd0db482af3ce2 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 29 May 2025 15:36:59 +0200 Subject: [PATCH 009/116] gh-133260: Remove claim that PyUnicode_InternFromString immortalizes (GH-134213) --- Doc/c-api/unicode.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index cdd90d05b70b36..684432da81c61f 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1726,10 +1726,6 @@ They all return ``NULL`` or ``-1`` if an exception occurs. from user input, prefer calling :c:func:`PyUnicode_FromString` and :c:func:`PyUnicode_InternInPlace` directly. - .. impl-detail:: - - Strings interned this way are made :term:`immortal`. - .. c:function:: unsigned int PyUnicode_CHECK_INTERNED(PyObject *str) From f49a07b531543dd8a42d90f5b1c89c0312fbf806 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 29 May 2025 16:54:30 +0200 Subject: [PATCH 010/116] gh-133968: Add PyUnicodeWriter_WriteASCII() function (#133973) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace most PyUnicodeWriter_WriteUTF8() calls with PyUnicodeWriter_WriteASCII(). Unrelated change to please the linter: remove an unused import in test_ctypes. Co-authored-by: Peter Bierma Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/c-api/unicode.rst | 17 +++++++++++++- Doc/whatsnew/3.15.rst | 7 ++++++ Include/cpython/unicodeobject.h | 4 ++++ Lib/test/test_capi/test_unicode.py | 7 ++++++ Lib/test/test_ctypes/test_incomplete.py | 1 - ...-05-13-16-06-46.gh-issue-133968.6alWst.rst | 4 ++++ Modules/_json.c | 10 ++++----- Modules/_ssl.c | 2 +- Modules/_testcapi/unicode.c | 22 +++++++++++++++++++ Objects/genericaliasobject.c | 6 ++--- Objects/typevarobject.c | 4 ++-- Objects/unicodeobject.c | 14 ++++++++++++ Objects/unionobject.c | 8 +++---- Parser/asdl_c.py | 6 ++--- Python/Python-ast.c | 6 ++--- Python/context.c | 10 ++++----- Python/hamt.c | 6 ++--- 17 files changed, 103 insertions(+), 31 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-05-13-16-06-46.gh-issue-133968.6alWst.rst diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 684432da81c61f..ef180464ef1688 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1802,9 +1802,24 @@ object. See also :c:func:`PyUnicodeWriter_DecodeUTF8Stateful`. +.. c:function:: int PyUnicodeWriter_WriteASCII(PyUnicodeWriter *writer, const char *str, Py_ssize_t size) + + Write the ASCII string *str* into *writer*. + + *size* is the string length in bytes. If *size* is equal to ``-1``, call + ``strlen(str)`` to get the string length. + + *str* must only contain ASCII characters. The behavior is undefined if + *str* contains non-ASCII characters. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + + .. versionadded:: next + .. c:function:: int PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t size) - Writer the wide string *str* into *writer*. + Write the wide string *str* into *writer*. *size* is a number of wide characters. If *size* is equal to ``-1``, call ``wcslen(str)`` to get the string length. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index ced9c63071a53c..6d1f653f086a15 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -226,6 +226,13 @@ New features functions as replacements for :c:func:`PySys_GetObject`. (Contributed by Serhiy Storchaka in :gh:`108512`.) +* Add :c:func:`PyUnicodeWriter_WriteASCII` function to write an ASCII string + into a :c:type:`PyUnicodeWriter`. The function is faster than + :c:func:`PyUnicodeWriter_WriteUTF8`, but has an undefined behavior if the + input string contains non-ASCII characters. + (Contributed by Victor Stinner in :gh:`133968`.) + + Porting to Python 3.15 ---------------------- diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 136f5d5c5f8425..3d0414f5291fe4 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -478,6 +478,10 @@ PyAPI_FUNC(int) PyUnicodeWriter_WriteUTF8( PyUnicodeWriter *writer, const char *str, Py_ssize_t size); +PyAPI_FUNC(int) PyUnicodeWriter_WriteASCII( + PyUnicodeWriter *writer, + const char *str, + Py_ssize_t size); PyAPI_FUNC(int) PyUnicodeWriter_WriteWideChar( PyUnicodeWriter *writer, const wchar_t *str, diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 3408c10f426058..c8be4f3faa9483 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -1776,6 +1776,13 @@ def test_utf8(self): self.assertEqual(writer.finish(), "ascii-latin1=\xE9-euro=\u20AC.") + def test_ascii(self): + writer = self.create_writer(0) + writer.write_ascii(b"Hello ", -1) + writer.write_ascii(b"", 0) + writer.write_ascii(b"Python! ", 6) + self.assertEqual(writer.finish(), "Hello Python") + def test_invalid_utf8(self): writer = self.create_writer(0) with self.assertRaises(UnicodeDecodeError): diff --git a/Lib/test/test_ctypes/test_incomplete.py b/Lib/test/test_ctypes/test_incomplete.py index 2f344611995b2c..3189fcd1bd1330 100644 --- a/Lib/test/test_ctypes/test_incomplete.py +++ b/Lib/test/test_ctypes/test_incomplete.py @@ -1,6 +1,5 @@ import ctypes import unittest -import warnings from ctypes import Structure, POINTER, pointer, c_char_p # String-based "incomplete pointers" were implemented in ctypes 0.6.3 (2003, when diff --git a/Misc/NEWS.d/next/C_API/2025-05-13-16-06-46.gh-issue-133968.6alWst.rst b/Misc/NEWS.d/next/C_API/2025-05-13-16-06-46.gh-issue-133968.6alWst.rst new file mode 100644 index 00000000000000..47d5a3bda39942 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-05-13-16-06-46.gh-issue-133968.6alWst.rst @@ -0,0 +1,4 @@ +Add :c:func:`PyUnicodeWriter_WriteASCII` function to write an ASCII string +into a :c:type:`PyUnicodeWriter`. The function is faster than +:c:func:`PyUnicodeWriter_WriteUTF8`, but has an undefined behavior if the +input string contains non-ASCII characters. Patch by Victor Stinner. diff --git a/Modules/_json.c b/Modules/_json.c index 89b0a41dd10acb..4aa6ae650651b3 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1476,13 +1476,13 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, int rv; if (obj == Py_None) { - return PyUnicodeWriter_WriteUTF8(writer, "null", 4); + return PyUnicodeWriter_WriteASCII(writer, "null", 4); } else if (obj == Py_True) { - return PyUnicodeWriter_WriteUTF8(writer, "true", 4); + return PyUnicodeWriter_WriteASCII(writer, "true", 4); } else if (obj == Py_False) { - return PyUnicodeWriter_WriteUTF8(writer, "false", 5); + return PyUnicodeWriter_WriteASCII(writer, "false", 5); } else if (PyUnicode_Check(obj)) { PyObject *encoded = encoder_encode_string(s, obj); @@ -1649,7 +1649,7 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, if (PyDict_GET_SIZE(dct) == 0) { /* Fast path */ - return PyUnicodeWriter_WriteUTF8(writer, "{}", 2); + return PyUnicodeWriter_WriteASCII(writer, "{}", 2); } if (s->markers != Py_None) { @@ -1753,7 +1753,7 @@ encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, return -1; if (PySequence_Fast_GET_SIZE(s_fast) == 0) { Py_DECREF(s_fast); - return PyUnicodeWriter_WriteUTF8(writer, "[]", 2); + return PyUnicodeWriter_WriteASCII(writer, "[]", 2); } if (s->markers != Py_None) { diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 976da1340ecf1e..014e624f6c2f00 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -563,7 +563,7 @@ fill_and_set_sslerror(_sslmodulestate *state, goto fail; } } - if (PyUnicodeWriter_WriteUTF8(writer, "] ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "] ", 2) < 0) { goto fail; } } diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index b8ecf53f4f8b9c..e70f5c68bc3b69 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -332,6 +332,27 @@ writer_write_utf8(PyObject *self_raw, PyObject *args) } +static PyObject* +writer_write_ascii(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + char *str; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "yn", &str, &size)) { + return NULL; + } + + if (PyUnicodeWriter_WriteASCII(self->writer, str, size) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + static PyObject* writer_write_widechar(PyObject *self_raw, PyObject *args) { @@ -513,6 +534,7 @@ writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) static PyMethodDef writer_methods[] = { {"write_char", _PyCFunction_CAST(writer_write_char), METH_VARARGS}, {"write_utf8", _PyCFunction_CAST(writer_write_utf8), METH_VARARGS}, + {"write_ascii", _PyCFunction_CAST(writer_write_ascii), METH_VARARGS}, {"write_widechar", _PyCFunction_CAST(writer_write_widechar), METH_VARARGS}, {"write_ucs4", _PyCFunction_CAST(writer_write_ucs4), METH_VARARGS}, {"write_str", _PyCFunction_CAST(writer_write_str), METH_VARARGS}, diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index ec3d01f00a3c3c..07b57f0c552ce9 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -65,7 +65,7 @@ ga_repr_items_list(PyUnicodeWriter *writer, PyObject *p) for (Py_ssize_t i = 0; i < len; i++) { if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { return -1; } } @@ -109,7 +109,7 @@ ga_repr(PyObject *self) } for (Py_ssize_t i = 0; i < len; i++) { if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { goto error; } } @@ -126,7 +126,7 @@ ga_repr(PyObject *self) } if (len == 0) { // for something like tuple[()] we should print a "()" - if (PyUnicodeWriter_WriteUTF8(writer, "()", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "()", 2) < 0) { goto error; } } diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 6c199a52aa0ae6..cead6e69af5451 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -192,7 +192,7 @@ constevaluator_call(PyObject *self, PyObject *args, PyObject *kwargs) for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(value); i++) { PyObject *item = PyTuple_GET_ITEM(value, i); if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { PyUnicodeWriter_Discard(writer); return NULL; } @@ -273,7 +273,7 @@ _Py_typing_type_repr(PyUnicodeWriter *writer, PyObject *p) } if (p == (PyObject *)&_PyNone_Type) { - return PyUnicodeWriter_WriteUTF8(writer, "None", 4); + return PyUnicodeWriter_WriteASCII(writer, "None", 4); } if ((rc = PyObject_HasAttrWithError(p, &_Py_ID(__origin__))) > 0 && diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index aa94fb91e65fc3..5611f839627a2e 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -14083,6 +14083,20 @@ _PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, return 0; } + +int +PyUnicodeWriter_WriteASCII(PyUnicodeWriter *writer, + const char *str, + Py_ssize_t size) +{ + assert(writer != NULL); + _Py_AssertHoldsTstate(); + + _PyUnicodeWriter *priv_writer = (_PyUnicodeWriter*)writer; + return _PyUnicodeWriter_WriteASCIIString(priv_writer, str, size); +} + + int PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, const char *str, diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 66435924b6c6c3..00ca5b9bf80341 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -290,7 +290,7 @@ union_repr(PyObject *self) } for (Py_ssize_t i = 0; i < len; i++) { - if (i > 0 && PyUnicodeWriter_WriteUTF8(writer, " | ", 3) < 0) { + if (i > 0 && PyUnicodeWriter_WriteASCII(writer, " | ", 3) < 0) { goto error; } PyObject *p = PyTuple_GET_ITEM(alias->args, i); @@ -300,12 +300,12 @@ union_repr(PyObject *self) } #if 0 - PyUnicodeWriter_WriteUTF8(writer, "|args=", 6); + PyUnicodeWriter_WriteASCII(writer, "|args=", 6); PyUnicodeWriter_WriteRepr(writer, alias->args); - PyUnicodeWriter_WriteUTF8(writer, "|h=", 3); + PyUnicodeWriter_WriteASCII(writer, "|h=", 3); PyUnicodeWriter_WriteRepr(writer, alias->hashable_args); if (alias->unhashable_args) { - PyUnicodeWriter_WriteUTF8(writer, "|u=", 3); + PyUnicodeWriter_WriteASCII(writer, "|u=", 3); PyUnicodeWriter_WriteRepr(writer, alias->unhashable_args); } #endif diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 22dcfe1b0d99bf..dba20226c3283a 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1512,7 +1512,7 @@ def visitModule(self, mod): for (Py_ssize_t i = 0; i < Py_MIN(length, 2); i++) { if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { goto error; } } @@ -1536,7 +1536,7 @@ def visitModule(self, mod): } if (i == 0 && length > 2) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ...", 5) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ...", 5) < 0) { goto error; } } @@ -1640,7 +1640,7 @@ def visitModule(self, mod): } if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { Py_DECREF(name); Py_DECREF(value_repr); goto error; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index f7625ab1205bdc..660bc598a4862c 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -5796,7 +5796,7 @@ ast_repr_list(PyObject *list, int depth) for (Py_ssize_t i = 0; i < Py_MIN(length, 2); i++) { if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { goto error; } } @@ -5820,7 +5820,7 @@ ast_repr_list(PyObject *list, int depth) } if (i == 0 && length > 2) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ...", 5) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ...", 5) < 0) { goto error; } } @@ -5924,7 +5924,7 @@ ast_repr_max_depth(AST_object *self, int depth) } if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { Py_DECREF(name); Py_DECREF(value_repr); goto error; diff --git a/Python/context.c b/Python/context.c index dceaae9b42979d..9927cab915cae7 100644 --- a/Python/context.c +++ b/Python/context.c @@ -979,7 +979,7 @@ contextvar_tp_repr(PyObject *op) return NULL; } - if (PyUnicodeWriter_WriteUTF8(writer, "tok_used) { - if (PyUnicodeWriter_WriteUTF8(writer, " used", 5) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, " used", 5) < 0) { goto error; } } - if (PyUnicodeWriter_WriteUTF8(writer, " var=", 5) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, " var=", 5) < 0) { goto error; } if (PyUnicodeWriter_WriteRepr(writer, (PyObject *)self->tok_var) < 0) { diff --git a/Python/hamt.c b/Python/hamt.c index f9bbf63961d8de..906149cc6cdbdc 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -1176,7 +1176,7 @@ hamt_node_bitmap_dump(PyHamtNode_Bitmap *node, } if (key_or_null == NULL) { - if (PyUnicodeWriter_WriteUTF8(writer, "NULL:\n", -1) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "NULL:\n", 6) < 0) { goto error; } @@ -1194,7 +1194,7 @@ hamt_node_bitmap_dump(PyHamtNode_Bitmap *node, } } - if (PyUnicodeWriter_WriteUTF8(writer, "\n", 1) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "\n", 1) < 0) { goto error; } } @@ -1915,7 +1915,7 @@ hamt_node_array_dump(PyHamtNode_Array *node, goto error; } - if (PyUnicodeWriter_WriteUTF8(writer, "\n", 1) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "\n", 1) < 0) { goto error; } } From d96343679fd6137c9d87d1bb120228b162ea0f8c Mon Sep 17 00:00:00 2001 From: Collin Funk Date: Thu, 29 May 2025 10:35:53 -0700 Subject: [PATCH 011/116] gh-134771: Fix time_clockid_converter() on Cygwin (#134772) Use long for clockid_t instead of int. --- .../next/Library/2025-05-26-22-18-32.gh-issue-134771.RKXpLT.rst | 2 ++ Modules/timemodule.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-26-22-18-32.gh-issue-134771.RKXpLT.rst diff --git a/Misc/NEWS.d/next/Library/2025-05-26-22-18-32.gh-issue-134771.RKXpLT.rst b/Misc/NEWS.d/next/Library/2025-05-26-22-18-32.gh-issue-134771.RKXpLT.rst new file mode 100644 index 00000000000000..4b70c6ef398793 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-26-22-18-32.gh-issue-134771.RKXpLT.rst @@ -0,0 +1,2 @@ +The ``time_clockid_converter()`` function now selects correct type for +``clockid_t`` on Cygwin which fixes a build error. diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 1bfbf3f6a0b991..3271d87ddc27f2 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -187,7 +187,7 @@ time_clockid_converter(PyObject *obj, clockid_t *p) { #ifdef _AIX long long clk_id = PyLong_AsLongLong(obj); -#elif defined(__DragonFly__) +#elif defined(__DragonFly__) || defined(__CYGWIN__) long clk_id = PyLong_AsLong(obj); #else int clk_id = PyLong_AsInt(obj); From dafd14146f7ca18932894ea445a2f9f98f2a8b01 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 29 May 2025 14:13:12 -0600 Subject: [PATCH 012/116] gh-132775: Fix _PyFunctIon_VerifyStateless() (#134900) The problem we're fixing here is that we were using PyDict_Size() on "defaults", which it is actually a tuple. We're also adding some explicit type checks. This is a follow-up to gh-133221/gh-133528. --- Objects/funcobject.c | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 27214a129c2fb8..f87b0e5d8f1e47 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -1264,26 +1264,32 @@ _PyFunction_VerifyStateless(PyThreadState *tstate, PyObject *func) } // Disallow __defaults__. PyObject *defaults = PyFunction_GET_DEFAULTS(func); - if (defaults != NULL && defaults != Py_None && PyDict_Size(defaults) > 0) - { - _PyErr_SetString(tstate, PyExc_ValueError, "defaults not supported"); - return -1; + if (defaults != NULL) { + assert(PyTuple_Check(defaults)); // per PyFunction_New() + if (PyTuple_GET_SIZE(defaults) > 0) { + _PyErr_SetString(tstate, PyExc_ValueError, + "defaults not supported"); + return -1; + } } // Disallow __kwdefaults__. PyObject *kwdefaults = PyFunction_GET_KW_DEFAULTS(func); - if (kwdefaults != NULL && kwdefaults != Py_None - && PyDict_Size(kwdefaults) > 0) - { - _PyErr_SetString(tstate, PyExc_ValueError, - "keyword defaults not supported"); - return -1; + if (kwdefaults != NULL) { + assert(PyDict_Check(kwdefaults)); // per PyFunction_New() + if (PyDict_Size(kwdefaults) > 0) { + _PyErr_SetString(tstate, PyExc_ValueError, + "keyword defaults not supported"); + return -1; + } } // Disallow __closure__. PyObject *closure = PyFunction_GET_CLOSURE(func); - if (closure != NULL && closure != Py_None && PyTuple_GET_SIZE(closure) > 0) - { - _PyErr_SetString(tstate, PyExc_ValueError, "closures not supported"); - return -1; + if (closure != NULL) { + assert(PyTuple_Check(closure)); // per PyFunction_New() + if (PyTuple_GET_SIZE(closure) > 0) { + _PyErr_SetString(tstate, PyExc_ValueError, "closures not supported"); + return -1; + } } // Check the code. PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); From cb8a72b301f47e76d93a7fe5b259e9a5758792e1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 30 May 2025 00:32:44 +0300 Subject: [PATCH 013/116] gh-134857: Improve error report for doctests run with unittest (GH-134858) Remove doctest module frames from tracebacks and redundant newline character from a failure message. --- Lib/doctest.py | 24 ++-- Lib/test/test_doctest/test_doctest.py | 130 +----------------- ...-05-28-20-49-29.gh-issue-134857.dVYXVO.rst | 3 + 3 files changed, 22 insertions(+), 135 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-28-20-49-29.gh-issue-134857.dVYXVO.rst diff --git a/Lib/doctest.py b/Lib/doctest.py index 2acb6cb79f394d..dec10a345165da 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -108,6 +108,8 @@ def _test(): from _colorize import ANSIColors, can_colorize +__unittest = True + class TestResults(namedtuple('TestResults', 'failed attempted')): def __new__(cls, failed, attempted, *, skipped=0): results = super().__new__(cls, failed, attempted) @@ -1395,11 +1397,11 @@ def __run(self, test, compileflags, out): exec(compile(example.source, filename, "single", compileflags, True), test.globs) self.debugger.set_continue() # ==== Example Finished ==== - exception = None + exc_info = None except KeyboardInterrupt: raise - except: - exception = sys.exc_info() + except BaseException as exc: + exc_info = type(exc), exc, exc.__traceback__.tb_next self.debugger.set_continue() # ==== Example Finished ==== got = self._fakeout.getvalue() # the actual output @@ -1408,21 +1410,21 @@ def __run(self, test, compileflags, out): # If the example executed without raising any exceptions, # verify its output. - if exception is None: + if exc_info is None: if check(example.want, got, self.optionflags): outcome = SUCCESS # The example raised an exception: check if it was expected. else: - formatted_ex = traceback.format_exception_only(*exception[:2]) - if issubclass(exception[0], SyntaxError): + formatted_ex = traceback.format_exception_only(*exc_info[:2]) + if issubclass(exc_info[0], SyntaxError): # SyntaxError / IndentationError is special: # we don't care about the carets / suggestions / etc # We only care about the error message and notes. # They start with `SyntaxError:` (or any other class name) exception_line_prefixes = ( - f"{exception[0].__qualname__}:", - f"{exception[0].__module__}.{exception[0].__qualname__}:", + f"{exc_info[0].__qualname__}:", + f"{exc_info[0].__module__}.{exc_info[0].__qualname__}:", ) exc_msg_index = next( index @@ -1433,7 +1435,7 @@ def __run(self, test, compileflags, out): exc_msg = "".join(formatted_ex) if not quiet: - got += _exception_traceback(exception) + got += _exception_traceback(exc_info) # If `example.exc_msg` is None, then we weren't expecting # an exception. @@ -1462,7 +1464,7 @@ def __run(self, test, compileflags, out): elif outcome is BOOM: if not quiet: self.report_unexpected_exception(out, test, example, - exception) + exc_info) failures += 1 else: assert False, ("unknown outcome", outcome) @@ -2324,7 +2326,7 @@ def runTest(self): sys.stdout = old if results.failed: - raise self.failureException(self.format_failure(new.getvalue())) + raise self.failureException(self.format_failure(new.getvalue().rstrip('\n'))) def format_failure(self, err): test = self._dt_test diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index c5b247797c321d..2bfaa6c599cd47 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -2411,9 +2411,6 @@ def test_DocTestSuite_errors(): >>> result >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - Traceback (most recent call last): - File ... - raise self.failureException(self.format_failure(new.getvalue())) AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors File "...sample_doctest_errors.py", line 0, in sample_doctest_errors @@ -2431,21 +2428,12 @@ def test_DocTestSuite_errors(): 1/0 Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in 1/0 ~^~ ZeroDivisionError: division by zero - >>> print(result.failures[1][1]) # doctest: +ELLIPSIS - Traceback (most recent call last): - File ... - raise self.failureException(self.format_failure(new.getvalue())) AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.__test__.bad File "...sample_doctest_errors.py", line unknown line number, in bad @@ -2463,21 +2451,12 @@ def test_DocTestSuite_errors(): 1/0 Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in 1/0 ~^~ ZeroDivisionError: division by zero - >>> print(result.failures[2][1]) # doctest: +ELLIPSIS - Traceback (most recent call last): - File ... - raise self.failureException(self.format_failure(new.getvalue())) AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.errors File "...sample_doctest_errors.py", line 14, in errors @@ -2495,11 +2474,6 @@ def test_DocTestSuite_errors(): 1/0 Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in 1/0 ~^~ @@ -2510,11 +2484,6 @@ def test_DocTestSuite_errors(): f() Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in f() ~^^ @@ -2528,11 +2497,6 @@ def test_DocTestSuite_errors(): g() Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in g() ~^^ @@ -2541,11 +2505,7 @@ def test_DocTestSuite_errors(): ~~^^^ IndexError: list index out of range - >>> print(result.failures[3][1]) # doctest: +ELLIPSIS - Traceback (most recent call last): - File ... - raise self.failureException(self.format_failure(new.getvalue())) AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.syntax_error File "...sample_doctest_errors.py", line 29, in syntax_error @@ -2554,18 +2514,11 @@ def test_DocTestSuite_errors(): Failed example: 2+*3 Exception raised: - Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^ File "", line 1 2+*3 ^ SyntaxError: invalid syntax - """ def test_DocFileSuite(): @@ -2740,9 +2693,6 @@ def test_DocFileSuite_errors(): >>> result >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - Traceback (most recent call last): - File ... - raise self.failureException(self.format_failure(new.getvalue())) AssertionError: Failed doctest test for test_doctest_errors.txt File "...test_doctest_errors.txt", line 0 @@ -2760,11 +2710,6 @@ def test_DocFileSuite_errors(): 1/0 Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in 1/0 ~^~ @@ -2775,11 +2720,6 @@ def test_DocFileSuite_errors(): f() Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in f() ~^^ @@ -2792,18 +2732,11 @@ def test_DocFileSuite_errors(): Failed example: 2+*3 Exception raised: - Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^ File "", line 1 2+*3 ^ SyntaxError: invalid syntax - """ def test_trailing_space_in_test(): @@ -2876,7 +2809,8 @@ def test_unittest_reportflags(): >>> result >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - Traceback ... + AssertionError: Failed doctest test for test_doctest.txt + ... Failed example: favorite_color ... @@ -2895,14 +2829,14 @@ def test_unittest_reportflags(): >>> result >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - Traceback ... + AssertionError: Failed doctest test for test_doctest.txt + ... Failed example: favorite_color Exception raised: ... NameError: name 'favorite_color' is not defined - We get only the first failure. @@ -2922,7 +2856,8 @@ def test_unittest_reportflags(): the trailing whitespace using `\x20` in the diff below. >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - Traceback ... + AssertionError: Failed doctest test for test_doctest.txt + ... Failed example: favorite_color ... @@ -2937,7 +2872,6 @@ def test_unittest_reportflags(): +\x20 b - Test runners can restore the formatting flags after they run: @@ -3145,11 +3079,6 @@ def test_testfile_errors(): r""" 1/0 Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in 1/0 ~^~ @@ -3160,11 +3089,6 @@ def test_testfile_errors(): r""" f() Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in f() ~^^ @@ -3177,12 +3101,6 @@ def test_testfile_errors(): r""" Failed example: 2+*3 Exception raised: - Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^ File "", line 1 2+*3 ^ @@ -3343,11 +3261,6 @@ def test_testmod_errors(): r""" 1/0 Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in 1/0 ~^~ @@ -3366,11 +3279,6 @@ def test_testmod_errors(): r""" 1/0 Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in 1/0 ~^~ @@ -3389,11 +3297,6 @@ def test_testmod_errors(): r""" 1/0 Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in 1/0 ~^~ @@ -3404,11 +3307,6 @@ def test_testmod_errors(): r""" f() Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in f() ~^^ @@ -3422,11 +3320,6 @@ def test_testmod_errors(): r""" g() Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in g() ~^^ @@ -3439,12 +3332,6 @@ def test_testmod_errors(): r""" Failed example: 2+*3 Exception raised: - Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^ File "", line 1 2+*3 ^ @@ -3490,11 +3377,6 @@ def test_unicode(): """ raise Exception('clé') Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in raise Exception('clé') Exception: clé diff --git a/Misc/NEWS.d/next/Library/2025-05-28-20-49-29.gh-issue-134857.dVYXVO.rst b/Misc/NEWS.d/next/Library/2025-05-28-20-49-29.gh-issue-134857.dVYXVO.rst new file mode 100644 index 00000000000000..92e38c0bb5ac87 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-28-20-49-29.gh-issue-134857.dVYXVO.rst @@ -0,0 +1,3 @@ +Improve error report for :mod:`doctest`\ s run with :mod:`unittest`. Remove +:mod:`!doctest` module frames from tracebacks and redundant newline +character from a failure message. From a4251411a97304ab001721c6231d86ddf4eac3f0 Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Thu, 29 May 2025 20:34:34 -0700 Subject: [PATCH 014/116] GH-106235: Clarify `parse_known_args` documentation by removing "remaining" (#126921) --- Doc/library/argparse.rst | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 29396c7a0366a1..17f126cc065a82 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -2122,12 +2122,15 @@ Partial parsing .. method:: ArgumentParser.parse_known_args(args=None, namespace=None) - Sometimes a script may only parse a few of the command-line arguments, passing - the remaining arguments on to another script or program. In these cases, the - :meth:`~ArgumentParser.parse_known_args` method can be useful. It works much like - :meth:`~ArgumentParser.parse_args` except that it does not produce an error when - extra arguments are present. Instead, it returns a two item tuple containing - the populated namespace and the list of remaining argument strings. + Sometimes a script only needs to handle a specific set of command-line + arguments, leaving any unrecognized arguments for another script or program. + In these cases, the :meth:`~ArgumentParser.parse_known_args` method can be + useful. + + This method works similarly to :meth:`~ArgumentParser.parse_args`, but it does + not raise an error for extra, unrecognized arguments. Instead, it parses the + known arguments and returns a two item tuple that contains the populated + namespace and the list of any unrecognized arguments. :: From 381020d41fb1f8b33421f01c609ba0d0edb99764 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 29 May 2025 21:11:20 -0700 Subject: [PATCH 015/116] ast docs: Fix description of ast.Constant (#134741) Contrary to the current docs, ast.Constant will never hold containers such as frozenset or tuple; the Python parser only emits it for simple literals. For precision, add the exact list of types that may be contained in an ast.Constant. --- Doc/library/ast.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index ca9a6b0712c9a2..cf22250cac6091 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -268,9 +268,9 @@ Literals .. class:: Constant(value) A constant value. The ``value`` attribute of the ``Constant`` literal contains the - Python object it represents. The values represented can be simple types - such as a number, string or ``None``, but also immutable container types - (tuples and frozensets) if all of their elements are constant. + Python object it represents. The values represented can be instances of :class:`str`, + :class:`bytes`, :class:`int`, :class:`float`, :class:`complex`, and :class:`bool`, + and the constants :data:`None` and :data:`Ellipsis`. .. doctest:: From 5f60d0fcccbf6676f5bc924f05452bd5321446f0 Mon Sep 17 00:00:00 2001 From: Emma Smith Date: Thu, 29 May 2025 21:37:43 -0700 Subject: [PATCH 016/116] gh-134906: Document CompressionParameter.content_size_flag (#134907) * Document CompressionParameter.content_size_flag --- Doc/library/compression.zstd.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Doc/library/compression.zstd.rst b/Doc/library/compression.zstd.rst index 1e1802155a19ec..35bcbc2bfd8eac 100644 --- a/Doc/library/compression.zstd.rst +++ b/Doc/library/compression.zstd.rst @@ -615,6 +615,24 @@ Advanced parameter control A value of zero causes the value to be selected automatically. + .. attribute:: content_size_flag + + Write the size of the data to be compressed into the Zstandard frame + header when known prior to compressing. + + This flag only takes effect under the following two scenarios: + + * Calling :func:`compress` for one-shot compression + * Providing all of the data to be compressed in the frame in a single + :meth:`ZstdCompressor.compress` call, with the + :attr:`ZstdCompressor.FLUSH_FRAME` mode. + + All other compression calls may not write the size information into the + frame header. + + ``True`` or ``1`` enable the content size flag while ``False`` or ``0`` + disable it. + .. attribute:: checksum_flag A four-byte checksum using XXHash64 of the uncompressed content is From 2f2bee21118adce653ee5bc4eb31d30327465966 Mon Sep 17 00:00:00 2001 From: Sam James Date: Fri, 30 May 2025 05:42:19 +0100 Subject: [PATCH 017/116] gh-134768: Fix definition of `mt_continue_should_break()` (#134769) In 121ed71f4e395948d313249b2ad33e1e21581f8a, mt_continue_should_break was changed to be guarded by `Py_DEBUG`, but it's used in `compress_mt_continue_lock_held` with just `assert`, so it needs to be available when `NDEBUG` is undefined too. `Py_DEBUG` implies `NDEBUG` is undefined, so we can check just that. Fixes: 121ed71f4e395948d313249b2ad33e1e21581f8a --- Modules/_zstd/compressor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 0fc3d7d36c68fe..8ff2a3aadc1cd6 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -512,7 +512,7 @@ compress_lock_held(ZstdCompressor *self, Py_buffer *data, return NULL; } -#ifdef Py_DEBUG +#ifndef NDEBUG static inline int mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out) { From b367e27af9b52528e395f95b277ec7b69e98e287 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 30 May 2025 17:59:23 +0900 Subject: [PATCH 018/116] Doc: remove unnecessary section header (GH-134917) --- Doc/c-api/unicode.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index ef180464ef1688..45f50ba5f97d26 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1461,10 +1461,6 @@ the user settings on the machine running the codec. .. versionadded:: 3.3 -Methods & Slots -""""""""""""""" - - .. _unicodemethodsandslots: Methods and Slot Functions From 45c6c48afc13f9897010e32171a3e02d0624258c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 30 May 2025 02:30:05 -0700 Subject: [PATCH 019/116] gh-134885: zstd: Use Py_XSETREF (GH-134886) --- .../2025-05-29-06-53-40.gh-issue-134885.-_L22o.rst | 2 ++ Modules/_zstd/_zstdmodule.c | 11 ++++------- 2 files changed, 6 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-29-06-53-40.gh-issue-134885.-_L22o.rst diff --git a/Misc/NEWS.d/next/Library/2025-05-29-06-53-40.gh-issue-134885.-_L22o.rst b/Misc/NEWS.d/next/Library/2025-05-29-06-53-40.gh-issue-134885.-_L22o.rst new file mode 100644 index 00000000000000..4b05d42c109d06 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-29-06-53-40.gh-issue-134885.-_L22o.rst @@ -0,0 +1,2 @@ +Fix possible crash in the :mod:`compression.zstd` module related to setting +parameter types. Patch by Jelle Zijlstra. diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 5ad697d2b83dd6..986b3579479f0f 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -514,13 +514,10 @@ _zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, return NULL; } - Py_XDECREF(mod_state->CParameter_type); - Py_INCREF(c_parameter_type); - mod_state->CParameter_type = (PyTypeObject*)c_parameter_type; - - Py_XDECREF(mod_state->DParameter_type); - Py_INCREF(d_parameter_type); - mod_state->DParameter_type = (PyTypeObject*)d_parameter_type; + Py_XSETREF( + mod_state->CParameter_type, (PyTypeObject*)Py_NewRef(c_parameter_type)); + Py_XSETREF( + mod_state->DParameter_type, (PyTypeObject*)Py_NewRef(d_parameter_type)); Py_RETURN_NONE; } From ebf6d13567287d04683dab36f52cde7a3c9915e7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 30 May 2025 12:15:47 +0200 Subject: [PATCH 020/116] gh-134745: Change PyThread_allocate_lock() implementation to PyMutex (#134747) Co-authored-by: Sam Gross --- Include/internal/pycore_lock.h | 3 + Lib/test/test_sys.py | 2 +- ...-05-30-11-33-17.gh-issue-134745.GN-zk2.rst | 3 + Python/lock.c | 3 + Python/thread.c | 82 +++- Python/thread_nt.h | 92 ---- Python/thread_pthread.h | 392 ------------------ 7 files changed, 86 insertions(+), 491 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-05-30-11-33-17.gh-issue-134745.GN-zk2.rst diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index 7484b05d7f2446..32b60cc33a21f1 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -48,6 +48,9 @@ typedef enum _PyLockFlags { // Handle signals if interrupted while waiting on the lock. _PY_LOCK_HANDLE_SIGNALS = 2, + + // Fail if interrupted by a signal while waiting on the lock. + _PY_FAIL_IF_INTERRUPTED = 4, } _PyLockFlags; // Lock a mutex with an optional timeout and additional options. See diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 795d1ecbb59f8f..65d15610ed1505 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -729,7 +729,7 @@ def test_thread_info(self): info = sys.thread_info self.assertEqual(len(info), 3) self.assertIn(info.name, ('nt', 'pthread', 'pthread-stubs', 'solaris', None)) - self.assertIn(info.lock, ('semaphore', 'mutex+cond', None)) + self.assertIn(info.lock, ('pymutex', None)) if sys.platform.startswith(("linux", "android", "freebsd")): self.assertEqual(info.name, "pthread") elif sys.platform == "win32": diff --git a/Misc/NEWS.d/next/C_API/2025-05-30-11-33-17.gh-issue-134745.GN-zk2.rst b/Misc/NEWS.d/next/C_API/2025-05-30-11-33-17.gh-issue-134745.GN-zk2.rst new file mode 100644 index 00000000000000..a85d2e90576a49 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-05-30-11-33-17.gh-issue-134745.GN-zk2.rst @@ -0,0 +1,3 @@ +Change :c:func:`!PyThread_allocate_lock` implementation to ``PyMutex``. +On Windows, :c:func:`!PyThread_acquire_lock_timed` now supports the *intr_flag* +parameter: it can be interrupted. Patch by Victor Stinner. diff --git a/Python/lock.c b/Python/lock.c index 28a12ad18352d1..b125ad0c9e356d 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -119,6 +119,9 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) return PY_LOCK_INTR; } } + else if (ret == Py_PARK_INTR && (flags & _PY_FAIL_IF_INTERRUPTED)) { + return PY_LOCK_INTR; + } else if (ret == Py_PARK_TIMEOUT) { assert(timeout >= 0); return PY_LOCK_FAILURE; diff --git a/Python/thread.c b/Python/thread.c index 4ff5f11a34852b..18c4af7f634c75 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -39,7 +39,8 @@ const long long PY_TIMEOUT_MAX = PY_TIMEOUT_MAX_VALUE; -static void PyThread__init_thread(void); /* Forward */ +/* Forward declaration */ +static void PyThread__init_thread(void); #define initialized _PyRuntime.threads.initialized @@ -71,6 +72,79 @@ PyThread_init_thread(void) #endif +/* + * Lock support. + */ + +PyThread_type_lock +PyThread_allocate_lock(void) +{ + if (!initialized) { + PyThread_init_thread(); + } + + PyMutex *lock = (PyMutex *)PyMem_RawMalloc(sizeof(PyMutex)); + if (lock) { + *lock = (PyMutex){0}; + } + + return (PyThread_type_lock)lock; +} + +void +PyThread_free_lock(PyThread_type_lock lock) +{ + PyMem_RawFree(lock); +} + +PyLockStatus +PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, + int intr_flag) +{ + PyTime_t timeout; // relative timeout + if (microseconds >= 0) { + // bpo-41710: PyThread_acquire_lock_timed() cannot report timeout + // overflow to the caller, so clamp the timeout to + // [PyTime_MIN, PyTime_MAX]. + // + // PyTime_MAX nanoseconds is around 292.3 years. + // + // _thread.Lock.acquire() and _thread.RLock.acquire() raise an + // OverflowError if microseconds is greater than PY_TIMEOUT_MAX. + timeout = _PyTime_FromMicrosecondsClamp(microseconds); + } + else { + timeout = -1; + } + + _PyLockFlags flags = _Py_LOCK_DONT_DETACH; + if (intr_flag) { + flags |= _PY_FAIL_IF_INTERRUPTED; + } + + return _PyMutex_LockTimed((PyMutex *)lock, timeout, flags); +} + +void +PyThread_release_lock(PyThread_type_lock lock) +{ + PyMutex_Unlock((PyMutex *)lock); +} + +int +_PyThread_at_fork_reinit(PyThread_type_lock *lock) +{ + _PyMutex_at_fork_reinit((PyMutex *)lock); + return 0; +} + +int +PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) +{ + return PyThread_acquire_lock_timed(lock, waitflag ? -1 : 0, /*intr_flag=*/0); +} + + /* return the current thread stack size */ size_t PyThread_get_stacksize(void) @@ -261,11 +335,7 @@ PyThread_GetInfo(void) #ifdef HAVE_PTHREAD_STUBS value = Py_NewRef(Py_None); #elif defined(_POSIX_THREADS) -#ifdef USE_SEMAPHORES - value = PyUnicode_FromString("semaphore"); -#else - value = PyUnicode_FromString("mutex+cond"); -#endif + value = PyUnicode_FromString("pymutex"); if (value == NULL) { Py_DECREF(threadinfo); return NULL; diff --git a/Python/thread_nt.h b/Python/thread_nt.h index e078b98be3cdf4..9a29d14ef67678 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -300,98 +300,6 @@ PyThread_hang_thread(void) } } -/* - * Lock support. It has to be implemented as semaphores. - * I [Dag] tried to implement it with mutex but I could find a way to - * tell whether a thread already own the lock or not. - */ -PyThread_type_lock -PyThread_allocate_lock(void) -{ - PNRMUTEX mutex; - - if (!initialized) - PyThread_init_thread(); - - mutex = AllocNonRecursiveMutex() ; - - PyThread_type_lock aLock = (PyThread_type_lock) mutex; - assert(aLock); - - return aLock; -} - -void -PyThread_free_lock(PyThread_type_lock aLock) -{ - FreeNonRecursiveMutex(aLock) ; -} - -// WaitForSingleObject() accepts timeout in milliseconds in the range -// [0; 0xFFFFFFFE] (DWORD type). INFINITE value (0xFFFFFFFF) means no -// timeout. 0xFFFFFFFE milliseconds is around 49.7 days. -const DWORD TIMEOUT_MS_MAX = 0xFFFFFFFE; - -/* - * Return 1 on success if the lock was acquired - * - * and 0 if the lock was not acquired. This means a 0 is returned - * if the lock has already been acquired by this thread! - */ -PyLockStatus -PyThread_acquire_lock_timed(PyThread_type_lock aLock, - PY_TIMEOUT_T microseconds, int intr_flag) -{ - assert(aLock); - - /* Fow now, intr_flag does nothing on Windows, and lock acquires are - * uninterruptible. */ - PyLockStatus success; - PY_TIMEOUT_T milliseconds; - - if (microseconds >= 0) { - milliseconds = microseconds / 1000; - // Round milliseconds away from zero - if (microseconds % 1000 > 0) { - milliseconds++; - } - if (milliseconds > (PY_TIMEOUT_T)TIMEOUT_MS_MAX) { - // bpo-41710: PyThread_acquire_lock_timed() cannot report timeout - // overflow to the caller, so clamp the timeout to - // [0, TIMEOUT_MS_MAX] milliseconds. - // - // _thread.Lock.acquire() and _thread.RLock.acquire() raise an - // OverflowError if microseconds is greater than PY_TIMEOUT_MAX. - milliseconds = TIMEOUT_MS_MAX; - } - assert(milliseconds != INFINITE); - } - else { - milliseconds = INFINITE; - } - - if (EnterNonRecursiveMutex((PNRMUTEX)aLock, - (DWORD)milliseconds) == WAIT_OBJECT_0) { - success = PY_LOCK_ACQUIRED; - } - else { - success = PY_LOCK_FAILURE; - } - - return success; -} -int -PyThread_acquire_lock(PyThread_type_lock aLock, int waitflag) -{ - return PyThread_acquire_lock_timed(aLock, waitflag ? -1 : 0, 0); -} - -void -PyThread_release_lock(PyThread_type_lock aLock) -{ - assert(aLock); - (void)LeaveNonRecursiveMutex((PNRMUTEX) aLock); -} /* minimum/maximum thread stack sizes supported */ #define THREAD_MIN_STACKSIZE 0x8000 /* 32 KiB */ diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index da4058242448f3..13992f95723866 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -99,16 +99,6 @@ #undef HAVE_SEM_CLOCKWAIT #endif -/* Whether or not to use semaphores directly rather than emulating them with - * mutexes and condition variables: - */ -#if (defined(_POSIX_SEMAPHORES) && !defined(HAVE_BROKEN_POSIX_SEMAPHORES) && \ - (defined(HAVE_SEM_TIMEDWAIT) || defined(HAVE_SEM_CLOCKWAIT))) -# define USE_SEMAPHORES -#else -# undef USE_SEMAPHORES -#endif - /* On platforms that don't use standard POSIX threads pthread_sigmask() * isn't present. DEC threads uses sigprocmask() instead as do most @@ -442,388 +432,6 @@ PyThread_hang_thread(void) } } -#ifdef USE_SEMAPHORES - -/* - * Lock support. - */ - -PyThread_type_lock -PyThread_allocate_lock(void) -{ - sem_t *lock; - int status, error = 0; - - if (!initialized) - PyThread_init_thread(); - - lock = (sem_t *)PyMem_RawMalloc(sizeof(sem_t)); - - if (lock) { - status = sem_init(lock,0,1); - CHECK_STATUS("sem_init"); - - if (error) { - PyMem_RawFree((void *)lock); - lock = NULL; - } - } - - return (PyThread_type_lock)lock; -} - -void -PyThread_free_lock(PyThread_type_lock lock) -{ - sem_t *thelock = (sem_t *)lock; - int status, error = 0; - - (void) error; /* silence unused-but-set-variable warning */ - - if (!thelock) - return; - - status = sem_destroy(thelock); - CHECK_STATUS("sem_destroy"); - - PyMem_RawFree((void *)thelock); -} - -/* - * As of February 2002, Cygwin thread implementations mistakenly report error - * codes in the return value of the sem_ calls (like the pthread_ functions). - * Correct implementations return -1 and put the code in errno. This supports - * either. - */ -static int -fix_status(int status) -{ - return (status == -1) ? errno : status; -} - -PyLockStatus -PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, - int intr_flag) -{ - PyLockStatus success; - sem_t *thelock = (sem_t *)lock; - int status, error = 0; - - (void) error; /* silence unused-but-set-variable warning */ - - PyTime_t timeout; // relative timeout - if (microseconds >= 0) { - // bpo-41710: PyThread_acquire_lock_timed() cannot report timeout - // overflow to the caller, so clamp the timeout to - // [PyTime_MIN, PyTime_MAX]. - // - // PyTime_MAX nanoseconds is around 292.3 years. - // - // _thread.Lock.acquire() and _thread.RLock.acquire() raise an - // OverflowError if microseconds is greater than PY_TIMEOUT_MAX. - timeout = _PyTime_FromMicrosecondsClamp(microseconds); - } - else { - timeout = -1; - } - -#ifdef HAVE_SEM_CLOCKWAIT - struct timespec abs_timeout; - // Local scope for deadline - { - PyTime_t now; - // silently ignore error: cannot report error to the caller - (void)PyTime_MonotonicRaw(&now); - PyTime_t deadline = _PyTime_Add(now, timeout); - _PyTime_AsTimespec_clamp(deadline, &abs_timeout); - } -#else - PyTime_t deadline = 0; - if (timeout > 0 && !intr_flag) { - deadline = _PyDeadline_Init(timeout); - } -#endif - - while (1) { - if (timeout > 0) { -#ifdef HAVE_SEM_CLOCKWAIT - status = fix_status(sem_clockwait(thelock, CLOCK_MONOTONIC, - &abs_timeout)); -#else - PyTime_t now; - // silently ignore error: cannot report error to the caller - (void)PyTime_TimeRaw(&now); - PyTime_t abs_time = _PyTime_Add(now, timeout); - - struct timespec ts; - _PyTime_AsTimespec_clamp(abs_time, &ts); - status = fix_status(sem_timedwait(thelock, &ts)); -#endif - } - else if (timeout == 0) { - status = fix_status(sem_trywait(thelock)); - } - else { - status = fix_status(sem_wait(thelock)); - } - - /* Retry if interrupted by a signal, unless the caller wants to be - notified. */ - if (intr_flag || status != EINTR) { - break; - } - - // sem_clockwait() uses an absolute timeout, there is no need - // to recompute the relative timeout. -#ifndef HAVE_SEM_CLOCKWAIT - if (timeout > 0) { - /* wait interrupted by a signal (EINTR): recompute the timeout */ - timeout = _PyDeadline_Get(deadline); - if (timeout < 0) { - status = ETIMEDOUT; - break; - } - } -#endif - } - - /* Don't check the status if we're stopping because of an interrupt. */ - if (!(intr_flag && status == EINTR)) { - if (timeout > 0) { - if (status != ETIMEDOUT) { -#ifdef HAVE_SEM_CLOCKWAIT - CHECK_STATUS("sem_clockwait"); -#else - CHECK_STATUS("sem_timedwait"); -#endif - } - } - else if (timeout == 0) { - if (status != EAGAIN) { - CHECK_STATUS("sem_trywait"); - } - } - else { - CHECK_STATUS("sem_wait"); - } - } - - if (status == 0) { - success = PY_LOCK_ACQUIRED; - } else if (intr_flag && status == EINTR) { - success = PY_LOCK_INTR; - } else { - success = PY_LOCK_FAILURE; - } - - return success; -} - -void -PyThread_release_lock(PyThread_type_lock lock) -{ - sem_t *thelock = (sem_t *)lock; - int status, error = 0; - - (void) error; /* silence unused-but-set-variable warning */ - - status = sem_post(thelock); - CHECK_STATUS("sem_post"); -} - -#else /* USE_SEMAPHORES */ - -/* - * Lock support. - */ -PyThread_type_lock -PyThread_allocate_lock(void) -{ - pthread_lock *lock; - int status, error = 0; - - if (!initialized) - PyThread_init_thread(); - - lock = (pthread_lock *) PyMem_RawCalloc(1, sizeof(pthread_lock)); - if (lock) { - lock->locked = 0; - - status = pthread_mutex_init(&lock->mut, NULL); - CHECK_STATUS_PTHREAD("pthread_mutex_init"); - /* Mark the pthread mutex underlying a Python mutex as - pure happens-before. We can't simply mark the - Python-level mutex as a mutex because it can be - acquired and released in different threads, which - will cause errors. */ - _Py_ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX(&lock->mut); - - status = _PyThread_cond_init(&lock->lock_released); - CHECK_STATUS_PTHREAD("pthread_cond_init"); - - if (error) { - PyMem_RawFree((void *)lock); - lock = 0; - } - } - - return (PyThread_type_lock) lock; -} - -void -PyThread_free_lock(PyThread_type_lock lock) -{ - pthread_lock *thelock = (pthread_lock *)lock; - int status, error = 0; - - (void) error; /* silence unused-but-set-variable warning */ - - /* some pthread-like implementations tie the mutex to the cond - * and must have the cond destroyed first. - */ - status = pthread_cond_destroy( &thelock->lock_released ); - CHECK_STATUS_PTHREAD("pthread_cond_destroy"); - - status = pthread_mutex_destroy( &thelock->mut ); - CHECK_STATUS_PTHREAD("pthread_mutex_destroy"); - - PyMem_RawFree((void *)thelock); -} - -PyLockStatus -PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, - int intr_flag) -{ - PyLockStatus success = PY_LOCK_FAILURE; - pthread_lock *thelock = (pthread_lock *)lock; - int status, error = 0; - - if (microseconds == 0) { - status = pthread_mutex_trylock( &thelock->mut ); - if (status != EBUSY) { - CHECK_STATUS_PTHREAD("pthread_mutex_trylock[1]"); - } - } - else { - status = pthread_mutex_lock( &thelock->mut ); - CHECK_STATUS_PTHREAD("pthread_mutex_lock[1]"); - } - if (status != 0) { - goto done; - } - - if (thelock->locked == 0) { - success = PY_LOCK_ACQUIRED; - goto unlock; - } - if (microseconds == 0) { - goto unlock; - } - - struct timespec abs_timeout; - if (microseconds > 0) { - _PyThread_cond_after(microseconds, &abs_timeout); - } - // Continue trying until we get the lock - - // mut must be locked by me -- part of the condition protocol - while (1) { - if (microseconds > 0) { - status = pthread_cond_timedwait(&thelock->lock_released, - &thelock->mut, &abs_timeout); - if (status == 1) { - break; - } - if (status == ETIMEDOUT) { - break; - } - CHECK_STATUS_PTHREAD("pthread_cond_timedwait"); - } - else { - status = pthread_cond_wait( - &thelock->lock_released, - &thelock->mut); - CHECK_STATUS_PTHREAD("pthread_cond_wait"); - } - - if (intr_flag && status == 0 && thelock->locked) { - // We were woken up, but didn't get the lock. We probably received - // a signal. Return PY_LOCK_INTR to allow the caller to handle - // it and retry. - success = PY_LOCK_INTR; - break; - } - - if (status == 0 && !thelock->locked) { - success = PY_LOCK_ACQUIRED; - break; - } - - // Wait got interrupted by a signal: retry - } - -unlock: - if (success == PY_LOCK_ACQUIRED) { - thelock->locked = 1; - } - status = pthread_mutex_unlock( &thelock->mut ); - CHECK_STATUS_PTHREAD("pthread_mutex_unlock[1]"); - -done: - if (error) { - success = PY_LOCK_FAILURE; - } - return success; -} - -void -PyThread_release_lock(PyThread_type_lock lock) -{ - pthread_lock *thelock = (pthread_lock *)lock; - int status, error = 0; - - (void) error; /* silence unused-but-set-variable warning */ - - status = pthread_mutex_lock( &thelock->mut ); - CHECK_STATUS_PTHREAD("pthread_mutex_lock[3]"); - - thelock->locked = 0; - - /* wake up someone (anyone, if any) waiting on the lock */ - status = pthread_cond_signal( &thelock->lock_released ); - CHECK_STATUS_PTHREAD("pthread_cond_signal"); - - status = pthread_mutex_unlock( &thelock->mut ); - CHECK_STATUS_PTHREAD("pthread_mutex_unlock[3]"); -} - -#endif /* USE_SEMAPHORES */ - -int -_PyThread_at_fork_reinit(PyThread_type_lock *lock) -{ - PyThread_type_lock new_lock = PyThread_allocate_lock(); - if (new_lock == NULL) { - return -1; - } - - /* bpo-6721, bpo-40089: The old lock can be in an inconsistent state. - fork() can be called in the middle of an operation on the lock done by - another thread. So don't call PyThread_free_lock(*lock). - - Leak memory on purpose. Don't release the memory either since the - address of a mutex is relevant. Putting two mutexes at the same address - can lead to problems. */ - - *lock = new_lock; - return 0; -} - -int -PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) -{ - return PyThread_acquire_lock_timed(lock, waitflag ? -1 : 0, /*intr_flag=*/0); -} /* set the thread stack size. * Return 0 if size is valid, -1 if size is invalid, From c600310663277e24607890298e6d9bf7e1d4f584 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 30 May 2025 14:44:03 +0200 Subject: [PATCH 021/116] gh-134586: mark `_mi_assert_fail` as `noreturn`, `cold` and `throw` (#134624) We add the following attributes on `_mi_assert_fail` to help IDE introspection: * `__attribute__((__noreturn__))` * `__attribute__((cold))` * `__THROW` (GCC only) --- Include/internal/mimalloc/mimalloc/types.h | 29 +++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h index 4f77bd7bc525db..a04169f7fb84e8 100644 --- a/Include/internal/mimalloc/mimalloc/types.h +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -50,6 +50,32 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_cache_align #endif +#if (MI_DEBUG) +#if defined(_MSC_VER) +#define mi_decl_noreturn __declspec(noreturn) +#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) +#define mi_decl_noreturn __attribute__((__noreturn__)) +#else +#define mi_decl_noreturn +#endif + +/* + * 'cold' attribute seems to have been fully supported since GCC 4.x. + * See https://github.com/gcc-mirror/gcc/commit/52bf96d2f299e9e6. + */ +#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) +#define mi_decl_cold __attribute__((cold)) +#else +#define mi_decl_cold +#endif + +#if (defined(__GNUC__) && defined(__THROW)) +#define mi_decl_throw __THROW +#else +#define mi_decl_throw +#endif +#endif + // ------------------------------------------------------ // Variants // ------------------------------------------------------ @@ -582,7 +608,8 @@ struct mi_heap_s { #if (MI_DEBUG) // use our own assertion to print without memory allocation -void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func ); +mi_decl_noreturn mi_decl_cold mi_decl_throw +void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func); #define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) #else #define mi_assert(x) From ce6a6371a23dc57ed4257eb102ebfb2827477abf Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 30 May 2025 15:11:42 +0100 Subject: [PATCH 022/116] GH-134879: Fix `INSTRUMENT_FOR_ITER` for list/tuple (#134897) Fix INSTRUMENT_FOR_ITER for list/tuple --- Lib/test/test_monitoring.py | 15 +++++++++++++++ Python/bytecodes.c | 2 +- Python/generated_cases.c.h | 4 ++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 263e4e6f394155..a932ac80117d27 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -2157,6 +2157,21 @@ def callback(code, instruction_offset): sys.monitoring.restart_events() sys.monitoring.set_events(0, 0) + def test_134879(self): + # gh-134789 + # Specialized FOR_ITER not incrementing index + def foo(): + t = 0 + for i in [1,2,3,4]: + t += i + self.assertEqual(t, 10) + + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.set_local_events(0, foo.__code__, E.BRANCH_LEFT | E.BRANCH_RIGHT) + foo() + sys.monitoring.set_local_events(0, foo.__code__, 0) + class TestOptimizer(MonitoringTestBase, unittest.TestCase): diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a1f8d3605283e3..f02e32fd1d312a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3130,7 +3130,6 @@ dummy_func( if (PyStackRef_IsTaggedInt(null_or_index)) { next = _PyForIter_NextWithIndex(iter_o, null_or_index); if (PyStackRef_IsNull(next)) { - null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); DISPATCH(); } @@ -3191,6 +3190,7 @@ dummy_func( JUMPBY(oparg + 1); DISPATCH(); } + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } else { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 5be2671700aee7..e3cd3b71a1de08 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5759,9 +5759,7 @@ next = _PyForIter_NextWithIndex(iter_o, null_or_index); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(next)) { - null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); - stack_pointer[-1] = null_or_index; DISPATCH(); } null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); @@ -7070,6 +7068,7 @@ JUMPBY(oparg + 1); DISPATCH(); } + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } else { @@ -7099,6 +7098,7 @@ DISPATCH(); } } + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); From cc344e8dd0a6fdc83a032c229f9b3cf53f76a887 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 30 May 2025 17:25:07 +0300 Subject: [PATCH 023/116] gh-134718: Fix ast.dump() for empty non-default values (GH-134926) --- Lib/ast.py | 22 ++++++++--------- Lib/test/test_ast/test_ast.py | 24 +++++++++++++++++++ ...-05-30-13-07-29.gh-issue-134718.9Qvhxn.rst | 2 ++ 3 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-30-13-07-29.gh-issue-134718.9Qvhxn.rst diff --git a/Lib/ast.py b/Lib/ast.py index b9791bf52d3e08..2f11683ecf7c68 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -147,18 +147,16 @@ def _format(node, level=0): if value is None and getattr(cls, name, ...) is None: keywords = True continue - if ( - not show_empty - and (value is None or value == []) - # Special cases: - # `Constant(value=None)` and `MatchSingleton(value=None)` - and not isinstance(node, (Constant, MatchSingleton)) - ): - args_buffer.append(repr(value)) - continue - elif not keywords: - args.extend(args_buffer) - args_buffer = [] + if not show_empty: + if value == []: + field_type = cls._field_types.get(name, object) + if getattr(field_type, '__origin__', ...) is list: + if not keywords: + args_buffer.append(repr(value)) + continue + if not keywords: + args.extend(args_buffer) + args_buffer = [] value, simple = _format(value, level) allsimple = allsimple and simple if keywords: diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 46745cfa8f8325..59263012bc1440 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -1543,18 +1543,42 @@ def check_text(code, empty, full, **kwargs): full="MatchSingleton(value=None)", ) + check_node( + ast.MatchSingleton(value=[]), + empty="MatchSingleton(value=[])", + full="MatchSingleton(value=[])", + ) + check_node( ast.Constant(value=None), empty="Constant(value=None)", full="Constant(value=None)", ) + check_node( + ast.Constant(value=[]), + empty="Constant(value=[])", + full="Constant(value=[])", + ) + check_node( ast.Constant(value=''), empty="Constant(value='')", full="Constant(value='')", ) + check_node( + ast.Interpolation(value=ast.Constant(42), str=None, conversion=-1), + empty="Interpolation(value=Constant(value=42), str=None, conversion=-1)", + full="Interpolation(value=Constant(value=42), str=None, conversion=-1)", + ) + + check_node( + ast.Interpolation(value=ast.Constant(42), str=[], conversion=-1), + empty="Interpolation(value=Constant(value=42), str=[], conversion=-1)", + full="Interpolation(value=Constant(value=42), str=[], conversion=-1)", + ) + check_text( "def a(b: int = 0, *, c): ...", empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))])])", diff --git a/Misc/NEWS.d/next/Library/2025-05-30-13-07-29.gh-issue-134718.9Qvhxn.rst b/Misc/NEWS.d/next/Library/2025-05-30-13-07-29.gh-issue-134718.9Qvhxn.rst new file mode 100644 index 00000000000000..922ab168fdd095 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-30-13-07-29.gh-issue-134718.9Qvhxn.rst @@ -0,0 +1,2 @@ +:func:`ast.dump` now only omits ``None`` and ``[]`` values if they are +default values. From eb145fabbdd755f803296beba2d235fc32efe592 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 30 May 2025 16:27:54 +0200 Subject: [PATCH 024/116] gh-134160: Improve multi-phase init note on isolation & subinterpreters (GH-134775) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/c-api/module.rst | 42 +++++++++++++++++++++--------- Doc/howto/isolating-extensions.rst | 4 ++- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index f7f4d37d4c721f..710135dca89eda 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -288,22 +288,40 @@ An alternate way to specify extensions is to request "multi-phase initialization Extension modules created this way behave more like Python modules: the initialization is split between the *creation phase*, when the module object is created, and the *execution phase*, when it is populated. -The distinction is similar to the :py:meth:`!__new__` and :py:meth:`!__init__` methods -of classes. +The distinction is similar to the :py:meth:`~object.__new__` and +:py:meth:`~object.__init__` methods of classes. Unlike modules created using single-phase initialization, these modules are not -singletons: if the *sys.modules* entry is removed and the module is re-imported, -a new module object is created, and the old module is subject to normal garbage -collection -- as with Python modules. -By default, multiple modules created from the same definition should be -independent: changes to one should not affect the others. -This means that all state should be specific to the module object (using e.g. -using :c:func:`PyModule_GetState`), or its contents (such as the module's -:attr:`~object.__dict__` or individual classes created with :c:func:`PyType_FromSpec`). +singletons. +For example, if the :py:attr:`sys.modules` entry is removed and the module +is re-imported, a new module object is created, and typically populated with +fresh method and type objects. +The old module is subject to normal garbage collection. +This mirrors the behavior of pure-Python modules. + +Additional module instances may be created in +:ref:`sub-interpreters ` +or after after Python runtime reinitialization +(:c:func:`Py_Finalize` and :c:func:`Py_Initialize`). +In these cases, sharing Python objects between module instances would likely +cause crashes or undefined behavior. + +To avoid such issues, each instance of an extension module should +be *isolated*: changes to one instance should not implicitly affect the others, +and all state, including references to Python objects, should be specific to +a particular module instance. +See :ref:`isolating-extensions-howto` for more details and a practical guide. + +A simpler way to avoid these issues is +:ref:`raising an error on repeated initialization `. All modules created using multi-phase initialization are expected to support -:ref:`sub-interpreters `. Making sure multiple modules -are independent is typically enough to achieve this. +:ref:`sub-interpreters `, or otherwise explicitly +signal a lack of support. +This is usually achieved by isolation or blocking repeated initialization, +as above. +A module may also be limited to the main interpreter using +the :c:data:`Py_mod_multiple_interpreters` slot. To request multi-phase initialization, the initialization function (PyInit_modulename) returns a :c:type:`PyModuleDef` instance with non-empty diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 5513cd7367519f..b2109b1503992b 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -168,7 +168,7 @@ possible, consider explicit locking. If it is necessary to use process-global state, the simplest way to avoid issues with multiple interpreters is to explicitly prevent a module from being loaded more than once per process—see -`Opt-Out: Limiting to One Module Object per Process`_. +:ref:`isolating-extensions-optout`. Managing Per-Module State @@ -207,6 +207,8 @@ An example of a module with per-module state is currently available as example module initialization shown at the bottom of the file. +.. _isolating-extensions-optout: + Opt-Out: Limiting to One Module Object per Process -------------------------------------------------- From 52deabefd0af8fc6d9b40823323437bf210f50a5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 30 May 2025 09:15:00 -0600 Subject: [PATCH 025/116] gh-132775: Expand the Capability of Interpreter.call() (gh-133484) It now supports most callables, full args, and return values. --- Include/internal/pycore_crossinterp.h | 35 +- Lib/test/_code_definitions.py | 20 + Lib/test/support/interpreters/__init__.py | 31 +- Lib/test/test_code.py | 48 +- Lib/test/test_interpreters/test_api.py | 541 ++++++++++++++++++---- Modules/_interpchannelsmodule.c | 8 +- Modules/_interpqueuesmodule.c | 8 +- Modules/_interpretersmodule.c | 414 +++++++++++++---- Python/crossinterp.c | 449 ++++++++++++++---- Python/import.c | 4 +- 10 files changed, 1257 insertions(+), 301 deletions(-) diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 1272927413868b..713ddc66ba7382 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -317,7 +317,9 @@ typedef enum error_code { _PyXI_ERR_ALREADY_RUNNING = -4, _PyXI_ERR_MAIN_NS_FAILURE = -5, _PyXI_ERR_APPLY_NS_FAILURE = -6, - _PyXI_ERR_NOT_SHAREABLE = -7, + _PyXI_ERR_PRESERVE_FAILURE = -7, + _PyXI_ERR_EXC_PROPAGATION_FAILURE = -8, + _PyXI_ERR_NOT_SHAREABLE = -9, } _PyXI_errcode; @@ -350,16 +352,33 @@ typedef struct xi_session _PyXI_session; PyAPI_FUNC(_PyXI_session *) _PyXI_NewSession(void); PyAPI_FUNC(void) _PyXI_FreeSession(_PyXI_session *); +typedef struct { + PyObject *preserved; + PyObject *excinfo; + _PyXI_errcode errcode; +} _PyXI_session_result; +PyAPI_FUNC(void) _PyXI_ClearResult(_PyXI_session_result *); + PyAPI_FUNC(int) _PyXI_Enter( _PyXI_session *session, PyInterpreterState *interp, - PyObject *nsupdates); -PyAPI_FUNC(void) _PyXI_Exit(_PyXI_session *session); - -PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace(_PyXI_session *); - -PyAPI_FUNC(PyObject *) _PyXI_ApplyCapturedException(_PyXI_session *session); -PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session); + PyObject *nsupdates, + _PyXI_session_result *); +PyAPI_FUNC(int) _PyXI_Exit( + _PyXI_session *, + _PyXI_errcode, + _PyXI_session_result *); + +PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace( + _PyXI_session *, + _PyXI_errcode *); + +PyAPI_FUNC(int) _PyXI_Preserve( + _PyXI_session *, + const char *, + PyObject *, + _PyXI_errcode *); +PyAPI_FUNC(PyObject *) _PyXI_GetPreserved(_PyXI_session_result *, const char *); /*************/ diff --git a/Lib/test/_code_definitions.py b/Lib/test/_code_definitions.py index 733a15b25f6894..274beb65a6d0f4 100644 --- a/Lib/test/_code_definitions.py +++ b/Lib/test/_code_definitions.py @@ -57,6 +57,15 @@ def spam_with_globals_and_builtins(): print(res) +def spam_full_args(a, b, /, c, d, *args, e, f, **kwargs): + return (a, b, c, d, e, f, args, kwargs) + + +def spam_full_args_with_defaults(a=-1, b=-2, /, c=-3, d=-4, *args, + e=-5, f=-6, **kwargs): + return (a, b, c, d, e, f, args, kwargs) + + def spam_args_attrs_and_builtins(a, b, /, c, d, *args, e, f, **kwargs): if args.__len__() > 2: return None @@ -67,6 +76,10 @@ def spam_returns_arg(x): return x +def spam_raises(): + raise Exception('spam!') + + def spam_with_inner_not_closure(): def eggs(): pass @@ -177,8 +190,11 @@ def ham_C_closure(z): spam_minimal, spam_with_builtins, spam_with_globals_and_builtins, + spam_full_args, + spam_full_args_with_defaults, spam_args_attrs_and_builtins, spam_returns_arg, + spam_raises, spam_with_inner_not_closure, spam_with_inner_closure, spam_annotated, @@ -219,8 +235,10 @@ def ham_C_closure(z): spam, spam_minimal, spam_with_builtins, + spam_full_args, spam_args_attrs_and_builtins, spam_returns_arg, + spam_raises, spam_annotated, spam_with_inner_not_closure, spam_with_inner_closure, @@ -238,6 +256,7 @@ def ham_C_closure(z): STATELESS_CODE = [ *STATELESS_FUNCTIONS, script_with_globals, + spam_full_args_with_defaults, spam_with_globals_and_builtins, spam_full, ] @@ -248,6 +267,7 @@ def ham_C_closure(z): script_with_explicit_empty_return, spam_minimal, spam_with_builtins, + spam_raises, spam_with_inner_not_closure, spam_with_inner_closure, ] diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index e067f259364d2a..6d1b0690805d2d 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -226,33 +226,32 @@ def exec(self, code, /): if excinfo is not None: raise ExecutionFailed(excinfo) - def call(self, callable, /): - """Call the object in the interpreter with given args/kwargs. + def _call(self, callable, args, kwargs): + res, excinfo = _interpreters.call(self._id, callable, args, kwargs, restrict=True) + if excinfo is not None: + raise ExecutionFailed(excinfo) + return res - Only functions that take no arguments and have no closure - are supported. + def call(self, callable, /, *args, **kwargs): + """Call the object in the interpreter with given args/kwargs. - The return value is discarded. + Nearly all callables, args, kwargs, and return values are + supported. All "shareable" objects are supported, as are + "stateless" functions (meaning non-closures that do not use + any globals). This method will fall back to pickle. If the callable raises an exception then the error display - (including full traceback) is send back between the interpreters + (including full traceback) is sent back between the interpreters and an ExecutionFailed exception is raised, much like what happens with Interpreter.exec(). """ - # XXX Support args and kwargs. - # XXX Support arbitrary callables. - # XXX Support returning the return value (e.g. via pickle). - excinfo = _interpreters.call(self._id, callable, restrict=True) - if excinfo is not None: - raise ExecutionFailed(excinfo) + return self._call(callable, args, kwargs) - def call_in_thread(self, callable, /): + def call_in_thread(self, callable, /, *args, **kwargs): """Return a new thread that calls the object in the interpreter. The return value and any raised exception are discarded. """ - def task(): - self.call(callable) - t = threading.Thread(target=task) + t = threading.Thread(target=self._call, args=(callable, args, kwargs)) t.start() return t diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 32cf8aacaf6b72..9fc2b047bef719 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -701,6 +701,26 @@ def test_local_kinds(self): 'checks': CO_FAST_LOCAL, 'res': CO_FAST_LOCAL, }, + defs.spam_full_args: { + 'a': POSONLY, + 'b': POSONLY, + 'c': POSORKW, + 'd': POSORKW, + 'e': KWONLY, + 'f': KWONLY, + 'args': VARARGS, + 'kwargs': VARKWARGS, + }, + defs.spam_full_args_with_defaults: { + 'a': POSONLY, + 'b': POSONLY, + 'c': POSORKW, + 'd': POSORKW, + 'e': KWONLY, + 'f': KWONLY, + 'args': VARARGS, + 'kwargs': VARKWARGS, + }, defs.spam_args_attrs_and_builtins: { 'a': POSONLY, 'b': POSONLY, @@ -714,6 +734,7 @@ def test_local_kinds(self): defs.spam_returns_arg: { 'x': POSORKW, }, + defs.spam_raises: {}, defs.spam_with_inner_not_closure: { 'eggs': CO_FAST_LOCAL, }, @@ -934,6 +955,20 @@ def new_var_counts(*, purelocals=5, globalvars=6, ), + defs.spam_full_args: new_var_counts( + posonly=2, + posorkw=2, + kwonly=2, + varargs=1, + varkwargs=1, + ), + defs.spam_full_args_with_defaults: new_var_counts( + posonly=2, + posorkw=2, + kwonly=2, + varargs=1, + varkwargs=1, + ), defs.spam_args_attrs_and_builtins: new_var_counts( posonly=2, posorkw=2, @@ -945,6 +980,9 @@ def new_var_counts(*, defs.spam_returns_arg: new_var_counts( posorkw=1, ), + defs.spam_raises: new_var_counts( + globalvars=1, + ), defs.spam_with_inner_not_closure: new_var_counts( purelocals=1, ), @@ -1097,10 +1135,16 @@ def new_var_counts(*, def test_stateless(self): self.maxDiff = None + STATELESS_FUNCTIONS = [ + *defs.STATELESS_FUNCTIONS, + # stateless with defaults + defs.spam_full_args_with_defaults, + ] + for func in defs.STATELESS_CODE: with self.subTest((func, '(code)')): _testinternalcapi.verify_stateless_code(func.__code__) - for func in defs.STATELESS_FUNCTIONS: + for func in STATELESS_FUNCTIONS: with self.subTest((func, '(func)')): _testinternalcapi.verify_stateless_code(func) @@ -1110,7 +1154,7 @@ def test_stateless(self): with self.assertRaises(Exception): _testinternalcapi.verify_stateless_code(func.__code__) - if func not in defs.STATELESS_FUNCTIONS: + if func not in STATELESS_FUNCTIONS: with self.subTest((func, '(func)')): with self.assertRaises(Exception): _testinternalcapi.verify_stateless_code(func) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 165949167ceba8..b3c9ef8efba37a 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1,17 +1,22 @@ +import contextlib import os import pickle +import sys from textwrap import dedent import threading import types import unittest from test import support +from test.support import os_helper +from test.support import script_helper from test.support import import_helper # Raise SkipTest if subinterpreters not supported. _interpreters = import_helper.import_module('_interpreters') from test.support import Py_GIL_DISABLED from test.support import interpreters from test.support import force_not_colorized +import test._crossinterp_definitions as defs from test.support.interpreters import ( InterpreterError, InterpreterNotFoundError, ExecutionFailed, ) @@ -29,6 +34,59 @@ WHENCE_STR_STDLIB = '_interpreters module' +def is_pickleable(obj): + try: + pickle.dumps(obj) + except Exception: + return False + return True + + +@contextlib.contextmanager +def defined_in___main__(name, script, *, remove=False): + import __main__ as mainmod + mainns = vars(mainmod) + assert name not in mainns + exec(script, mainns, mainns) + if remove: + yield mainns.pop(name) + else: + try: + yield mainns[name] + finally: + mainns.pop(name, None) + + +def build_excinfo(exctype, msg=None, formatted=None, errdisplay=None): + if isinstance(exctype, type): + assert issubclass(exctype, BaseException), exctype + exctype = types.SimpleNamespace( + __name__=exctype.__name__, + __qualname__=exctype.__qualname__, + __module__=exctype.__module__, + ) + elif isinstance(exctype, str): + module, _, name = exctype.rpartition(exctype) + if not module and name in __builtins__: + module = 'builtins' + exctype = types.SimpleNamespace( + __name__=name, + __qualname__=exctype, + __module__=module or None, + ) + else: + assert isinstance(exctype, types.SimpleNamespace) + assert msg is None or isinstance(msg, str), msg + assert formatted is None or isinstance(formatted, str), formatted + assert errdisplay is None or isinstance(errdisplay, str), errdisplay + return types.SimpleNamespace( + type=exctype, + msg=msg, + formatted=formatted, + errdisplay=errdisplay, + ) + + class ModuleTests(TestBase): def test_queue_aliases(self): @@ -890,24 +948,26 @@ def test_created_with_capi(self): # Interpreter.exec() behavior. -def call_func_noop(): - pass +call_func_noop = defs.spam_minimal +call_func_ident = defs.spam_returns_arg +call_func_failure = defs.spam_raises def call_func_return_shareable(): return (1, None) -def call_func_return_not_shareable(): - return [1, 2, 3] +def call_func_return_stateless_func(): + return (lambda x: x) -def call_func_failure(): - raise Exception('spam!') +def call_func_return_pickleable(): + return [1, 2, 3] -def call_func_ident(value): - return value +def call_func_return_unpickleable(): + x = 42 + return (lambda: x) def get_call_func_closure(value): @@ -916,6 +976,11 @@ def call_func_closure(): return call_func_closure +def call_func_exec_wrapper(script, ns): + res = exec(script, ns, ns) + return res, ns, id(ns) + + class Spam: @staticmethod @@ -1012,86 +1077,375 @@ class TestInterpreterCall(TestBase): # - preserves info (e.g. SyntaxError) # - matching error display - def test_call(self): + @contextlib.contextmanager + def assert_fails(self, expected): + with self.assertRaises(ExecutionFailed) as cm: + yield cm + uncaught = cm.exception.excinfo + self.assertEqual(uncaught.type.__name__, expected.__name__) + + def assert_fails_not_shareable(self): + return self.assert_fails(interpreters.NotShareableError) + + def assert_code_equal(self, code1, code2): + if code1 == code2: + return + self.assertEqual(code1.co_name, code2.co_name) + self.assertEqual(code1.co_flags, code2.co_flags) + self.assertEqual(code1.co_consts, code2.co_consts) + self.assertEqual(code1.co_varnames, code2.co_varnames) + self.assertEqual(code1.co_cellvars, code2.co_cellvars) + self.assertEqual(code1.co_freevars, code2.co_freevars) + self.assertEqual(code1.co_names, code2.co_names) + self.assertEqual( + _testinternalcapi.get_code_var_counts(code1), + _testinternalcapi.get_code_var_counts(code2), + ) + self.assertEqual(code1.co_code, code2.co_code) + + def assert_funcs_equal(self, func1, func2): + if func1 == func2: + return + self.assertIs(type(func1), type(func2)) + self.assertEqual(func1.__name__, func2.__name__) + self.assertEqual(func1.__defaults__, func2.__defaults__) + self.assertEqual(func1.__kwdefaults__, func2.__kwdefaults__) + self.assertEqual(func1.__closure__, func2.__closure__) + self.assert_code_equal(func1.__code__, func2.__code__) + self.assertEqual( + _testinternalcapi.get_code_var_counts(func1), + _testinternalcapi.get_code_var_counts(func2), + ) + + def assert_exceptions_equal(self, exc1, exc2): + assert isinstance(exc1, Exception) + assert isinstance(exc2, Exception) + if exc1 == exc2: + return + self.assertIs(type(exc1), type(exc2)) + self.assertEqual(exc1.args, exc2.args) + + def test_stateless_funcs(self): interp = interpreters.create() - for i, (callable, args, kwargs) in enumerate([ - (call_func_noop, (), {}), - (Spam.noop, (), {}), + func = call_func_noop + with self.subTest('no args, no return'): + res = interp.call(func) + self.assertIsNone(res) + + func = call_func_return_shareable + with self.subTest('no args, returns shareable'): + res = interp.call(func) + self.assertEqual(res, (1, None)) + + func = call_func_return_stateless_func + expected = (lambda x: x) + with self.subTest('no args, returns stateless func'): + res = interp.call(func) + self.assert_funcs_equal(res, expected) + + func = call_func_return_pickleable + with self.subTest('no args, returns pickleable'): + res = interp.call(func) + self.assertEqual(res, [1, 2, 3]) + + func = call_func_return_unpickleable + with self.subTest('no args, returns unpickleable'): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func) + + def test_stateless_func_returns_arg(self): + interp = interpreters.create() + + for arg in [ + None, + 10, + 'spam!', + b'spam!', + (1, 2, 'spam!'), + memoryview(b'spam!'), + ]: + with self.subTest(f'shareable {arg!r}'): + assert _interpreters.is_shareable(arg) + res = interp.call(defs.spam_returns_arg, arg) + self.assertEqual(res, arg) + + for arg in defs.STATELESS_FUNCTIONS: + with self.subTest(f'stateless func {arg!r}'): + res = interp.call(defs.spam_returns_arg, arg) + self.assert_funcs_equal(res, arg) + + for arg in defs.TOP_FUNCTIONS: + if arg in defs.STATELESS_FUNCTIONS: + continue + with self.subTest(f'stateful func {arg!r}'): + res = interp.call(defs.spam_returns_arg, arg) + self.assert_funcs_equal(res, arg) + assert is_pickleable(arg) + + for arg in [ + Ellipsis, + NotImplemented, + object(), + 2**1000, + [1, 2, 3], + {'a': 1, 'b': 2}, + types.SimpleNamespace(x=42), + # builtin types + object, + type, + Exception, + ModuleNotFoundError, + # builtin exceptions + Exception('uh-oh!'), + ModuleNotFoundError('mymodule'), + # builtin fnctions + len, + sys.exit, + # user classes + *defs.TOP_CLASSES, + *(c(*a) for c, a in defs.TOP_CLASSES.items() + if c not in defs.CLASSES_WITHOUT_EQUALITY), + ]: + with self.subTest(f'pickleable {arg!r}'): + res = interp.call(defs.spam_returns_arg, arg) + if type(arg) is object: + self.assertIs(type(res), object) + elif isinstance(arg, BaseException): + self.assert_exceptions_equal(res, arg) + else: + self.assertEqual(res, arg) + assert is_pickleable(arg) + + for arg in [ + types.MappingProxyType({}), + *(f for f in defs.NESTED_FUNCTIONS + if f not in defs.STATELESS_FUNCTIONS), + ]: + with self.subTest(f'unpickleable {arg!r}'): + assert not _interpreters.is_shareable(arg) + assert not is_pickleable(arg) + with self.assertRaises(interpreters.NotShareableError): + interp.call(defs.spam_returns_arg, arg) + + def test_full_args(self): + interp = interpreters.create() + expected = (1, 2, 3, 4, 5, 6, ('?',), {'g': 7, 'h': 8}) + func = defs.spam_full_args + res = interp.call(func, 1, 2, 3, 4, '?', e=5, f=6, g=7, h=8) + self.assertEqual(res, expected) + + def test_full_defaults(self): + # pickleable, but not stateless + interp = interpreters.create() + expected = (-1, -2, -3, -4, -5, -6, (), {'g': 8, 'h': 9}) + res = interp.call(defs.spam_full_args_with_defaults, g=8, h=9) + self.assertEqual(res, expected) + + def test_modified_arg(self): + interp = interpreters.create() + script = dedent(""" + a = 7 + b = 2 + c = a ** b + """) + ns = {} + expected = {'a': 7, 'b': 2, 'c': 49} + res = interp.call(call_func_exec_wrapper, script, ns) + obj, resns, resid = res + del resns['__builtins__'] + self.assertIsNone(obj) + self.assertEqual(ns, {}) + self.assertEqual(resns, expected) + self.assertNotEqual(resid, id(ns)) + self.assertNotEqual(resid, id(resns)) + + def test_func_in___main___valid(self): + # pickleable, already there' + + with os_helper.temp_dir() as tempdir: + def new_mod(name, text): + script_helper.make_script(tempdir, name, dedent(text)) + + def run(text): + name = 'myscript' + text = dedent(f""" + import sys + sys.path.insert(0, {tempdir!r}) + + """) + dedent(text) + filename = script_helper.make_script(tempdir, name, text) + res = script_helper.assert_python_ok(filename) + return res.out.decode('utf-8').strip() + + # no module indirection + with self.subTest('no indirection'): + text = run(f""" + from test.support import interpreters + + def spam(): + # This a global var... + return __name__ + + if __name__ == '__main__': + interp = interpreters.create() + res = interp.call(spam) + print(res) + """) + self.assertEqual(text, '') + + # indirect as func, direct interp + new_mod('mymod', f""" + def run(interp, func): + return interp.call(func) + """) + with self.subTest('indirect as func, direct interp'): + text = run(f""" + from test.support import interpreters + import mymod + + def spam(): + # This a global var... + return __name__ + + if __name__ == '__main__': + interp = interpreters.create() + res = mymod.run(interp, spam) + print(res) + """) + self.assertEqual(text, '') + + # indirect as func, indirect interp + new_mod('mymod', f""" + from test.support import interpreters + def run(func): + interp = interpreters.create() + return interp.call(func) + """) + with self.subTest('indirect as func, indirect interp'): + text = run(f""" + import mymod + + def spam(): + # This a global var... + return __name__ + + if __name__ == '__main__': + res = mymod.run(spam) + print(res) + """) + self.assertEqual(text, '') + + def test_func_in___main___invalid(self): + interp = interpreters.create() + + funcname = f'{__name__.replace(".", "_")}_spam_okay' + script = dedent(f""" + def {funcname}(): + # This a global var... + return __name__ + """) + + with self.subTest('pickleable, added dynamically'): + with defined_in___main__(funcname, script) as arg: + with self.assertRaises(interpreters.NotShareableError): + interp.call(defs.spam_returns_arg, arg) + + with self.subTest('lying about __main__'): + with defined_in___main__(funcname, script, remove=True) as arg: + with self.assertRaises(interpreters.NotShareableError): + interp.call(defs.spam_returns_arg, arg) + + def test_raises(self): + interp = interpreters.create() + with self.assertRaises(ExecutionFailed): + interp.call(call_func_failure) + + with self.assert_fails(ValueError): + interp.call(call_func_complex, '???', exc=ValueError('spam')) + + def test_call_valid(self): + interp = interpreters.create() + + for i, (callable, args, kwargs, expected) in enumerate([ + (call_func_noop, (), {}, None), + (call_func_ident, ('spamspamspam',), {}, 'spamspamspam'), + (call_func_return_shareable, (), {}, (1, None)), + (call_func_return_pickleable, (), {}, [1, 2, 3]), + (Spam.noop, (), {}, None), + (Spam.from_values, (), {}, Spam(())), + (Spam.from_values, (1, 2, 3), {}, Spam((1, 2, 3))), + (Spam, ('???',), {}, Spam('???')), + (Spam(101), (), {}, (101, (), {})), + (Spam(10101).run, (), {}, (10101, (), {})), + (call_func_complex, ('ident', 'spam'), {}, 'spam'), + (call_func_complex, ('full-ident', 'spam'), {}, ('spam', (), {})), + (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}, + ('spam', ('ham',), {'eggs': '!!!'})), + (call_func_complex, ('globals',), {}, __name__), + (call_func_complex, ('interpid',), {}, interp.id), + (call_func_complex, ('custom', 'spam!'), {}, Spam('spam!')), ]): with self.subTest(f'success case #{i+1}'): - res = interp.call(callable) - self.assertIs(res, None) + res = interp.call(callable, *args, **kwargs) + self.assertEqual(res, expected) + + def test_call_invalid(self): + interp = interpreters.create() + + func = get_call_func_closure + with self.subTest(func): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func, 42) + + func = get_call_func_closure(42) + with self.subTest(func): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func) + + func = call_func_complex + op = 'closure' + with self.subTest(f'{func} ({op})'): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func, op, value='~~~') + + op = 'custom-inner' + with self.subTest(f'{func} ({op})'): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func, op, 'eggs!') + + def test_call_in_thread(self): + interp = interpreters.create() for i, (callable, args, kwargs) in enumerate([ - (call_func_ident, ('spamspamspam',), {}), - (get_call_func_closure, (42,), {}), - (get_call_func_closure(42), (), {}), + (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_pickleable, (), {}), (Spam.from_values, (), {}), (Spam.from_values, (1, 2, 3), {}), - (Spam, ('???'), {}), (Spam(101), (), {}), (Spam(10101).run, (), {}), + (Spam.noop, (), {}), (call_func_complex, ('ident', 'spam'), {}), (call_func_complex, ('full-ident', 'spam'), {}), (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), (call_func_complex, ('globals',), {}), (call_func_complex, ('interpid',), {}), - (call_func_complex, ('closure',), {'value': '~~~'}), (call_func_complex, ('custom', 'spam!'), {}), - (call_func_complex, ('custom-inner', 'eggs!'), {}), - (call_func_complex, ('???',), {'exc': ValueError('spam')}), - (call_func_return_shareable, (), {}), - (call_func_return_not_shareable, (), {}), - ]): - with self.subTest(f'invalid case #{i+1}'): - with self.assertRaises(Exception): - if args or kwargs: - raise Exception((args, kwargs)) - interp.call(callable) - - with self.assertRaises(ExecutionFailed): - interp.call(call_func_failure) - - def test_call_in_thread(self): - interp = interpreters.create() - - for i, (callable, args, kwargs) in enumerate([ - (call_func_noop, (), {}), - (Spam.noop, (), {}), ]): with self.subTest(f'success case #{i+1}'): with self.captured_thread_exception() as ctx: - t = interp.call_in_thread(callable) + t = interp.call_in_thread(callable, *args, **kwargs) t.join() self.assertIsNone(ctx.caught) for i, (callable, args, kwargs) in enumerate([ - (call_func_ident, ('spamspamspam',), {}), (get_call_func_closure, (42,), {}), (get_call_func_closure(42), (), {}), - (Spam.from_values, (), {}), - (Spam.from_values, (1, 2, 3), {}), - (Spam, ('???'), {}), - (Spam(101), (), {}), - (Spam(10101).run, (), {}), - (call_func_complex, ('ident', 'spam'), {}), - (call_func_complex, ('full-ident', 'spam'), {}), - (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), - (call_func_complex, ('globals',), {}), - (call_func_complex, ('interpid',), {}), - (call_func_complex, ('closure',), {'value': '~~~'}), - (call_func_complex, ('custom', 'spam!'), {}), - (call_func_complex, ('custom-inner', 'eggs!'), {}), - (call_func_complex, ('???',), {'exc': ValueError('spam')}), - (call_func_return_shareable, (), {}), - (call_func_return_not_shareable, (), {}), ]): with self.subTest(f'invalid case #{i+1}'): - if args or kwargs: - continue with self.captured_thread_exception() as ctx: - t = interp.call_in_thread(callable) + t = interp.call_in_thread(callable, *args, **kwargs) t.join() self.assertIsNotNone(ctx.caught) @@ -1600,18 +1954,14 @@ def test_exec(self): with results: exc = _interpreters.exec(interpid, script) out = results.stdout() - self.assertEqual(out, '') - self.assert_ns_equal(exc, types.SimpleNamespace( - type=types.SimpleNamespace( - __name__='Exception', - __qualname__='Exception', - __module__='builtins', - ), - msg='uh-oh!', + expected = build_excinfo( + Exception, 'uh-oh!', # We check these in other tests. formatted=exc.formatted, errdisplay=exc.errdisplay, - )) + ) + self.assertEqual(out, '') + self.assert_ns_equal(exc, expected) with self.subTest('from C-API'): with self.interpreter_from_capi() as interpid: @@ -1623,25 +1973,50 @@ def test_exec(self): self.assertEqual(exc.msg, 'it worked!') def test_call(self): - with self.subTest('no args'): - interpid = _interpreters.create() - with self.assertRaises(ValueError): - _interpreters.call(interpid, call_func_return_shareable) + interpid = _interpreters.create() + + # Here we focus on basic args and return values. + # See TestInterpreterCall for full operational coverage, + # including supported callables. + + with self.subTest('no args, return None'): + func = defs.spam_minimal + res, exc = _interpreters.call(interpid, func) + self.assertIsNone(exc) + self.assertIsNone(res) + + with self.subTest('empty args, return None'): + func = defs.spam_minimal + res, exc = _interpreters.call(interpid, func, (), {}) + self.assertIsNone(exc) + self.assertIsNone(res) + + with self.subTest('no args, return non-None'): + func = defs.script_with_return + res, exc = _interpreters.call(interpid, func) + self.assertIsNone(exc) + self.assertIs(res, True) + + with self.subTest('full args, return non-None'): + expected = (1, 2, 3, 4, 5, 6, (7, 8), {'g': 9, 'h': 0}) + func = defs.spam_full_args + args = (1, 2, 3, 4, 7, 8) + kwargs = dict(e=5, f=6, g=9, h=0) + res, exc = _interpreters.call(interpid, func, args, kwargs) + self.assertIsNone(exc) + self.assertEqual(res, expected) with self.subTest('uncaught exception'): - interpid = _interpreters.create() - exc = _interpreters.call(interpid, call_func_failure) - self.assertEqual(exc, types.SimpleNamespace( - type=types.SimpleNamespace( - __name__='Exception', - __qualname__='Exception', - __module__='builtins', - ), - msg='spam!', + func = defs.spam_raises + res, exc = _interpreters.call(interpid, func) + expected = build_excinfo( + Exception, 'spam!', # We check these in other tests. formatted=exc.formatted, errdisplay=exc.errdisplay, - )) + ) + self.assertIsNone(res) + self.assertEqual(exc, expected) @requires_test_modules def test_set___main___attrs(self): diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index bfd805bf5e4072..ea2e5f99dfa308 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -254,10 +254,10 @@ _get_current_module_state(void) { PyObject *mod = _get_current_module(); if (mod == NULL) { - // XXX import it? - PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME_STR " module not imported yet"); - return NULL; + mod = PyImport_ImportModule(MODULE_NAME_STR); + if (mod == NULL) { + return NULL; + } } module_state *state = get_module_state(mod); Py_DECREF(mod); diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index ffc52c8ee74d85..71d8fd8716cd94 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -1356,10 +1356,10 @@ _queueobj_from_xid(_PyXIData_t *data) PyObject *mod = _get_current_module(); if (mod == NULL) { - // XXX import it? - PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME_STR " module not imported yet"); - return NULL; + mod = PyImport_ImportModule(MODULE_NAME_STR); + if (mod == NULL) { + return NULL; + } } PyTypeObject *cls = get_external_queue_type(mod); diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 376517ab92360f..037e9544543c4d 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -72,6 +72,32 @@ is_running_main(PyInterpreterState *interp) } +static inline int +is_notshareable_raised(PyThreadState *tstate) +{ + PyObject *exctype = _PyXIData_GetNotShareableErrorType(tstate); + return _PyErr_ExceptionMatches(tstate, exctype); +} + +static void +unwrap_not_shareable(PyThreadState *tstate) +{ + if (!is_notshareable_raised(tstate)) { + return; + } + PyObject *exc = _PyErr_GetRaisedException(tstate); + PyObject *cause = PyException_GetCause(exc); + if (cause != NULL) { + Py_DECREF(exc); + exc = cause; + } + else { + assert(PyException_GetContext(exc) == NULL); + } + _PyErr_SetRaisedException(tstate, exc); +} + + /* Cross-interpreter Buffer Views *******************************************/ /* When a memoryview object is "shared" between interpreters, @@ -320,10 +346,10 @@ _get_current_module_state(void) { PyObject *mod = _get_current_module(); if (mod == NULL) { - // XXX import it? - PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME_STR " module not imported yet"); - return NULL; + mod = PyImport_ImportModule(MODULE_NAME_STR); + if (mod == NULL) { + return NULL; + } } module_state *state = get_module_state(mod); Py_DECREF(mod); @@ -422,76 +448,265 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config) } +struct interp_call { + _PyXIData_t *func; + _PyXIData_t *args; + _PyXIData_t *kwargs; + struct { + _PyXIData_t func; + _PyXIData_t args; + _PyXIData_t kwargs; + } _preallocated; +}; + +static void +_interp_call_clear(struct interp_call *call) +{ + if (call->func != NULL) { + _PyXIData_Clear(NULL, call->func); + } + if (call->args != NULL) { + _PyXIData_Clear(NULL, call->args); + } + if (call->kwargs != NULL) { + _PyXIData_Clear(NULL, call->kwargs); + } + *call = (struct interp_call){0}; +} + +static int +_interp_call_pack(PyThreadState *tstate, struct interp_call *call, + PyObject *func, PyObject *args, PyObject *kwargs) +{ + xidata_fallback_t fallback = _PyXIDATA_FULL_FALLBACK; + assert(call->func == NULL); + assert(call->args == NULL); + assert(call->kwargs == NULL); + // Handle the func. + if (!PyCallable_Check(func)) { + _PyErr_Format(tstate, PyExc_TypeError, + "expected a callable, got %R", func); + return -1; + } + if (_PyFunction_GetXIData(tstate, func, &call->_preallocated.func) < 0) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (_PyPickle_GetXIData(tstate, func, &call->_preallocated.func) < 0) { + _PyErr_SetRaisedException(tstate, exc); + return -1; + } + Py_DECREF(exc); + } + call->func = &call->_preallocated.func; + // Handle the args. + if (args == NULL || args == Py_None) { + // Leave it empty. + } + else { + assert(PyTuple_Check(args)); + if (PyTuple_GET_SIZE(args) > 0) { + if (_PyObject_GetXIData( + tstate, args, fallback, &call->_preallocated.args) < 0) + { + _interp_call_clear(call); + return -1; + } + call->args = &call->_preallocated.args; + } + } + // Handle the kwargs. + if (kwargs == NULL || kwargs == Py_None) { + // Leave it empty. + } + else { + assert(PyDict_Check(kwargs)); + if (PyDict_GET_SIZE(kwargs) > 0) { + if (_PyObject_GetXIData( + tstate, kwargs, fallback, &call->_preallocated.kwargs) < 0) + { + _interp_call_clear(call); + return -1; + } + call->kwargs = &call->_preallocated.kwargs; + } + } + return 0; +} + +static int +_interp_call_unpack(struct interp_call *call, + PyObject **p_func, PyObject **p_args, PyObject **p_kwargs) +{ + // Unpack the func. + PyObject *func = _PyXIData_NewObject(call->func); + if (func == NULL) { + return -1; + } + // Unpack the args. + PyObject *args; + if (call->args == NULL) { + args = PyTuple_New(0); + if (args == NULL) { + Py_DECREF(func); + return -1; + } + } + else { + args = _PyXIData_NewObject(call->args); + if (args == NULL) { + Py_DECREF(func); + return -1; + } + assert(PyTuple_Check(args)); + } + // Unpack the kwargs. + PyObject *kwargs = NULL; + if (call->kwargs != NULL) { + kwargs = _PyXIData_NewObject(call->kwargs); + if (kwargs == NULL) { + Py_DECREF(func); + Py_DECREF(args); + return -1; + } + assert(PyDict_Check(kwargs)); + } + *p_func = func; + *p_args = args; + *p_kwargs = kwargs; + return 0; +} + static int -_run_script(_PyXIData_t *script, PyObject *ns) +_make_call(struct interp_call *call, + PyObject **p_result, _PyXI_errcode *p_errcode) +{ + assert(call != NULL && call->func != NULL); + PyThreadState *tstate = _PyThreadState_GET(); + + // Get the func and args. + PyObject *func = NULL, *args = NULL, *kwargs = NULL; + if (_interp_call_unpack(call, &func, &args, &kwargs) < 0) { + assert(func == NULL); + assert(args == NULL); + assert(kwargs == NULL); + *p_errcode = is_notshareable_raised(tstate) + ? _PyXI_ERR_NOT_SHAREABLE + : _PyXI_ERR_OTHER; + return -1; + } + *p_errcode = _PyXI_ERR_NO_ERROR; + + // Make the call. + PyObject *resobj = PyObject_Call(func, args, kwargs); + Py_DECREF(func); + Py_XDECREF(args); + Py_XDECREF(kwargs); + if (resobj == NULL) { + return -1; + } + *p_result = resobj; + return 0; +} + +static int +_run_script(_PyXIData_t *script, PyObject *ns, _PyXI_errcode *p_errcode) { PyObject *code = _PyXIData_NewObject(script); if (code == NULL) { + *p_errcode = _PyXI_ERR_NOT_SHAREABLE; return -1; } PyObject *result = PyEval_EvalCode(code, ns, ns); Py_DECREF(code); if (result == NULL) { + *p_errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION; return -1; } + assert(result == Py_None); Py_DECREF(result); // We throw away the result. return 0; } +struct run_result { + PyObject *result; + PyObject *excinfo; +}; + +static void +_run_result_clear(struct run_result *runres) +{ + Py_CLEAR(runres->result); + Py_CLEAR(runres->excinfo); +} + static int -_exec_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, - _PyXIData_t *script, PyObject *shareables, - PyObject **p_excinfo) +_run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, + _PyXIData_t *script, struct interp_call *call, + PyObject *shareables, struct run_result *runres) { assert(!_PyErr_Occurred(tstate)); _PyXI_session *session = _PyXI_NewSession(); if (session == NULL) { return -1; } + _PyXI_session_result result = {0}; // Prep and switch interpreters. - if (_PyXI_Enter(session, interp, shareables) < 0) { - if (_PyErr_Occurred(tstate)) { - // If an error occured at this step, it means that interp - // was not prepared and switched. - _PyXI_FreeSession(session); - return -1; - } - // Now, apply the error from another interpreter: - PyObject *excinfo = _PyXI_ApplyCapturedException(session); - if (excinfo != NULL) { - *p_excinfo = excinfo; - } - assert(PyErr_Occurred()); + if (_PyXI_Enter(session, interp, shareables, &result) < 0) { + // If an error occured at this step, it means that interp + // was not prepared and switched. _PyXI_FreeSession(session); + assert(result.excinfo == NULL); return -1; } - // Run the script. + // Run in the interpreter. int res = -1; - PyObject *mainns = _PyXI_GetMainNamespace(session); - if (mainns == NULL) { - goto finally; + _PyXI_errcode errcode = _PyXI_ERR_NO_ERROR; + if (script != NULL) { + assert(call == NULL); + PyObject *mainns = _PyXI_GetMainNamespace(session, &errcode); + if (mainns == NULL) { + goto finally; + } + res = _run_script(script, mainns, &errcode); } - res = _run_script(script, mainns); + else { + assert(call != NULL); + PyObject *resobj; + res = _make_call(call, &resobj, &errcode); + if (res == 0) { + res = _PyXI_Preserve(session, "resobj", resobj, &errcode); + Py_DECREF(resobj); + if (res < 0) { + goto finally; + } + } + } + int exitres; finally: // Clean up and switch back. - _PyXI_Exit(session); + exitres = _PyXI_Exit(session, errcode, &result); + assert(res == 0 || exitres != 0); + _PyXI_FreeSession(session); - // Propagate any exception out to the caller. - assert(!PyErr_Occurred()); - if (res < 0) { - PyObject *excinfo = _PyXI_ApplyCapturedException(session); - if (excinfo != NULL) { - *p_excinfo = excinfo; - } + res = exitres; + if (_PyErr_Occurred(tstate)) { + assert(res < 0); + } + else if (res < 0) { + assert(result.excinfo != NULL); + runres->excinfo = Py_NewRef(result.excinfo); + res = -1; } else { - assert(!_PyXI_HasCapturedException(session)); + assert(result.excinfo == NULL); + runres->result = _PyXI_GetPreserved(&result, "resobj"); + if (_PyErr_Occurred(tstate)) { + res = -1; + } } - - _PyXI_FreeSession(session); + _PyXI_ClearResult(&result); return res; } @@ -842,21 +1057,23 @@ interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs) } // Prep and switch interpreters, including apply the updates. - if (_PyXI_Enter(session, interp, updates) < 0) { - if (!PyErr_Occurred()) { - _PyXI_ApplyCapturedException(session); - assert(PyErr_Occurred()); - } - else { - assert(!_PyXI_HasCapturedException(session)); - } + if (_PyXI_Enter(session, interp, updates, NULL) < 0) { _PyXI_FreeSession(session); return NULL; } // Clean up and switch back. - _PyXI_Exit(session); + assert(!PyErr_Occurred()); + int res = _PyXI_Exit(session, _PyXI_ERR_NO_ERROR, NULL); _PyXI_FreeSession(session); + assert(res == 0); + if (res < 0) { + // unreachable + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "unresolved error"); + } + return NULL; + } Py_RETURN_NONE; } @@ -867,23 +1084,16 @@ PyDoc_STRVAR(set___main___attrs_doc, Bind the given attributes in the interpreter's __main__ module."); -static void -unwrap_not_shareable(PyThreadState *tstate) +static PyObject * +_handle_script_error(struct run_result *runres) { - PyObject *exctype = _PyXIData_GetNotShareableErrorType(tstate); - if (!_PyErr_ExceptionMatches(tstate, exctype)) { - return; - } - PyObject *exc = _PyErr_GetRaisedException(tstate); - PyObject *cause = PyException_GetCause(exc); - if (cause != NULL) { - Py_DECREF(exc); - exc = cause; - } - else { - assert(PyException_GetContext(exc) == NULL); + assert(runres->result == NULL); + if (runres->excinfo == NULL) { + assert(PyErr_Occurred()); + return NULL; } - _PyErr_SetRaisedException(tstate, exc); + assert(!PyErr_Occurred()); + return runres->excinfo; } static PyObject * @@ -918,13 +1128,14 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyObject *excinfo = NULL; - int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo); + struct run_result runres = {0}; + int res = _run_in_interpreter( + tstate, interp, &xidata, NULL, shared, &runres); _PyXIData_Release(&xidata); if (res < 0) { - assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); - return excinfo; + return _handle_script_error(&runres); } + assert(runres.result == NULL); Py_RETURN_NONE; #undef FUNCNAME } @@ -981,13 +1192,14 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyObject *excinfo = NULL; - int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo); + struct run_result runres = {0}; + int res = _run_in_interpreter( + tstate, interp, &xidata, NULL, shared, &runres); _PyXIData_Release(&xidata); if (res < 0) { - assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); - return excinfo; + return _handle_script_error(&runres); } + assert(runres.result == NULL); Py_RETURN_NONE; #undef FUNCNAME } @@ -1043,13 +1255,14 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyObject *excinfo = NULL; - int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo); + struct run_result runres = {0}; + int res = _run_in_interpreter( + tstate, interp, &xidata, NULL, shared, &runres); _PyXIData_Release(&xidata); if (res < 0) { - assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); - return excinfo; + return _handle_script_error(&runres); } + assert(runres.result == NULL); Py_RETURN_NONE; #undef FUNCNAME } @@ -1069,15 +1282,18 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) #define FUNCNAME MODULE_NAME_STR ".call" PyThreadState *tstate = _PyThreadState_GET(); static char *kwlist[] = {"id", "callable", "args", "kwargs", - "restrict", NULL}; + "preserve_exc", "restrict", NULL}; PyObject *id, *callable; PyObject *args_obj = NULL; PyObject *kwargs_obj = NULL; + int preserve_exc = 0; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|OO$p:" FUNCNAME, kwlist, - &id, &callable, &args_obj, &kwargs_obj, - &restricted)) + "OO|O!O!$pp:" FUNCNAME, kwlist, + &id, &callable, + &PyTuple_Type, &args_obj, + &PyDict_Type, &kwargs_obj, + &preserve_exc, &restricted)) { return NULL; } @@ -1089,29 +1305,29 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - if (args_obj != NULL) { - _PyErr_SetString(tstate, PyExc_ValueError, "got unexpected args"); - return NULL; - } - if (kwargs_obj != NULL) { - _PyErr_SetString(tstate, PyExc_ValueError, "got unexpected kwargs"); + struct interp_call call = {0}; + if (_interp_call_pack(tstate, &call, callable, args_obj, kwargs_obj) < 0) { return NULL; } - _PyXIData_t xidata = {0}; - if (_PyCode_GetPureScriptXIData(tstate, callable, &xidata) < 0) { - unwrap_not_shareable(tstate); - return NULL; + PyObject *res_and_exc = NULL; + struct run_result runres = {0}; + if (_run_in_interpreter(tstate, interp, NULL, &call, NULL, &runres) < 0) { + if (runres.excinfo == NULL) { + assert(_PyErr_Occurred(tstate)); + goto finally; + } + assert(!_PyErr_Occurred(tstate)); } + assert(runres.result == NULL || runres.excinfo == NULL); + res_and_exc = Py_BuildValue("OO", + (runres.result ? runres.result : Py_None), + (runres.excinfo ? runres.excinfo : Py_None)); - PyObject *excinfo = NULL; - int res = _exec_in_interpreter(tstate, interp, &xidata, NULL, &excinfo); - _PyXIData_Release(&xidata); - if (res < 0) { - assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); - return excinfo; - } - Py_RETURN_NONE; +finally: + _interp_call_clear(&call); + _run_result_clear(&runres); + return res_and_exc; #undef FUNCNAME } @@ -1119,13 +1335,7 @@ PyDoc_STRVAR(call_doc, "call(id, callable, args=None, kwargs=None, *, restrict=False)\n\ \n\ Call the provided object in the identified interpreter.\n\ -Pass the given args and kwargs, if possible.\n\ -\n\ -\"callable\" may be a plain function with no free vars that takes\n\ -no arguments.\n\ -\n\ -The function's code object is used and all its state\n\ -is ignored, including its __globals__ dict."); +Pass the given args and kwargs, if possible."); static PyObject * diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 13d91c508c41fa..5e73ab28f2b663 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -70,6 +70,17 @@ runpy_run_path(const char *filename, const char *modname) } +static void +set_exc_with_cause(PyObject *exctype, const char *msg) +{ + PyObject *cause = PyErr_GetRaisedException(); + PyErr_SetString(exctype, msg); + PyObject *exc = PyErr_GetRaisedException(); + PyException_SetCause(exc, cause); + PyErr_SetRaisedException(exc); +} + + static PyObject * pyerr_get_message(PyObject *exc) { @@ -1314,7 +1325,7 @@ _excinfo_normalize_type(struct _excinfo_type *info, } static void -_PyXI_excinfo_Clear(_PyXI_excinfo *info) +_PyXI_excinfo_clear(_PyXI_excinfo *info) { _excinfo_clear_type(&info->type); if (info->msg != NULL) { @@ -1364,7 +1375,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) assert(exc != NULL); if (PyErr_GivenExceptionMatches(exc, PyExc_MemoryError)) { - _PyXI_excinfo_Clear(info); + _PyXI_excinfo_clear(info); return NULL; } const char *failure = NULL; @@ -1410,7 +1421,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) error: assert(failure != NULL); - _PyXI_excinfo_Clear(info); + _PyXI_excinfo_clear(info); return failure; } @@ -1461,7 +1472,7 @@ _PyXI_excinfo_InitFromObject(_PyXI_excinfo *info, PyObject *obj) error: assert(failure != NULL); - _PyXI_excinfo_Clear(info); + _PyXI_excinfo_clear(info); return failure; } @@ -1656,7 +1667,7 @@ _PyXI_ExcInfoAsObject(_PyXI_excinfo *info) void _PyXI_ClearExcInfo(_PyXI_excinfo *info) { - _PyXI_excinfo_Clear(info); + _PyXI_excinfo_clear(info); } @@ -1694,6 +1705,14 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) PyErr_SetString(PyExc_InterpreterError, "failed to apply namespace to __main__"); break; + case _PyXI_ERR_PRESERVE_FAILURE: + PyErr_SetString(PyExc_InterpreterError, + "failed to preserve objects across session"); + break; + case _PyXI_ERR_EXC_PROPAGATION_FAILURE: + PyErr_SetString(PyExc_InterpreterError, + "failed to transfer exception between interpreters"); + break; case _PyXI_ERR_NOT_SHAREABLE: _set_xid_lookup_failure(tstate, NULL, NULL, NULL); break; @@ -1743,7 +1762,7 @@ _PyXI_InitError(_PyXI_error *error, PyObject *excobj, _PyXI_errcode code) assert(excobj == NULL); assert(code != _PyXI_ERR_NO_ERROR); error->code = code; - _PyXI_excinfo_Clear(&error->uncaught); + _PyXI_excinfo_clear(&error->uncaught); } return failure; } @@ -1753,7 +1772,7 @@ _PyXI_ApplyError(_PyXI_error *error) { PyThreadState *tstate = PyThreadState_Get(); if (error->code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { - // Raise an exception that proxies the propagated exception. + // We will raise an exception that proxies the propagated exception. return _PyXI_excinfo_AsObject(&error->uncaught); } else if (error->code == _PyXI_ERR_NOT_SHAREABLE) { @@ -1839,7 +1858,8 @@ _sharednsitem_has_value(_PyXI_namespace_item *item, int64_t *p_interpid) } static int -_sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value) +_sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value, + xidata_fallback_t fallback) { assert(_sharednsitem_is_initialized(item)); assert(item->xidata == NULL); @@ -1848,8 +1868,7 @@ _sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value) return -1; } PyThreadState *tstate = PyThreadState_Get(); - // XXX Use _PyObject_GetXIDataWithFallback()? - if (_PyObject_GetXIDataNoFallback(tstate, value, item->xidata) != 0) { + if (_PyObject_GetXIData(tstate, value, fallback, item->xidata) < 0) { PyMem_RawFree(item->xidata); item->xidata = NULL; // The caller may want to propagate PyExc_NotShareableError @@ -1881,7 +1900,8 @@ _sharednsitem_clear(_PyXI_namespace_item *item) } static int -_sharednsitem_copy_from_ns(struct _sharednsitem *item, PyObject *ns) +_sharednsitem_copy_from_ns(struct _sharednsitem *item, PyObject *ns, + xidata_fallback_t fallback) { assert(item->name != NULL); assert(item->xidata == NULL); @@ -1893,7 +1913,7 @@ _sharednsitem_copy_from_ns(struct _sharednsitem *item, PyObject *ns) // When applied, this item will be set to the default (or fail). return 0; } - if (_sharednsitem_set_value(item, value) < 0) { + if (_sharednsitem_set_value(item, value, fallback) < 0) { return -1; } return 0; @@ -2144,18 +2164,21 @@ _create_sharedns(PyObject *names) return NULL; } -static void _propagate_not_shareable_error(_PyXI_session *); +static void _propagate_not_shareable_error(_PyXI_errcode *); static int -_fill_sharedns(_PyXI_namespace *ns, PyObject *nsobj, _PyXI_session *session) +_fill_sharedns(_PyXI_namespace *ns, PyObject *nsobj, + xidata_fallback_t fallback, _PyXI_errcode *p_errcode) { // All items are expected to be shareable. assert(_sharedns_check_counts(ns)); assert(ns->numnames == ns->maxitems); assert(ns->numvalues == 0); for (Py_ssize_t i=0; i < ns->maxitems; i++) { - if (_sharednsitem_copy_from_ns(&ns->items[i], nsobj) < 0) { - _propagate_not_shareable_error(session); + if (_sharednsitem_copy_from_ns(&ns->items[i], nsobj, fallback) < 0) { + if (p_errcode != NULL) { + _propagate_not_shareable_error(p_errcode); + } // Clear out the ones we set so far. for (Py_ssize_t j=0; j < i; j++) { _sharednsitem_clear_value(&ns->items[j]); @@ -2221,6 +2244,18 @@ _apply_sharedns(_PyXI_namespace *ns, PyObject *nsobj, PyObject *dflt) /* switched-interpreter sessions */ /*********************************/ +struct xi_session_error { + // This is set if the interpreter is entered and raised an exception + // that needs to be handled in some special way during exit. + _PyXI_errcode *override; + // This is set if exit captured an exception to propagate. + _PyXI_error *info; + + // -- pre-allocated memory -- + _PyXI_error _info; + _PyXI_errcode _override; +}; + struct xi_session { #define SESSION_UNUSED 0 #define SESSION_ACTIVE 1 @@ -2249,18 +2284,14 @@ struct xi_session { // beginning of the session as a convenience. PyObject *main_ns; - // This is set if the interpreter is entered and raised an exception - // that needs to be handled in some special way during exit. - _PyXI_errcode *error_override; - // This is set if exit captured an exception to propagate. - _PyXI_error *error; + // This is a dict of objects that will be available (via sharing) + // once the session exits. Do not access this directly; use + // _PyXI_Preserve() and _PyXI_GetPreserved() instead; + PyObject *_preserved; - // -- pre-allocated memory -- - _PyXI_error _error; - _PyXI_errcode _error_override; + struct xi_session_error error; }; - _PyXI_session * _PyXI_NewSession(void) { @@ -2286,9 +2317,25 @@ _session_is_active(_PyXI_session *session) return session->status == SESSION_ACTIVE; } -static int _ensure_main_ns(_PyXI_session *); +static int +_session_pop_error(_PyXI_session *session, struct xi_session_error *err) +{ + if (session->error.info == NULL) { + assert(session->error.override == NULL); + *err = (struct xi_session_error){0}; + return 0; + } + *err = session->error; + err->info = &err->_info; + if (err->override != NULL) { + err->override = &err->_override; + } + session->error = (struct xi_session_error){0}; + return 1; +} + +static int _ensure_main_ns(_PyXI_session *, _PyXI_errcode *); static inline void _session_set_error(_PyXI_session *, _PyXI_errcode); -static void _capture_current_exception(_PyXI_session *); /* enter/exit a cross-interpreter session */ @@ -2305,9 +2352,9 @@ _enter_session(_PyXI_session *session, PyInterpreterState *interp) assert(!session->running); assert(session->main_ns == NULL); // Set elsewhere and cleared in _capture_current_exception(). - assert(session->error_override == NULL); - // Set elsewhere and cleared in _PyXI_ApplyCapturedException(). - assert(session->error == NULL); + assert(session->error.override == NULL); + // Set elsewhere and cleared in _PyXI_Exit(). + assert(session->error.info == NULL); // Switch to interpreter. PyThreadState *tstate = PyThreadState_Get(); @@ -2336,14 +2383,16 @@ _exit_session(_PyXI_session *session) PyThreadState *tstate = session->init_tstate; assert(tstate != NULL); assert(PyThreadState_Get() == tstate); + assert(!_PyErr_Occurred(tstate)); // Release any of the entered interpreters resources. Py_CLEAR(session->main_ns); + Py_CLEAR(session->_preserved); // Ensure this thread no longer owns __main__. if (session->running) { _PyInterpreterState_SetNotRunningMain(tstate->interp); - assert(!PyErr_Occurred()); + assert(!_PyErr_Occurred(tstate)); session->running = 0; } @@ -2360,21 +2409,16 @@ _exit_session(_PyXI_session *session) assert(!session->own_init_tstate); } - // For now the error data persists past the exit. - *session = (_PyXI_session){ - .error_override = session->error_override, - .error = session->error, - ._error = session->_error, - ._error_override = session->_error_override, - }; + assert(session->error.info == NULL); + assert(session->error.override == _PyXI_ERR_NO_ERROR); + + *session = (_PyXI_session){0}; } static void -_propagate_not_shareable_error(_PyXI_session *session) +_propagate_not_shareable_error(_PyXI_errcode *p_errcode) { - if (session == NULL) { - return; - } + assert(p_errcode != NULL); PyThreadState *tstate = PyThreadState_Get(); PyObject *exctype = get_notshareableerror_type(tstate); if (exctype == NULL) { @@ -2384,46 +2428,46 @@ _propagate_not_shareable_error(_PyXI_session *session) } if (PyErr_ExceptionMatches(exctype)) { // We want to propagate the exception directly. - _session_set_error(session, _PyXI_ERR_NOT_SHAREABLE); + *p_errcode = _PyXI_ERR_NOT_SHAREABLE; } } -PyObject * -_PyXI_ApplyCapturedException(_PyXI_session *session) -{ - assert(!PyErr_Occurred()); - assert(session->error != NULL); - PyObject *res = _PyXI_ApplyError(session->error); - assert((res == NULL) != (PyErr_Occurred() == NULL)); - session->error = NULL; - return res; -} - -int -_PyXI_HasCapturedException(_PyXI_session *session) -{ - return session->error != NULL; -} - int _PyXI_Enter(_PyXI_session *session, - PyInterpreterState *interp, PyObject *nsupdates) + PyInterpreterState *interp, PyObject *nsupdates, + _PyXI_session_result *result) { // Convert the attrs for cross-interpreter use. _PyXI_namespace *sharedns = NULL; if (nsupdates != NULL) { Py_ssize_t len = PyDict_Size(nsupdates); if (len < 0) { + if (result != NULL) { + result->errcode = _PyXI_ERR_APPLY_NS_FAILURE; + } return -1; } if (len > 0) { sharedns = _create_sharedns(nsupdates); if (sharedns == NULL) { + if (result != NULL) { + result->errcode = _PyXI_ERR_APPLY_NS_FAILURE; + } return -1; } - if (_fill_sharedns(sharedns, nsupdates, NULL) < 0) { - assert(session->error == NULL); + // For now we limit it to shareable objects. + xidata_fallback_t fallback = _PyXIDATA_XIDATA_ONLY; + _PyXI_errcode errcode = _PyXI_ERR_NO_ERROR; + if (_fill_sharedns(sharedns, nsupdates, fallback, &errcode) < 0) { + assert(PyErr_Occurred()); + assert(session->error.info == NULL); + if (errcode == _PyXI_ERR_NO_ERROR) { + errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION; + } _destroy_sharedns(sharedns); + if (result != NULL) { + result->errcode = errcode; + } return -1; } } @@ -2445,8 +2489,7 @@ _PyXI_Enter(_PyXI_session *session, // Apply the cross-interpreter data. if (sharedns != NULL) { - if (_ensure_main_ns(session) < 0) { - errcode = _PyXI_ERR_MAIN_NS_FAILURE; + if (_ensure_main_ns(session, &errcode) < 0) { goto error; } if (_apply_sharedns(sharedns, session->main_ns, NULL) < 0) { @@ -2462,19 +2505,124 @@ _PyXI_Enter(_PyXI_session *session, error: // We want to propagate all exceptions here directly (best effort). + assert(errcode != _PyXI_ERR_NO_ERROR); _session_set_error(session, errcode); + assert(!PyErr_Occurred()); + + // Exit the session. + struct xi_session_error err; + (void)_session_pop_error(session, &err); _exit_session(session); + if (sharedns != NULL) { _destroy_sharedns(sharedns); } + + // Apply the error from the other interpreter. + PyObject *excinfo = _PyXI_ApplyError(err.info); + _PyXI_excinfo_clear(&err.info->uncaught); + if (excinfo != NULL) { + if (result != NULL) { + result->excinfo = excinfo; + } + else { +#ifdef Py_DEBUG + fprintf(stderr, "_PyXI_Enter(): uncaught exception discarded"); +#endif + } + } + assert(PyErr_Occurred()); + return -1; } -void -_PyXI_Exit(_PyXI_session *session) +static int _pop_preserved(_PyXI_session *, _PyXI_namespace **, PyObject **, + _PyXI_errcode *); +static int _finish_preserved(_PyXI_namespace *, PyObject **); + +int +_PyXI_Exit(_PyXI_session *session, _PyXI_errcode errcode, + _PyXI_session_result *result) { - _capture_current_exception(session); + int res = 0; + + // Capture the raised exception, if any. + assert(session->error.info == NULL); + if (PyErr_Occurred()) { + _session_set_error(session, errcode); + assert(!PyErr_Occurred()); + } + else { + assert(errcode == _PyXI_ERR_NO_ERROR); + assert(session->error.override == NULL); + } + + // Capture the preserved namespace. + _PyXI_namespace *preserved = NULL; + PyObject *preservedobj = NULL; + if (result != NULL) { + errcode = _PyXI_ERR_NO_ERROR; + if (_pop_preserved(session, &preserved, &preservedobj, &errcode) < 0) { + if (session->error.info != NULL) { + // XXX Chain the exception (i.e. set __context__)? + PyErr_FormatUnraisable( + "Exception ignored while capturing preserved objects"); + } + else { + _session_set_error(session, errcode); + } + } + } + + // Exit the session. + struct xi_session_error err; + (void)_session_pop_error(session, &err); _exit_session(session); + + // Restore the preserved namespace. + assert(preserved == NULL || preservedobj == NULL); + if (_finish_preserved(preserved, &preservedobj) < 0) { + assert(preservedobj == NULL); + if (err.info != NULL) { + // XXX Chain the exception (i.e. set __context__)? + PyErr_FormatUnraisable( + "Exception ignored while capturing preserved objects"); + } + else { + errcode = _PyXI_ERR_PRESERVE_FAILURE; + _propagate_not_shareable_error(&errcode); + } + } + if (result != NULL) { + result->preserved = preservedobj; + result->errcode = errcode; + } + + // Apply the error from the other interpreter, if any. + if (err.info != NULL) { + res = -1; + assert(!PyErr_Occurred()); + PyObject *excinfo = _PyXI_ApplyError(err.info); + _PyXI_excinfo_clear(&err.info->uncaught); + if (excinfo == NULL) { + assert(PyErr_Occurred()); + if (result != NULL) { + _PyXI_ClearResult(result); + *result = (_PyXI_session_result){ + .errcode = _PyXI_ERR_EXC_PROPAGATION_FAILURE, + }; + } + } + else if (result != NULL) { + result->excinfo = excinfo; + } + else { +#ifdef Py_DEBUG + fprintf(stderr, "_PyXI_Exit(): uncaught exception discarded"); +#endif + } + } + return res; } @@ -2483,15 +2631,15 @@ _PyXI_Exit(_PyXI_session *session) static void _capture_current_exception(_PyXI_session *session) { - assert(session->error == NULL); + assert(session->error.info == NULL); if (!PyErr_Occurred()) { - assert(session->error_override == NULL); + assert(session->error.override == NULL); return; } // Handle the exception override. - _PyXI_errcode *override = session->error_override; - session->error_override = NULL; + _PyXI_errcode *override = session->error.override; + session->error.override = NULL; _PyXI_errcode errcode = override != NULL ? *override : _PyXI_ERR_UNCAUGHT_EXCEPTION; @@ -2514,7 +2662,7 @@ _capture_current_exception(_PyXI_session *session) } // Capture the exception. - _PyXI_error *err = &session->_error; + _PyXI_error *err = &session->error._info; *err = (_PyXI_error){ .interp = session->init_tstate->interp, }; @@ -2541,7 +2689,7 @@ _capture_current_exception(_PyXI_session *session) // Finished! assert(!PyErr_Occurred()); - session->error = err; + session->error.info = err; } static inline void @@ -2549,15 +2697,19 @@ _session_set_error(_PyXI_session *session, _PyXI_errcode errcode) { assert(_session_is_active(session)); assert(PyErr_Occurred()); + if (errcode == _PyXI_ERR_NO_ERROR) { + // We're a bit forgiving here. + errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION; + } if (errcode != _PyXI_ERR_UNCAUGHT_EXCEPTION) { - session->_error_override = errcode; - session->error_override = &session->_error_override; + session->error._override = errcode; + session->error.override = &session->error._override; } _capture_current_exception(session); } static int -_ensure_main_ns(_PyXI_session *session) +_ensure_main_ns(_PyXI_session *session, _PyXI_errcode *p_errcode) { assert(_session_is_active(session)); if (session->main_ns != NULL) { @@ -2566,11 +2718,17 @@ _ensure_main_ns(_PyXI_session *session) // Cache __main__.__dict__. PyObject *main_mod = _Py_GetMainModule(session->init_tstate); if (_Py_CheckMainModule(main_mod) < 0) { + if (p_errcode != NULL) { + *p_errcode = _PyXI_ERR_MAIN_NS_FAILURE; + } return -1; } PyObject *ns = PyModule_GetDict(main_mod); // borrowed Py_DECREF(main_mod); if (ns == NULL) { + if (p_errcode != NULL) { + *p_errcode = _PyXI_ERR_MAIN_NS_FAILURE; + } return -1; } session->main_ns = Py_NewRef(ns); @@ -2578,21 +2736,150 @@ _ensure_main_ns(_PyXI_session *session) } PyObject * -_PyXI_GetMainNamespace(_PyXI_session *session) +_PyXI_GetMainNamespace(_PyXI_session *session, _PyXI_errcode *p_errcode) { if (!_session_is_active(session)) { PyErr_SetString(PyExc_RuntimeError, "session not active"); return NULL; } - if (_ensure_main_ns(session) < 0) { - _session_set_error(session, _PyXI_ERR_MAIN_NS_FAILURE); - _capture_current_exception(session); + if (_ensure_main_ns(session, p_errcode) < 0) { return NULL; } return session->main_ns; } +static int +_pop_preserved(_PyXI_session *session, + _PyXI_namespace **p_xidata, PyObject **p_obj, + _PyXI_errcode *p_errcode) +{ + assert(_PyThreadState_GET() == session->init_tstate); // active session + if (session->_preserved == NULL) { + *p_xidata = NULL; + *p_obj = NULL; + return 0; + } + if (session->init_tstate == session->prev_tstate) { + // We did not switch interpreters. + *p_xidata = NULL; + *p_obj = session->_preserved; + session->_preserved = NULL; + return 0; + } + *p_obj = NULL; + + // We did switch interpreters. + Py_ssize_t len = PyDict_Size(session->_preserved); + if (len < 0) { + if (p_errcode != NULL) { + *p_errcode = _PyXI_ERR_PRESERVE_FAILURE; + } + return -1; + } + else if (len == 0) { + *p_xidata = NULL; + } + else { + _PyXI_namespace *xidata = _create_sharedns(session->_preserved); + if (xidata == NULL) { + if (p_errcode != NULL) { + *p_errcode = _PyXI_ERR_PRESERVE_FAILURE; + } + return -1; + } + _PyXI_errcode errcode = _PyXI_ERR_NO_ERROR; + if (_fill_sharedns(xidata, session->_preserved, + _PyXIDATA_FULL_FALLBACK, &errcode) < 0) + { + assert(session->error.info == NULL); + if (errcode != _PyXI_ERR_NOT_SHAREABLE) { + errcode = _PyXI_ERR_PRESERVE_FAILURE; + } + if (p_errcode != NULL) { + *p_errcode = errcode; + } + _destroy_sharedns(xidata); + return -1; + } + *p_xidata = xidata; + } + Py_CLEAR(session->_preserved); + return 0; +} + +static int +_finish_preserved(_PyXI_namespace *xidata, PyObject **p_preserved) +{ + if (xidata == NULL) { + return 0; + } + int res = -1; + if (p_preserved != NULL) { + PyObject *ns = PyDict_New(); + if (ns == NULL) { + goto finally; + } + if (_apply_sharedns(xidata, ns, NULL) < 0) { + Py_CLEAR(ns); + goto finally; + } + *p_preserved = ns; + } + res = 0; + +finally: + _destroy_sharedns(xidata); + return res; +} + +int +_PyXI_Preserve(_PyXI_session *session, const char *name, PyObject *value, + _PyXI_errcode *p_errcode) +{ + if (!_session_is_active(session)) { + PyErr_SetString(PyExc_RuntimeError, "session not active"); + return -1; + } + if (session->_preserved == NULL) { + session->_preserved = PyDict_New(); + if (session->_preserved == NULL) { + set_exc_with_cause(PyExc_RuntimeError, + "failed to initialize preserved objects"); + if (p_errcode != NULL) { + *p_errcode = _PyXI_ERR_PRESERVE_FAILURE; + } + return -1; + } + } + if (PyDict_SetItemString(session->_preserved, name, value) < 0) { + set_exc_with_cause(PyExc_RuntimeError, "failed to preserve object"); + if (p_errcode != NULL) { + *p_errcode = _PyXI_ERR_PRESERVE_FAILURE; + } + return -1; + } + return 0; +} + +PyObject * +_PyXI_GetPreserved(_PyXI_session_result *result, const char *name) +{ + PyObject *value = NULL; + if (result->preserved != NULL) { + (void)PyDict_GetItemStringRef(result->preserved, name, &value); + } + return value; +} + +void +_PyXI_ClearResult(_PyXI_session_result *result) +{ + Py_CLEAR(result->preserved); + Py_CLEAR(result->excinfo); +} + + /*********************/ /* runtime lifecycle */ /*********************/ diff --git a/Python/import.c b/Python/import.c index 98557991378e05..184dede335dfd6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3964,8 +3964,10 @@ PyImport_Import(PyObject *module_name) if (globals != NULL) { Py_INCREF(globals); builtins = PyObject_GetItem(globals, &_Py_ID(__builtins__)); - if (builtins == NULL) + if (builtins == NULL) { + // XXX Fall back to interp->builtins or sys.modules['builtins']? goto err; + } } else { /* No globals -- use standard builtins, and fake globals */ From 1a89991d2362867a9127e151376135615bc92a4c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 30 May 2025 18:52:36 +0300 Subject: [PATCH 026/116] gh-134733: Fix documentation for the show_empty option of ast.dump() (GH-134925) Optional None values are always omitted. --- Doc/library/ast.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index cf22250cac6091..ca0654acb33689 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2445,8 +2445,9 @@ and classes for traversing abstract syntax trees: indents that many spaces per level. If *indent* is a string (such as ``"\t"``), that string is used to indent each level. - If *show_empty* is ``False`` (the default), empty lists and fields that are ``None`` - will be omitted from the output. + If *show_empty* is false (the default), optional empty lists will be + omitted from the output. + Optional ``None`` values are always omitted. .. versionchanged:: 3.9 Added the *indent* option. From 98a5b830d2463351800f4d76edba1a306a3e0ec9 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Fri, 30 May 2025 10:46:16 -0700 Subject: [PATCH 027/116] .gitignore personal Claude Code configs (#134942) .gitignore personal Claude Code configs. https://docs.anthropic.com/en/docs/claude-code/memory --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 2a6f249275c32e..a2542eaf7b698a 100644 --- a/.gitignore +++ b/.gitignore @@ -173,3 +173,7 @@ Python/frozen_modules/MANIFEST # main branch only: ABI files are not checked/maintained. Doc/data/python*.abi + +# People's custom https://docs.anthropic.com/en/docs/claude-code/memory configs. +/.claude/ +CLAUDE.local.md From 310c8cd5e5dcb0fb9509e08c0d5cf32075416878 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Fri, 30 May 2025 11:28:14 -0700 Subject: [PATCH 028/116] rearrange my gitingore addition w/comment to make backporting easier (#134945) --- .gitignore | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index a2542eaf7b698a..cdb0352e0a8836 100644 --- a/.gitignore +++ b/.gitignore @@ -171,9 +171,10 @@ Python/frozen_modules/MANIFEST /python !/Python/ -# main branch only: ABI files are not checked/maintained. -Doc/data/python*.abi - # People's custom https://docs.anthropic.com/en/docs/claude-code/memory configs. /.claude/ CLAUDE.local.md + +#### main branch only stuff below this line, things to backport go above. #### +# main branch only: ABI files are not checked/maintained. +Doc/data/python*.abi From 8865b4f95b32097099d252111669b88ec7c1eb7f Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 30 May 2025 19:37:29 +0100 Subject: [PATCH 029/116] gh-134923: Use /GENPROFILE and /USEPROFILE for Windows PGO builds (GH-134924) --- .../2025-05-30-11-02-30.gh-issue-134923.gBkRg4.rst | 3 +++ PCbuild/_testclinic_limited.vcxproj | 1 + PCbuild/pyproject.props | 11 ++++------- 3 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-05-30-11-02-30.gh-issue-134923.gBkRg4.rst diff --git a/Misc/NEWS.d/next/Build/2025-05-30-11-02-30.gh-issue-134923.gBkRg4.rst b/Misc/NEWS.d/next/Build/2025-05-30-11-02-30.gh-issue-134923.gBkRg4.rst new file mode 100644 index 00000000000000..a742a6add8ae93 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-05-30-11-02-30.gh-issue-134923.gBkRg4.rst @@ -0,0 +1,3 @@ +Windows builds with profile-guided optimization enabled now use +``/GENPROFILE`` and ``/USEPROFILE`` instead of deprecated ``/LTCG:`` +options. diff --git a/PCbuild/_testclinic_limited.vcxproj b/PCbuild/_testclinic_limited.vcxproj index 183a55080e8693..95c205309b1f30 100644 --- a/PCbuild/_testclinic_limited.vcxproj +++ b/PCbuild/_testclinic_limited.vcxproj @@ -70,6 +70,7 @@ {01FDF29A-40A1-46DF-84F5-85EBBD2A2410} _testclinic_limited Win32Proj + false diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 7272542e13a5ca..ce51e342241686 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -96,19 +96,16 @@ MachineX64 MachineARM MachineARM64 - $(OutDir)$(TargetName).pgd - UseLinkTimeCodeGeneration - PGInstrument - PGUpdate + UseLinkTimeCodeGeneration advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;%(AdditionalDependencies) /OPT:REF,NOICF %(AdditionalOptions) -d2:-pattern-opt-disable:-932189325 %(AdditionalOptions) + /GENPROFILE %(AdditionalOptions) + /USEPROFILE %(AdditionalOptions) false - true - true - true + true $(PySourcePath)PC;$(PySourcePath)Include;$(IntDir);%(AdditionalIncludeDirectories) From 8e8786f8986353e20c1c4406c34409a6139fa073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20Kiss=20Koll=C3=A1r?= Date: Sat, 31 May 2025 03:32:36 +0300 Subject: [PATCH 030/116] gh-91048: Reorder result tuple of parse_code_object (#134898) Reorder result tuple of parse_code_object The standard followed by APIs like pstat.Stats is to take a file, line, function triplet. The parse_code_object function (and callers exposing this in Python like RemoteUnwinder.get_stack_trace) return function, file, line triplets which requires the caller to reorder these when using it in classes like pstat.Stats. --- Lib/test/test_external_inspection.py | 76 ++++++++++++++-------------- Modules/_remote_debugging_module.c | 8 +-- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 2b4b63a030b1af..303af25fc7a715 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -114,17 +114,17 @@ def foo(): p.wait(timeout=SHORT_TIMEOUT) thread_expected_stack_trace = [ - ("foo", script_name, 15), - ("baz", script_name, 12), - ("bar", script_name, 9), - ('Thread.run', threading.__file__, ANY) + (script_name, 15, "foo"), + (script_name, 12, "baz"), + (script_name, 9, "bar"), + (threading.__file__, ANY, 'Thread.run') ] # Is possible that there are more threads, so we check that the # expected stack traces are in the result (looking at you Windows!) self.assertIn((ANY, thread_expected_stack_trace), stack_trace) # Check that the main thread stack trace is in the result - frame = ("", script_name, 19) + frame = (script_name, 19, "") for _, stack in stack_trace: if frame in stack: break @@ -222,47 +222,47 @@ def new_eager_loop(): root_task = "Task-1" expected_stack_trace = [ [ - ("c5", script_name, 10), - ("c4", script_name, 14), - ("c3", script_name, 17), - ("c2", script_name, 20), + (script_name, 10, "c5"), + (script_name, 14, "c4"), + (script_name, 17, "c3"), + (script_name, 20, "c2"), ], "c2_root", [ [ [ ( - "TaskGroup._aexit", taskgroups.__file__, ANY, + "TaskGroup._aexit" ), ( - "TaskGroup.__aexit__", taskgroups.__file__, ANY, + "TaskGroup.__aexit__" ), - ("main", script_name, 26), + (script_name, 26, "main"), ], "Task-1", [], ], [ - [("c1", script_name, 23)], + [(script_name, 23, "c1")], "sub_main_1", [ [ [ ( - "TaskGroup._aexit", taskgroups.__file__, ANY, + "TaskGroup._aexit" ), ( - "TaskGroup.__aexit__", taskgroups.__file__, ANY, + "TaskGroup.__aexit__" ), - ("main", script_name, 26), + (script_name, 26, "main"), ], "Task-1", [], @@ -270,22 +270,22 @@ def new_eager_loop(): ], ], [ - [("c1", script_name, 23)], + [(script_name, 23, "c1")], "sub_main_2", [ [ [ ( - "TaskGroup._aexit", taskgroups.__file__, ANY, + "TaskGroup._aexit" ), ( - "TaskGroup.__aexit__", taskgroups.__file__, ANY, + "TaskGroup.__aexit__" ), - ("main", script_name, 26), + (script_name, 26, "main"), ], "Task-1", [], @@ -363,9 +363,9 @@ async def main(): expected_stack_trace = [ [ - ("gen_nested_call", script_name, 10), - ("gen", script_name, 16), - ("main", script_name, 19), + (script_name, 10, "gen_nested_call"), + (script_name, 16, "gen"), + (script_name, 19, "main"), ], "Task-1", [], @@ -439,9 +439,9 @@ async def main(): stack_trace[2].sort(key=lambda x: x[1]) expected_stack_trace = [ - [("deep", script_name, 11), ("c1", script_name, 15)], + [(script_name, 11, "deep"), (script_name, 15, "c1")], "Task-2", - [[[("main", script_name, 21)], "Task-1", []]], + [[[(script_name, 21, "main")], "Task-1", []]], ] self.assertEqual(stack_trace, expected_stack_trace) @@ -515,16 +515,16 @@ async def main(): stack_trace[2].sort(key=lambda x: x[1]) expected_stack_trace = [ [ - ("deep", script_name, 11), - ("c1", script_name, 15), - ("staggered_race..run_one_coro", staggered.__file__, ANY), + (script_name, 11, "deep"), + (script_name, 15, "c1"), + (staggered.__file__, ANY, "staggered_race..run_one_coro"), ], "Task-2", [ [ [ - ("staggered_race", staggered.__file__, ANY), - ("main", script_name, 21), + (staggered.__file__, ANY, "staggered_race"), + (script_name, 21, "main"), ], "Task-1", [], @@ -662,16 +662,16 @@ async def main(): self.assertIn((ANY, "Task-1", []), entries) main_stack = [ ( - "TaskGroup._aexit", taskgroups.__file__, ANY, + "TaskGroup._aexit", ), ( - "TaskGroup.__aexit__", taskgroups.__file__, ANY, + "TaskGroup.__aexit__", ), - ("main", script_name, 60), + (script_name, 60, "main"), ] self.assertIn( (ANY, "server task", [[main_stack, ANY]]), @@ -686,16 +686,16 @@ async def main(): [ [ ( - "TaskGroup._aexit", taskgroups.__file__, ANY, + "TaskGroup._aexit", ), ( - "TaskGroup.__aexit__", taskgroups.__file__, ANY, + "TaskGroup.__aexit__", ), - ("echo_client_spam", script_name, 41), + (script_name, 41, "echo_client_spam"), ], ANY, ] @@ -741,14 +741,14 @@ def test_self_trace(self): stack[:2], [ ( - "get_stack_trace", __file__, get_stack_trace.__code__.co_firstlineno + 2, + "get_stack_trace", ), ( - "TestGetStackTrace.test_self_trace", __file__, self.test_self_trace.__code__.co_firstlineno + 6, + "TestGetStackTrace.test_self_trace", ), ] ) diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index 86e269f31246e5..ea58f38006e199 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -1562,9 +1562,9 @@ parse_code_object(RemoteUnwinderObject *unwinder, Py_INCREF(meta->func_name); Py_INCREF(meta->file_name); - PyTuple_SET_ITEM(tuple, 0, meta->func_name); - PyTuple_SET_ITEM(tuple, 1, meta->file_name); - PyTuple_SET_ITEM(tuple, 2, lineno); + PyTuple_SET_ITEM(tuple, 0, meta->file_name); + PyTuple_SET_ITEM(tuple, 1, lineno); + PyTuple_SET_ITEM(tuple, 2, meta->func_name); *result = tuple; return 0; @@ -2921,4 +2921,4 @@ PyMODINIT_FUNC PyInit__remote_debugging(void) { return PyModuleDef_Init(&remote_debugging_module); -} \ No newline at end of file +} From 4d31d19a1df0a6e658e6a320cde8355f5f6ea27b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 31 May 2025 10:32:53 +0300 Subject: [PATCH 031/116] gh-134718: Omit optional Load() values in ast.dump() (GH-134934) --- Doc/library/ast.rst | 296 ++++++++---------- Lib/ast.py | 6 + Lib/test/test_ast/test_ast.py | 71 ++--- ...-05-30-18-13-48.gh-issue-134718.5FEspx.rst | 1 + 4 files changed, 177 insertions(+), 197 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-30-18-13-48.gh-issue-134718.5FEspx.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index ca0654acb33689..ef6c62dca1e124 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -252,12 +252,11 @@ Root nodes >>> print(ast.dump(ast.parse('(int, str) -> List[int]', mode='func_type'), indent=4)) FunctionType( argtypes=[ - Name(id='int', ctx=Load()), - Name(id='str', ctx=Load())], + Name(id='int'), + Name(id='str')], returns=Subscript( - value=Name(id='List', ctx=Load()), - slice=Name(id='int', ctx=Load()), - ctx=Load())) + value=Name(id='List'), + slice=Name(id='int'))) .. versionadded:: 3.8 @@ -312,14 +311,14 @@ Literals values=[ Constant(value='sin('), FormattedValue( - value=Name(id='a', ctx=Load()), + value=Name(id='a'), conversion=-1), Constant(value=') is '), FormattedValue( value=Call( - func=Name(id='sin', ctx=Load()), + func=Name(id='sin'), args=[ - Name(id='a', ctx=Load())]), + Name(id='a')]), conversion=-1, format_spec=JoinedStr( values=[ @@ -341,16 +340,14 @@ Literals elts=[ Constant(value=1), Constant(value=2), - Constant(value=3)], - ctx=Load())) + Constant(value=3)])) >>> print(ast.dump(ast.parse('(1, 2, 3)', mode='eval'), indent=4)) Expression( body=Tuple( elts=[ Constant(value=1), Constant(value=2), - Constant(value=3)], - ctx=Load())) + Constant(value=3)])) .. class:: Set(elts) @@ -388,7 +385,7 @@ Literals None], values=[ Constant(value=1), - Name(id='d', ctx=Load())])) + Name(id='d')])) Variables @@ -414,7 +411,7 @@ Variables Module( body=[ Expr( - value=Name(id='a', ctx=Load()))]) + value=Name(id='a'))]) >>> print(ast.dump(ast.parse('a = 1'), indent=4)) Module( @@ -452,7 +449,7 @@ Variables value=Name(id='b', ctx=Store()), ctx=Store())], ctx=Store())], - value=Name(id='it', ctx=Load()))]) + value=Name(id='it'))]) .. _ast-expressions: @@ -475,7 +472,7 @@ Expressions Expr( value=UnaryOp( op=USub(), - operand=Name(id='a', ctx=Load())))]) + operand=Name(id='a')))]) .. class:: UnaryOp(op, operand) @@ -498,7 +495,7 @@ Expressions Expression( body=UnaryOp( op=Not(), - operand=Name(id='x', ctx=Load()))) + operand=Name(id='x'))) .. class:: BinOp(left, op, right) @@ -511,9 +508,9 @@ Expressions >>> print(ast.dump(ast.parse('x + y', mode='eval'), indent=4)) Expression( body=BinOp( - left=Name(id='x', ctx=Load()), + left=Name(id='x'), op=Add(), - right=Name(id='y', ctx=Load()))) + right=Name(id='y'))) .. class:: Add @@ -549,8 +546,8 @@ Expressions body=BoolOp( op=Or(), values=[ - Name(id='x', ctx=Load()), - Name(id='y', ctx=Load())])) + Name(id='x'), + Name(id='y')])) .. class:: And @@ -575,7 +572,7 @@ Expressions LtE(), Lt()], comparators=[ - Name(id='a', ctx=Load()), + Name(id='a'), Constant(value=10)])) @@ -609,18 +606,17 @@ Expressions >>> print(ast.dump(ast.parse('func(a, b=c, *d, **e)', mode='eval'), indent=4)) Expression( body=Call( - func=Name(id='func', ctx=Load()), + func=Name(id='func'), args=[ - Name(id='a', ctx=Load()), + Name(id='a'), Starred( - value=Name(id='d', ctx=Load()), - ctx=Load())], + value=Name(id='d'))], keywords=[ keyword( arg='b', - value=Name(id='c', ctx=Load())), + value=Name(id='c')), keyword( - value=Name(id='e', ctx=Load()))])) + value=Name(id='e'))])) .. class:: keyword(arg, value) @@ -639,9 +635,9 @@ Expressions >>> print(ast.dump(ast.parse('a if b else c', mode='eval'), indent=4)) Expression( body=IfExp( - test=Name(id='b', ctx=Load()), - body=Name(id='a', ctx=Load()), - orelse=Name(id='c', ctx=Load()))) + test=Name(id='b'), + body=Name(id='a'), + orelse=Name(id='c'))) .. class:: Attribute(value, attr, ctx) @@ -656,9 +652,8 @@ Expressions >>> print(ast.dump(ast.parse('snake.colour', mode='eval'), indent=4)) Expression( body=Attribute( - value=Name(id='snake', ctx=Load()), - attr='colour', - ctx=Load())) + value=Name(id='snake'), + attr='colour')) .. class:: NamedExpr(target, value) @@ -694,15 +689,13 @@ Subscripting >>> print(ast.dump(ast.parse('l[1:2, 3]', mode='eval'), indent=4)) Expression( body=Subscript( - value=Name(id='l', ctx=Load()), + value=Name(id='l'), slice=Tuple( elts=[ Slice( lower=Constant(value=1), upper=Constant(value=2)), - Constant(value=3)], - ctx=Load()), - ctx=Load())) + Constant(value=3)]))) .. class:: Slice(lower, upper, step) @@ -716,11 +709,10 @@ Subscripting >>> print(ast.dump(ast.parse('l[1:2]', mode='eval'), indent=4)) Expression( body=Subscript( - value=Name(id='l', ctx=Load()), + value=Name(id='l'), slice=Slice( lower=Constant(value=1), - upper=Constant(value=2)), - ctx=Load())) + upper=Constant(value=2)))) Comprehensions @@ -745,11 +737,11 @@ Comprehensions ... )) Expression( body=ListComp( - elt=Name(id='x', ctx=Load()), + elt=Name(id='x'), generators=[ comprehension( target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), + iter=Name(id='numbers'), is_async=0)])) >>> print(ast.dump( ... ast.parse('{x: x**2 for x in numbers}', mode='eval'), @@ -757,15 +749,15 @@ Comprehensions ... )) Expression( body=DictComp( - key=Name(id='x', ctx=Load()), + key=Name(id='x'), value=BinOp( - left=Name(id='x', ctx=Load()), + left=Name(id='x'), op=Pow(), right=Constant(value=2)), generators=[ comprehension( target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), + iter=Name(id='numbers'), is_async=0)])) >>> print(ast.dump( ... ast.parse('{x for x in numbers}', mode='eval'), @@ -773,11 +765,11 @@ Comprehensions ... )) Expression( body=SetComp( - elt=Name(id='x', ctx=Load()), + elt=Name(id='x'), generators=[ comprehension( target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), + iter=Name(id='numbers'), is_async=0)])) @@ -798,17 +790,17 @@ Comprehensions Expression( body=ListComp( elt=Call( - func=Name(id='ord', ctx=Load()), + func=Name(id='ord'), args=[ - Name(id='c', ctx=Load())]), + Name(id='c')]), generators=[ comprehension( target=Name(id='line', ctx=Store()), - iter=Name(id='file', ctx=Load()), + iter=Name(id='file'), is_async=0), comprehension( target=Name(id='c', ctx=Store()), - iter=Name(id='line', ctx=Load()), + iter=Name(id='line'), is_async=0)])) >>> print(ast.dump(ast.parse('(n**2 for n in it if n>5 if n<10)', mode='eval'), @@ -816,22 +808,22 @@ Comprehensions Expression( body=GeneratorExp( elt=BinOp( - left=Name(id='n', ctx=Load()), + left=Name(id='n'), op=Pow(), right=Constant(value=2)), generators=[ comprehension( target=Name(id='n', ctx=Store()), - iter=Name(id='it', ctx=Load()), + iter=Name(id='it'), ifs=[ Compare( - left=Name(id='n', ctx=Load()), + left=Name(id='n'), ops=[ Gt()], comparators=[ Constant(value=5)]), Compare( - left=Name(id='n', ctx=Load()), + left=Name(id='n'), ops=[ Lt()], comparators=[ @@ -842,11 +834,11 @@ Comprehensions ... indent=4)) # Async comprehension Expression( body=ListComp( - elt=Name(id='i', ctx=Load()), + elt=Name(id='i'), generators=[ comprehension( target=Name(id='i', ctx=Store()), - iter=Name(id='soc', ctx=Load()), + iter=Name(id='soc'), is_async=1)])) @@ -888,7 +880,7 @@ Statements Name(id='a', ctx=Store()), Name(id='b', ctx=Store())], ctx=Store())], - value=Name(id='c', ctx=Load()))]) + value=Name(id='c'))]) .. class:: AnnAssign(target, annotation, value, simple) @@ -911,7 +903,7 @@ Statements body=[ AnnAssign( target=Name(id='c', ctx=Store()), - annotation=Name(id='int', ctx=Load()), + annotation=Name(id='int'), simple=1)]) >>> print(ast.dump(ast.parse('(a): int = 1'), indent=4)) # Annotation with parenthesis @@ -919,7 +911,7 @@ Statements body=[ AnnAssign( target=Name(id='a', ctx=Store()), - annotation=Name(id='int', ctx=Load()), + annotation=Name(id='int'), value=Constant(value=1), simple=0)]) @@ -928,10 +920,10 @@ Statements body=[ AnnAssign( target=Attribute( - value=Name(id='a', ctx=Load()), + value=Name(id='a'), attr='b', ctx=Store()), - annotation=Name(id='int', ctx=Load()), + annotation=Name(id='int'), simple=0)]) >>> print(ast.dump(ast.parse('a[1]: int'), indent=4)) # Subscript annotation @@ -939,10 +931,10 @@ Statements body=[ AnnAssign( target=Subscript( - value=Name(id='a', ctx=Load()), + value=Name(id='a'), slice=Constant(value=1), ctx=Store()), - annotation=Name(id='int', ctx=Load()), + annotation=Name(id='int'), simple=0)]) @@ -979,8 +971,8 @@ Statements Module( body=[ Raise( - exc=Name(id='x', ctx=Load()), - cause=Name(id='y', ctx=Load()))]) + exc=Name(id='x'), + cause=Name(id='y'))]) .. class:: Assert(test, msg) @@ -994,8 +986,8 @@ Statements Module( body=[ Assert( - test=Name(id='x', ctx=Load()), - msg=Name(id='y', ctx=Load()))]) + test=Name(id='x'), + msg=Name(id='y'))]) .. class:: Delete(targets) @@ -1041,7 +1033,7 @@ Statements body=[ TypeAlias( name=Name(id='Alias', ctx=Store()), - value=Name(id='int', ctx=Load()))]) + value=Name(id='int'))]) .. versionadded:: 3.12 @@ -1134,13 +1126,13 @@ Control flow Module( body=[ If( - test=Name(id='x', ctx=Load()), + test=Name(id='x'), body=[ Expr( value=Constant(value=Ellipsis))], orelse=[ If( - test=Name(id='y', ctx=Load()), + test=Name(id='y'), body=[ Expr( value=Constant(value=Ellipsis))], @@ -1174,7 +1166,7 @@ Control flow body=[ For( target=Name(id='x', ctx=Store()), - iter=Name(id='y', ctx=Load()), + iter=Name(id='y'), body=[ Expr( value=Constant(value=Ellipsis))], @@ -1199,7 +1191,7 @@ Control flow Module( body=[ While( - test=Name(id='x', ctx=Load()), + test=Name(id='x'), body=[ Expr( value=Constant(value=Ellipsis))], @@ -1227,11 +1219,11 @@ Control flow body=[ For( target=Name(id='a', ctx=Store()), - iter=Name(id='b', ctx=Load()), + iter=Name(id='b'), body=[ If( test=Compare( - left=Name(id='a', ctx=Load()), + left=Name(id='a'), ops=[ Gt()], comparators=[ @@ -1269,12 +1261,12 @@ Control flow value=Constant(value=Ellipsis))], handlers=[ ExceptHandler( - type=Name(id='Exception', ctx=Load()), + type=Name(id='Exception'), body=[ Expr( value=Constant(value=Ellipsis))]), ExceptHandler( - type=Name(id='OtherException', ctx=Load()), + type=Name(id='OtherException'), name='e', body=[ Expr( @@ -1309,7 +1301,7 @@ Control flow value=Constant(value=Ellipsis))], handlers=[ ExceptHandler( - type=Name(id='Exception', ctx=Load()), + type=Name(id='Exception'), body=[ Expr( value=Constant(value=Ellipsis))])])]) @@ -1337,12 +1329,12 @@ Control flow body=[ Expr( value=BinOp( - left=Name(id='a', ctx=Load()), + left=Name(id='a'), op=Add(), right=Constant(value=1)))], handlers=[ ExceptHandler( - type=Name(id='TypeError', ctx=Load()), + type=Name(id='TypeError'), body=[ Pass()])])]) @@ -1375,18 +1367,18 @@ Control flow With( items=[ withitem( - context_expr=Name(id='a', ctx=Load()), + context_expr=Name(id='a'), optional_vars=Name(id='b', ctx=Store())), withitem( - context_expr=Name(id='c', ctx=Load()), + context_expr=Name(id='c'), optional_vars=Name(id='d', ctx=Store()))], body=[ Expr( value=Call( - func=Name(id='something', ctx=Load()), + func=Name(id='something'), args=[ - Name(id='b', ctx=Load()), - Name(id='d', ctx=Load())]))])]) + Name(id='b'), + Name(id='d')]))])]) Pattern matching @@ -1426,14 +1418,14 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchSequence( patterns=[ MatchAs(name='x')]), guard=Compare( - left=Name(id='x', ctx=Load()), + left=Name(id='x'), ops=[ Gt()], comparators=[ @@ -1443,7 +1435,7 @@ Pattern matching value=Constant(value=Ellipsis))]), match_case( pattern=MatchClass( - cls=Name(id='tuple', ctx=Load())), + cls=Name(id='tuple')), body=[ Expr( value=Constant(value=Ellipsis))])])]) @@ -1467,7 +1459,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchValue( @@ -1494,7 +1486,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchSingleton(value=None), @@ -1521,7 +1513,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchSequence( @@ -1554,7 +1546,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchSequence( @@ -1603,7 +1595,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchMapping( @@ -1653,11 +1645,11 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchClass( - cls=Name(id='Point2D', ctx=Load()), + cls=Name(id='Point2D'), patterns=[ MatchValue( value=Constant(value=0)), @@ -1668,7 +1660,7 @@ Pattern matching value=Constant(value=Ellipsis))]), match_case( pattern=MatchClass( - cls=Name(id='Point3D', ctx=Load()), + cls=Name(id='Point3D'), kwd_attrs=[ 'x', 'y', @@ -1709,7 +1701,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchAs( @@ -1746,7 +1738,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchOr( @@ -1786,7 +1778,7 @@ Type annotations body=[ AnnAssign( target=Name(id='x', ctx=Store()), - annotation=Name(id='bool', ctx=Load()), + annotation=Name(id='bool'), value=Constant(value=1), simple=1)], type_ignores=[ @@ -1824,12 +1816,11 @@ aliases. type_params=[ TypeVar( name='T', - bound=Name(id='int', ctx=Load()), - default_value=Name(id='bool', ctx=Load()))], + bound=Name(id='int'), + default_value=Name(id='bool'))], value=Subscript( - value=Name(id='list', ctx=Load()), - slice=Name(id='T', ctx=Load()), - ctx=Load()))]) + value=Name(id='list'), + slice=Name(id='T')))]) .. versionadded:: 3.12 @@ -1854,17 +1845,14 @@ aliases. name='P', default_value=List( elts=[ - Name(id='int', ctx=Load()), - Name(id='str', ctx=Load())], - ctx=Load()))], + Name(id='int'), + Name(id='str')]))], value=Subscript( - value=Name(id='Callable', ctx=Load()), + value=Name(id='Callable'), slice=Tuple( elts=[ - Name(id='P', ctx=Load()), - Name(id='int', ctx=Load())], - ctx=Load()), - ctx=Load()))]) + Name(id='P'), + Name(id='int')])))]) .. versionadded:: 3.12 @@ -1885,18 +1873,13 @@ aliases. TypeAlias( name=Name(id='Alias', ctx=Store()), type_params=[ - TypeVarTuple( - name='Ts', - default_value=Tuple(ctx=Load()))], + TypeVarTuple(name='Ts', default_value=Tuple())], value=Subscript( - value=Name(id='tuple', ctx=Load()), + value=Name(id='tuple'), slice=Tuple( elts=[ Starred( - value=Name(id='Ts', ctx=Load()), - ctx=Load())], - ctx=Load()), - ctx=Load()))]) + value=Name(id='Ts'))])))]) .. versionadded:: 3.12 @@ -2001,8 +1984,8 @@ Function and class definitions body=[ Pass()], decorator_list=[ - Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())], + Name(id='decorator1'), + Name(id='decorator2')], returns=Constant(value='return annotation'))]) @@ -2032,14 +2015,14 @@ Function and class definitions body=[ Expr( value=Yield( - value=Name(id='x', ctx=Load())))]) + value=Name(id='x')))]) >>> print(ast.dump(ast.parse('yield from x'), indent=4)) Module( body=[ Expr( value=YieldFrom( - value=Name(id='x', ctx=Load())))]) + value=Name(id='x')))]) .. class:: Global(names) @@ -2094,17 +2077,17 @@ Function and class definitions ClassDef( name='Foo', bases=[ - Name(id='base1', ctx=Load()), - Name(id='base2', ctx=Load())], + Name(id='base1'), + Name(id='base2')], keywords=[ keyword( arg='metaclass', - value=Name(id='meta', ctx=Load()))], + value=Name(id='meta'))], body=[ Pass()], decorator_list=[ - Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())])]) + Name(id='decorator1'), + Name(id='decorator2')])]) .. versionchanged:: 3.12 Added ``type_params``. @@ -2141,7 +2124,7 @@ Async and await Expr( value=Await( value=Call( - func=Name(id='other_func', ctx=Load()))))])]) + func=Name(id='other_func'))))])]) .. class:: AsyncFor(target, iter, body, orelse, type_comment) @@ -2402,7 +2385,7 @@ and classes for traversing abstract syntax trees: def visit_Name(self, node): return Subscript( - value=Name(id='data', ctx=Load()), + value=Name(id='data'), slice=Constant(value=node.id), ctx=node.ctx ) @@ -2445,42 +2428,35 @@ and classes for traversing abstract syntax trees: indents that many spaces per level. If *indent* is a string (such as ``"\t"``), that string is used to indent each level. - If *show_empty* is false (the default), optional empty lists will be - omitted from the output. + If *show_empty* is false (the default), optional empty lists and + ``Load()`` values will be omitted from the output. Optional ``None`` values are always omitted. + .. doctest:: + + >>> tree = ast.parse('print(None)', '?', 'eval') + >>> print(ast.dump(tree, indent=4)) + Expression( + body=Call( + func=Name(id='print'), + args=[ + Constant(value=None)])) + >>> print(ast.dump(tree, indent=4, show_empty=True)) + Expression( + body=Call( + func=Name(id='print', ctx=Load()), + args=[ + Constant(value=None)], + keywords=[])) + .. versionchanged:: 3.9 Added the *indent* option. .. versionchanged:: 3.13 Added the *show_empty* option. - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... async def f(): - ... await other_func() - ... """), indent=4, show_empty=True)) - Module( - body=[ - AsyncFunctionDef( - name='f', - args=arguments( - posonlyargs=[], - args=[], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), - body=[ - Expr( - value=Await( - value=Call( - func=Name(id='other_func', ctx=Load()), - args=[], - keywords=[])))], - decorator_list=[], - type_params=[])], - type_ignores=[]) + .. versionchanged:: next + Omit optional ``Load()`` values by default. .. _ast-compiler-flags: diff --git a/Lib/ast.py b/Lib/ast.py index 2f11683ecf7c68..6d3daf64f5c6d7 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -154,6 +154,12 @@ def _format(node, level=0): if not keywords: args_buffer.append(repr(value)) continue + elif isinstance(value, Load): + field_type = cls._field_types.get(name, object) + if field_type is expr_context: + if not keywords: + args_buffer.append(repr(value)) + continue if not keywords: args.extend(args_buffer) args_buffer = [] diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 59263012bc1440..cc46529c0ef105 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -1372,17 +1372,17 @@ def test_parse_in_error(self): def test_dump(self): node = ast.parse('spam(eggs, "and cheese")') self.assertEqual(ast.dump(node), - "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " - "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])" + "Module(body=[Expr(value=Call(func=Name(id='spam'), " + "args=[Name(id='eggs'), Constant(value='and cheese')]))])" ) self.assertEqual(ast.dump(node, annotate_fields=False), - "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " + "Module([Expr(Call(Name('spam'), [Name('eggs'), " "Constant('and cheese')]))])" ) self.assertEqual(ast.dump(node, include_attributes=True), - "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " + "Module(body=[Expr(value=Call(func=Name(id='spam', " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), " - "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, " + "args=[Name(id='eggs', lineno=1, col_offset=5, " "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " @@ -1396,18 +1396,18 @@ def test_dump_indent(self): body=[ Expr( value=Call( - func=Name(id='spam', ctx=Load()), + func=Name(id='spam'), args=[ - Name(id='eggs', ctx=Load()), + Name(id='eggs'), Constant(value='and cheese')]))])""") self.assertEqual(ast.dump(node, annotate_fields=False, indent='\t'), """\ Module( \t[ \t\tExpr( \t\t\tCall( -\t\t\t\tName('spam', Load()), +\t\t\t\tName('spam'), \t\t\t\t[ -\t\t\t\t\tName('eggs', Load()), +\t\t\t\t\tName('eggs'), \t\t\t\t\tConstant('and cheese')]))])""") self.assertEqual(ast.dump(node, include_attributes=True, indent=3), """\ Module( @@ -1416,7 +1416,6 @@ def test_dump_indent(self): value=Call( func=Name( id='spam', - ctx=Load(), lineno=1, col_offset=0, end_lineno=1, @@ -1424,7 +1423,6 @@ def test_dump_indent(self): args=[ Name( id='eggs', - ctx=Load(), lineno=1, col_offset=5, end_lineno=1, @@ -1454,23 +1452,23 @@ def test_dump_incomplete(self): ) node = ast.Raise(exc=ast.Name(id='e', ctx=ast.Load()), lineno=3, col_offset=4) self.assertEqual(ast.dump(node), - "Raise(exc=Name(id='e', ctx=Load()))" + "Raise(exc=Name(id='e'))" ) self.assertEqual(ast.dump(node, annotate_fields=False), - "Raise(Name('e', Load()))" + "Raise(Name('e'))" ) self.assertEqual(ast.dump(node, include_attributes=True), - "Raise(exc=Name(id='e', ctx=Load()), lineno=3, col_offset=4)" + "Raise(exc=Name(id='e'), lineno=3, col_offset=4)" ) self.assertEqual(ast.dump(node, annotate_fields=False, include_attributes=True), - "Raise(Name('e', Load()), lineno=3, col_offset=4)" + "Raise(Name('e'), lineno=3, col_offset=4)" ) node = ast.Raise(cause=ast.Name(id='e', ctx=ast.Load())) self.assertEqual(ast.dump(node), - "Raise(cause=Name(id='e', ctx=Load()))" + "Raise(cause=Name(id='e'))" ) self.assertEqual(ast.dump(node, annotate_fields=False), - "Raise(cause=Name('e', Load()))" + "Raise(cause=Name('e'))" ) # Arguments: node = ast.arguments(args=[ast.arg("x")]) @@ -1502,10 +1500,10 @@ def test_dump_incomplete(self): [ast.Name('dataclass', ctx=ast.Load())], ) self.assertEqual(ast.dump(node), - "ClassDef(name='T', keywords=[keyword(arg='a', value=Constant(value=None))], decorator_list=[Name(id='dataclass', ctx=Load())])", + "ClassDef(name='T', keywords=[keyword(arg='a', value=Constant(value=None))], decorator_list=[Name(id='dataclass')])", ) self.assertEqual(ast.dump(node, annotate_fields=False), - "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass', Load())])", + "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass')])", ) def test_dump_show_empty(self): @@ -1533,7 +1531,7 @@ def check_text(code, empty, full, **kwargs): check_node( # Corner case: there are no real `Name` instances with `id=''`: ast.Name(id='', ctx=ast.Load()), - empty="Name(id='', ctx=Load())", + empty="Name(id='')", full="Name(id='', ctx=Load())", ) @@ -1581,26 +1579,26 @@ def check_text(code, empty, full, **kwargs): check_text( "def a(b: int = 0, *, c): ...", - empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))])])", + empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int'))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))])])", full="Module(body=[FunctionDef(name='a', args=arguments(posonlyargs=[], args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))], decorator_list=[], type_params=[])], type_ignores=[])", ) check_text( "def a(b: int = 0, *, c): ...", - empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load(), lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)])", + empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)])", full="Module(body=[FunctionDef(name='a', args=arguments(posonlyargs=[], args=[arg(arg='b', annotation=Name(id='int', ctx=Load(), lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], decorator_list=[], type_params=[], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)], type_ignores=[])", include_attributes=True, ) check_text( 'spam(eggs, "and cheese")', - empty="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])", + empty="Module(body=[Expr(value=Call(func=Name(id='spam'), args=[Name(id='eggs'), Constant(value='and cheese')]))])", full="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], keywords=[]))], type_ignores=[])", ) check_text( 'spam(eggs, text="and cheese")', - empty="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load())], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))])", + empty="Module(body=[Expr(value=Call(func=Name(id='spam'), args=[Name(id='eggs')], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))])", full="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load())], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))], type_ignores=[])", ) @@ -1634,12 +1632,12 @@ def test_fix_missing_locations(self): self.assertEqual(src, ast.fix_missing_locations(src)) self.maxDiff = None self.assertEqual(ast.dump(src, include_attributes=True), - "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " + "Module(body=[Expr(value=Call(func=Name(id='write', " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), " "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, " "end_col_offset=12)], lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), " + "end_col_offset=13), Expr(value=Call(func=Name(id='spam', " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), " "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=0)], lineno=1, col_offset=0, end_lineno=1, " @@ -3359,7 +3357,7 @@ def test_exec_mode_flag(self): body=[ AnnAssign( target=Name(id='x', ctx=Store()), - annotation=Name(id='bool', ctx=Load()), + annotation=Name(id='bool'), value=Constant(value=1), simple=1)], type_ignores=[ @@ -3387,7 +3385,7 @@ def test_eval_mode_flag(self): expect = ''' Expression( body=Call( - func=Name(id='print', ctx=Load()), + func=Name(id='print'), args=[ Constant(value=1), Constant(value=2), @@ -3403,12 +3401,11 @@ def test_func_type_mode_flag(self): expect = ''' FunctionType( argtypes=[ - Name(id='int', ctx=Load()), - Name(id='str', ctx=Load())], + Name(id='int'), + Name(id='str')], returns=Subscript( - value=Name(id='list', ctx=Load()), - slice=Name(id='int', ctx=Load()), - ctx=Load())) + value=Name(id='list'), + slice=Name(id='int'))) ''' for flag in ('-m=func_type', '--mode=func_type'): with self.subTest(flag=flag): @@ -3422,7 +3419,7 @@ def test_no_type_comments_flag(self): body=[ AnnAssign( target=Name(id='x', ctx=Store()), - annotation=Name(id='bool', ctx=Load()), + annotation=Name(id='bool'), value=Constant(value=1), simple=1)]) ''' @@ -3467,7 +3464,7 @@ def test_feature_version_flag(self): Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchValue( @@ -3490,7 +3487,7 @@ def test_no_optimize_flag(self): Module( body=[ Match( - subject=Name(id='a', ctx=Load()), + subject=Name(id='a'), cases=[ match_case( pattern=MatchValue( @@ -3516,7 +3513,7 @@ def test_optimize_flag(self): Module( body=[ Match( - subject=Name(id='a', ctx=Load()), + subject=Name(id='a'), cases=[ match_case( pattern=MatchValue( diff --git a/Misc/NEWS.d/next/Library/2025-05-30-18-13-48.gh-issue-134718.5FEspx.rst b/Misc/NEWS.d/next/Library/2025-05-30-18-13-48.gh-issue-134718.5FEspx.rst new file mode 100644 index 00000000000000..06c1d5583be0bb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-30-18-13-48.gh-issue-134718.5FEspx.rst @@ -0,0 +1 @@ +By default, omit optional ``Load()`` values in :func:`ast.dump`. From c6e63d9d351f6d952000ec3bf84b3a7607989f92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 31 May 2025 09:37:47 +0200 Subject: [PATCH 032/116] gh-134696: align OpenSSL and HACL*-based hash functions constructors AC signatures (#134713) OpenSSL and HACL*-based hash functions constructors now support both `data` and `string` parameters. Previously these constructor functions inconsistently supported sometimes `data` and sometimes `string`, while the documentation expected `data` to be given in all cases. --- Lib/hashlib.py | 12 +- Lib/test/test_hashlib.py | 62 ++- ...-05-26-14-04-39.gh-issue-134696.P04xUa.rst | 5 + Modules/_hashopenssl.c | 153 +++--- Modules/blake2module.c | 30 +- Modules/clinic/_hashopenssl.c.h | 441 +++++++++++------- Modules/clinic/blake2module.c.h | 98 ++-- Modules/clinic/md5module.c.h | 34 +- Modules/clinic/sha1module.c.h | 34 +- Modules/clinic/sha2module.c.h | 130 ++++-- Modules/clinic/sha3module.c.h | 128 ++++- Modules/hashlib.h | 29 ++ Modules/md5module.c | 13 +- Modules/sha1module.c | 12 +- Modules/sha2module.c | 49 +- Modules/sha3module.c | 21 +- 16 files changed, 830 insertions(+), 421 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-26-14-04-39.gh-issue-134696.P04xUa.rst diff --git a/Lib/hashlib.py b/Lib/hashlib.py index abacac22ea0106..0e9bd98aa1fc31 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -141,29 +141,29 @@ def __get_openssl_constructor(name): return __get_builtin_constructor(name) -def __py_new(name, data=b'', **kwargs): +def __py_new(name, *args, **kwargs): """new(name, data=b'', **kwargs) - Return a new hashing object using the named algorithm; optionally initialized with data (which must be a bytes-like object). """ - return __get_builtin_constructor(name)(data, **kwargs) + return __get_builtin_constructor(name)(*args, **kwargs) -def __hash_new(name, data=b'', **kwargs): +def __hash_new(name, *args, **kwargs): """new(name, data=b'') - Return a new hashing object using the named algorithm; optionally initialized with data (which must be a bytes-like object). """ if name in __block_openssl_constructor: # Prefer our builtin blake2 implementation. - return __get_builtin_constructor(name)(data, **kwargs) + return __get_builtin_constructor(name)(*args, **kwargs) try: - return _hashlib.new(name, data, **kwargs) + return _hashlib.new(name, *args, **kwargs) except ValueError: # If the _hashlib module (OpenSSL) doesn't support the named # hash, try using our builtin implementations. # This allows for SHA224/256 and SHA384/512 support even though # the OpenSSL library prior to 0.9.8 doesn't provide them. - return __get_builtin_constructor(name)(data) + return __get_builtin_constructor(name)(*args, **kwargs) try: diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 161c7652d7ab11..51b82fe3b516b5 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -12,6 +12,7 @@ import itertools import logging import os +import re import sys import sysconfig import tempfile @@ -140,11 +141,10 @@ def __init__(self, *args, **kwargs): # of hashlib.new given the algorithm name. for algorithm, constructors in self.constructors_to_test.items(): constructors.add(getattr(hashlib, algorithm)) - def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs): - if data is None: - return hashlib.new(_alg, **kwargs) - return hashlib.new(_alg, data, **kwargs) - constructors.add(_test_algorithm_via_hashlib_new) + def c(*args, __algorithm_name=algorithm, **kwargs): + return hashlib.new(__algorithm_name, *args, **kwargs) + c.__name__ = f'do_test_algorithm_via_hashlib_new_{algorithm}' + constructors.add(c) _hashlib = self._conditional_import_module('_hashlib') self._hashlib = _hashlib @@ -249,6 +249,56 @@ def test_usedforsecurity_false(self): self._hashlib.new("md5", usedforsecurity=False) self._hashlib.openssl_md5(usedforsecurity=False) + def test_clinic_signature(self): + for constructor in self.hash_constructors: + with self.subTest(constructor.__name__): + constructor(b'') + constructor(data=b'') + constructor(string=b'') # should be deprecated in the future + + def test_clinic_signature_errors(self): + nomsg = b'' + mymsg = b'msg' + conflicting_call = re.escape( + "'data' and 'string' are mutually exclusive " + "and support for 'string' keyword parameter " + "is slated for removal in a future version." + ) + duplicated_param = re.escape("given by name ('data') and position") + unexpected_param = re.escape("got an unexpected keyword argument '_'") + for args, kwds, errmsg in [ + # Reject duplicated arguments before unknown keyword arguments. + ((nomsg,), dict(data=nomsg, _=nomsg), duplicated_param), + ((mymsg,), dict(data=nomsg, _=nomsg), duplicated_param), + # Reject duplicated arguments before conflicting ones. + *itertools.product( + [[nomsg], [mymsg]], + [dict(data=nomsg), dict(data=nomsg, string=nomsg)], + [duplicated_param] + ), + # Reject unknown keyword arguments before conflicting ones. + *itertools.product( + [()], + [ + dict(_=None), + dict(data=nomsg, _=None), + dict(string=nomsg, _=None), + dict(string=nomsg, data=nomsg, _=None), + ], + [unexpected_param] + ), + ((nomsg,), dict(_=None), unexpected_param), + ((mymsg,), dict(_=None), unexpected_param), + # Reject conflicting arguments. + [(nomsg,), dict(string=nomsg), conflicting_call], + [(mymsg,), dict(string=nomsg), conflicting_call], + [(), dict(data=nomsg, string=nomsg), conflicting_call], + ]: + for constructor in self.hash_constructors: + with self.subTest(constructor.__name__, args=args, kwds=kwds): + with self.assertRaisesRegex(TypeError, errmsg): + constructor(*args, **kwds) + def test_unknown_hash(self): self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam') self.assertRaises(TypeError, hashlib.new, 1) @@ -718,8 +768,6 @@ def check_blake2(self, constructor, salt_size, person_size, key_size, self.assertRaises(ValueError, constructor, node_offset=-1) self.assertRaises(OverflowError, constructor, node_offset=max_offset+1) - self.assertRaises(TypeError, constructor, data=b'') - self.assertRaises(TypeError, constructor, string=b'') self.assertRaises(TypeError, constructor, '') constructor( diff --git a/Misc/NEWS.d/next/Library/2025-05-26-14-04-39.gh-issue-134696.P04xUa.rst b/Misc/NEWS.d/next/Library/2025-05-26-14-04-39.gh-issue-134696.P04xUa.rst new file mode 100644 index 00000000000000..282eb088b89503 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-26-14-04-39.gh-issue-134696.P04xUa.rst @@ -0,0 +1,5 @@ +Built-in HACL* and OpenSSL implementations of hash function constructors +now correctly accept the same *documented* named arguments. For instance, +:func:`~hashlib.md5` could be previously invoked as ``md5(data=data)`` +or ``md5(string=string)`` depending on the underlying implementation +but these calls were not compatible. Patch by Bénédikt Tran. diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index dab0bb9b67fa00..331275076d7937 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -1039,6 +1039,14 @@ _hashlib_HASH(PyObject *module, const char *digestname, PyObject *data_obj, return (PyObject *)self; } +#define CALL_HASHLIB_NEW(MODULE, NAME, DATA, STRING, USEDFORSECURITY) \ + do { \ + PyObject *data_obj; \ + if (_Py_hashlib_data_argument(&data_obj, DATA, STRING) < 0) { \ + return NULL; \ + } \ + return _hashlib_HASH(MODULE, NAME, data_obj, USEDFORSECURITY); \ + } while (0) /* The module-level function: new() */ @@ -1046,9 +1054,10 @@ _hashlib_HASH(PyObject *module, const char *digestname, PyObject *data_obj, _hashlib.new as _hashlib_HASH_new name: str - string as data_obj: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Return a new hash object using the named algorithm. @@ -1059,131 +1068,137 @@ The MD5 and SHA1 algorithms are always supported. [clinic start generated code]*/ static PyObject * -_hashlib_HASH_new_impl(PyObject *module, const char *name, - PyObject *data_obj, int usedforsecurity) -/*[clinic end generated code: output=30c6e7b9a5a4dce3 input=28848db5ccd0a9b5]*/ +_hashlib_HASH_new_impl(PyObject *module, const char *name, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=b905aaf9840c1bbd input=c34af6c6e696d44e]*/ { - return _hashlib_HASH(module, name, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, name, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_md5 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a md5 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=87b0186440a44f8c input=990e36d5e689b16e]*/ +_hashlib_openssl_md5_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=ca8cf184d90f7432 input=e7c0adbd6a867db1]*/ { - return _hashlib_HASH(module, Py_hash_md5, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_md5, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha1 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha1 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=6813024cf690670d input=948f2f4b6deabc10]*/ +_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=1736fb7b310d64be input=f7e5bb1711e952d8]*/ { - return _hashlib_HASH(module, Py_hash_sha1, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha1, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha224 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha224 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=a2dfe7cc4eb14ebb input=f9272821fadca505]*/ +_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=0d6ff57be5e5c140 input=3820fff7ed3a53b8]*/ { - return _hashlib_HASH(module, Py_hash_sha224, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha224, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha256 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha256 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=1f874a34870f0a68 input=549fad9d2930d4c5]*/ +_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=412ea7111555b6e7 input=9a2f115cf1f7e0eb]*/ { - return _hashlib_HASH(module, Py_hash_sha256, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha256, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha384 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha384 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=58529eff9ca457b2 input=48601a6e3bf14ad7]*/ +_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=2e0dc395b59ed726 input=1ea48f6f01e77cfb]*/ { - return _hashlib_HASH(module, Py_hash_sha384, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha384, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha512 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha512 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=2c744c9e4a40d5f6 input=c5c46a2a817aa98f]*/ +_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=4bdd760388dbfc0f input=3cf56903e07d1f5c]*/ { - return _hashlib_HASH(module, Py_hash_sha512, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha512, data, string, usedforsecurity); } @@ -1192,77 +1207,81 @@ _hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj, /*[clinic input] _hashlib.openssl_sha3_224 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha3-224 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha3_224_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=144641c1d144b974 input=e3a01b2888916157]*/ +_hashlib_openssl_sha3_224_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=6d8dc2a924f3ba35 input=7f14f16a9f6a3158]*/ { - return _hashlib_HASH(module, Py_hash_sha3_224, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha3_224, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha3_256 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha3-256 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha3_256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=c61f1ab772d06668 input=e2908126c1b6deed]*/ +_hashlib_openssl_sha3_256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=9e520f537b3a4622 input=7987150939d5e352]*/ { - return _hashlib_HASH(module, Py_hash_sha3_256, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha3_256, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha3_384 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha3-384 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha3_384_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=f68e4846858cf0ee input=ec0edf5c792f8252]*/ +_hashlib_openssl_sha3_384_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=d239ba0463fd6138 input=fc943401f67e3b81]*/ { - return _hashlib_HASH(module, Py_hash_sha3_384, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha3_384, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha3_512 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha3-512 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha3_512_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=2eede478c159354a input=64e2cc0c094d56f4]*/ +_hashlib_openssl_sha3_512_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=17662f21038c2278 input=6601ddd2c6c1516d]*/ { - return _hashlib_HASH(module, Py_hash_sha3_512, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha3_512, data, string, usedforsecurity); } #endif /* PY_OPENSSL_HAS_SHA3 */ @@ -1270,42 +1289,46 @@ _hashlib_openssl_sha3_512_impl(PyObject *module, PyObject *data_obj, /*[clinic input] _hashlib.openssl_shake_128 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a shake-128 variable hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_shake_128_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=bc49cdd8ada1fa97 input=6c9d67440eb33ec8]*/ +_hashlib_openssl_shake_128_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=4e6afed8d18980ad input=373c3f1c93d87b37]*/ { - return _hashlib_HASH(module, Py_hash_shake_128, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_shake_128, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_shake_256 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a shake-256 variable hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_shake_256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=358d213be8852df7 input=479cbe9fefd4a9f8]*/ +_hashlib_openssl_shake_256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=62481bce4a77d16c input=101c139ea2ddfcbf]*/ { - return _hashlib_HASH(module, Py_hash_shake_256, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_shake_256, data, string, usedforsecurity); } #endif /* PY_OPENSSL_HAS_SHAKE */ +#undef CALL_HASHLIB_NEW + /*[clinic input] _hashlib.pbkdf2_hmac as pbkdf2_hmac diff --git a/Modules/blake2module.c b/Modules/blake2module.c index f9acc57f1b2fa3..07aa89f573f05f 100644 --- a/Modules/blake2module.c +++ b/Modules/blake2module.c @@ -655,8 +655,7 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, /*[clinic input] @classmethod _blake2.blake2b.__new__ as py_blake2b_new - data: object(c_default="NULL") = b'' - / + data as data_obj: object(c_default="NULL") = b'' * digest_size: int(c_default="HACL_HASH_BLAKE2B_OUT_BYTES") = _blake2.blake2b.MAX_DIGEST_SIZE key: Py_buffer(c_default="NULL", py_default="b''") = None @@ -670,26 +669,31 @@ _blake2.blake2b.__new__ as py_blake2b_new inner_size: int = 0 last_node: bool = False usedforsecurity: bool = True + string: object(c_default="NULL") = None Return a new BLAKE2b hash object. [clinic start generated code]*/ static PyObject * -py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, +py_blake2b_new_impl(PyTypeObject *type, PyObject *data_obj, int digest_size, Py_buffer *key, Py_buffer *salt, Py_buffer *person, int fanout, int depth, unsigned long leaf_size, unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity) -/*[clinic end generated code: output=32bfd8f043c6896f input=8fee2b7b11428b2d]*/ + int inner_size, int last_node, int usedforsecurity, + PyObject *string) +/*[clinic end generated code: output=de64bd850606b6a0 input=78cf60a2922d2f90]*/ { + PyObject *data; + if (_Py_hashlib_data_argument(&data, data_obj, string) < 0) { + return NULL; + } return py_blake2b_or_s_new(type, data, digest_size, key, salt, person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); } /*[clinic input] @classmethod _blake2.blake2s.__new__ as py_blake2s_new - data: object(c_default="NULL") = b'' - / + data as data_obj: object(c_default="NULL") = b'' * digest_size: int(c_default="HACL_HASH_BLAKE2S_OUT_BYTES") = _blake2.blake2s.MAX_DIGEST_SIZE key: Py_buffer(c_default="NULL", py_default="b''") = None @@ -703,18 +707,24 @@ _blake2.blake2s.__new__ as py_blake2s_new inner_size: int = 0 last_node: bool = False usedforsecurity: bool = True + string: object(c_default="NULL") = None Return a new BLAKE2s hash object. [clinic start generated code]*/ static PyObject * -py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, +py_blake2s_new_impl(PyTypeObject *type, PyObject *data_obj, int digest_size, Py_buffer *key, Py_buffer *salt, Py_buffer *person, int fanout, int depth, unsigned long leaf_size, unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity) -/*[clinic end generated code: output=556181f73905c686 input=8165a11980eac7f3]*/ + int inner_size, int last_node, int usedforsecurity, + PyObject *string) +/*[clinic end generated code: output=582a0c4295cc3a3c input=6843d6332eefd295]*/ { + PyObject *data; + if (_Py_hashlib_data_argument(&data, data_obj, string) < 0) { + return NULL; + } return py_blake2b_or_s_new(type, data, digest_size, key, salt, person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); } diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index b2f6b25a235e68..61ea10e2a48284 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -233,7 +233,7 @@ _hashlib_HASHXOF_hexdigest(PyObject *self, PyObject *const *args, Py_ssize_t nar #endif /* defined(PY_OPENSSL_HAS_SHAKE) */ PyDoc_STRVAR(_hashlib_HASH_new__doc__, -"new($module, /, name, string=b\'\', *, usedforsecurity=True)\n" +"new($module, /, name, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new hash object using the named algorithm.\n" @@ -247,8 +247,8 @@ PyDoc_STRVAR(_hashlib_HASH_new__doc__, {"new", _PyCFunction_CAST(_hashlib_HASH_new), METH_FASTCALL|METH_KEYWORDS, _hashlib_HASH_new__doc__}, static PyObject * -_hashlib_HASH_new_impl(PyObject *module, const char *name, - PyObject *data_obj, int usedforsecurity); +_hashlib_HASH_new_impl(PyObject *module, const char *name, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_HASH_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -256,7 +256,7 @@ _hashlib_HASH_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 4 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -265,7 +265,7 @@ _hashlib_HASH_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(name), &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(name), &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -274,18 +274,19 @@ _hashlib_HASH_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"name", "data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "new", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; const char *name; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -309,7 +310,7 @@ _hashlib_HASH_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO goto skip_optional_pos; } if (args[1]) { - data_obj = args[1]; + data = args[1]; if (!--noptargs) { goto skip_optional_pos; } @@ -318,19 +319,25 @@ _hashlib_HASH_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[2]); - if (usedforsecurity < 0) { - goto exit; + if (args[2]) { + usedforsecurity = PyObject_IsTrue(args[2]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[3]; skip_optional_kwonly: - return_value = _hashlib_HASH_new_impl(module, name, data_obj, usedforsecurity); + return_value = _hashlib_HASH_new_impl(module, name, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_md5__doc__, -"openssl_md5($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_md5($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Returns a md5 hash object; optionally initialized with a string"); @@ -339,8 +346,8 @@ PyDoc_STRVAR(_hashlib_openssl_md5__doc__, {"openssl_md5", _PyCFunction_CAST(_hashlib_openssl_md5), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_md5__doc__}, static PyObject * -_hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_md5_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -348,7 +355,7 @@ _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -357,7 +364,7 @@ _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -366,17 +373,18 @@ _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_md5", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -387,7 +395,7 @@ _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -396,19 +404,25 @@ _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_md5_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_md5_impl(module, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_sha1__doc__, -"openssl_sha1($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha1($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Returns a sha1 hash object; optionally initialized with a string"); @@ -417,8 +431,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha1__doc__, {"openssl_sha1", _PyCFunction_CAST(_hashlib_openssl_sha1), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha1__doc__}, static PyObject * -_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -426,7 +440,7 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -435,7 +449,7 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -444,17 +458,18 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha1", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -465,7 +480,7 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -474,19 +489,26 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha1_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha1_impl(module, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_sha224__doc__, -"openssl_sha224($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha224($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha224 hash object; optionally initialized with a string"); @@ -495,8 +517,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha224__doc__, {"openssl_sha224", _PyCFunction_CAST(_hashlib_openssl_sha224), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha224__doc__}, static PyObject * -_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -504,7 +526,7 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -513,7 +535,7 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -522,17 +544,18 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha224", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -543,7 +566,7 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -552,19 +575,26 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha224_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha224_impl(module, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_sha256__doc__, -"openssl_sha256($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha256($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha256 hash object; optionally initialized with a string"); @@ -573,8 +603,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha256__doc__, {"openssl_sha256", _PyCFunction_CAST(_hashlib_openssl_sha256), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha256__doc__}, static PyObject * -_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -582,7 +612,7 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -591,7 +621,7 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -600,17 +630,18 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha256", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -621,7 +652,7 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -630,19 +661,26 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha256_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha256_impl(module, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_sha384__doc__, -"openssl_sha384($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha384($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha384 hash object; optionally initialized with a string"); @@ -651,8 +689,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha384__doc__, {"openssl_sha384", _PyCFunction_CAST(_hashlib_openssl_sha384), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha384__doc__}, static PyObject * -_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -660,7 +698,7 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -669,7 +707,7 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -678,17 +716,18 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha384", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -699,7 +738,7 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -708,19 +747,26 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha384_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha384_impl(module, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_sha512__doc__, -"openssl_sha512($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha512($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha512 hash object; optionally initialized with a string"); @@ -729,8 +775,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha512__doc__, {"openssl_sha512", _PyCFunction_CAST(_hashlib_openssl_sha512), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha512__doc__}, static PyObject * -_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -738,7 +784,7 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -747,7 +793,7 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -756,17 +802,18 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha512", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -777,7 +824,7 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -786,12 +833,18 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha512_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha512_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -800,7 +853,8 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg #if defined(PY_OPENSSL_HAS_SHA3) PyDoc_STRVAR(_hashlib_openssl_sha3_224__doc__, -"openssl_sha3_224($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha3_224($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha3-224 hash object; optionally initialized with a string"); @@ -809,8 +863,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha3_224__doc__, {"openssl_sha3_224", _PyCFunction_CAST(_hashlib_openssl_sha3_224), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha3_224__doc__}, static PyObject * -_hashlib_openssl_sha3_224_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha3_224_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -818,7 +872,7 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -827,7 +881,7 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -836,17 +890,18 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha3_224", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -857,7 +912,7 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -866,12 +921,18 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha3_224_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha3_224_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -882,7 +943,8 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na #if defined(PY_OPENSSL_HAS_SHA3) PyDoc_STRVAR(_hashlib_openssl_sha3_256__doc__, -"openssl_sha3_256($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha3_256($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha3-256 hash object; optionally initialized with a string"); @@ -891,8 +953,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha3_256__doc__, {"openssl_sha3_256", _PyCFunction_CAST(_hashlib_openssl_sha3_256), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha3_256__doc__}, static PyObject * -_hashlib_openssl_sha3_256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha3_256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -900,7 +962,7 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -909,7 +971,7 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -918,17 +980,18 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha3_256", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -939,7 +1002,7 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -948,12 +1011,18 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha3_256_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha3_256_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -964,7 +1033,8 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na #if defined(PY_OPENSSL_HAS_SHA3) PyDoc_STRVAR(_hashlib_openssl_sha3_384__doc__, -"openssl_sha3_384($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha3_384($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha3-384 hash object; optionally initialized with a string"); @@ -973,8 +1043,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha3_384__doc__, {"openssl_sha3_384", _PyCFunction_CAST(_hashlib_openssl_sha3_384), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha3_384__doc__}, static PyObject * -_hashlib_openssl_sha3_384_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha3_384_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -982,7 +1052,7 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -991,7 +1061,7 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1000,17 +1070,18 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha3_384", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -1021,7 +1092,7 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -1030,12 +1101,18 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha3_384_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha3_384_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -1046,7 +1123,8 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na #if defined(PY_OPENSSL_HAS_SHA3) PyDoc_STRVAR(_hashlib_openssl_sha3_512__doc__, -"openssl_sha3_512($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha3_512($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha3-512 hash object; optionally initialized with a string"); @@ -1055,8 +1133,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha3_512__doc__, {"openssl_sha3_512", _PyCFunction_CAST(_hashlib_openssl_sha3_512), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha3_512__doc__}, static PyObject * -_hashlib_openssl_sha3_512_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha3_512_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1064,7 +1142,7 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -1073,7 +1151,7 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1082,17 +1160,18 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha3_512", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -1103,7 +1182,7 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -1112,12 +1191,18 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha3_512_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha3_512_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -1128,7 +1213,8 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na #if defined(PY_OPENSSL_HAS_SHAKE) PyDoc_STRVAR(_hashlib_openssl_shake_128__doc__, -"openssl_shake_128($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_shake_128($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a shake-128 variable hash object; optionally initialized with a string"); @@ -1137,8 +1223,8 @@ PyDoc_STRVAR(_hashlib_openssl_shake_128__doc__, {"openssl_shake_128", _PyCFunction_CAST(_hashlib_openssl_shake_128), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_shake_128__doc__}, static PyObject * -_hashlib_openssl_shake_128_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_shake_128_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1146,7 +1232,7 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -1155,7 +1241,7 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1164,17 +1250,18 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_shake_128", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -1185,7 +1272,7 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -1194,12 +1281,18 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_shake_128_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_shake_128_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -1210,7 +1303,8 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n #if defined(PY_OPENSSL_HAS_SHAKE) PyDoc_STRVAR(_hashlib_openssl_shake_256__doc__, -"openssl_shake_256($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_shake_256($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a shake-256 variable hash object; optionally initialized with a string"); @@ -1219,8 +1313,8 @@ PyDoc_STRVAR(_hashlib_openssl_shake_256__doc__, {"openssl_shake_256", _PyCFunction_CAST(_hashlib_openssl_shake_256), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_shake_256__doc__}, static PyObject * -_hashlib_openssl_shake_256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_shake_256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1228,7 +1322,7 @@ _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -1237,7 +1331,7 @@ _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t n } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1246,17 +1340,18 @@ _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t n # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_shake_256", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -1267,7 +1362,7 @@ _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t n goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -1276,12 +1371,18 @@ _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t n if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_shake_256_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_shake_256_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -1883,4 +1984,4 @@ _hashlib_compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef _HASHLIB_SCRYPT_METHODDEF #define _HASHLIB_SCRYPT_METHODDEF #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */ -/*[clinic end generated code: output=dc03b64435166a64 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=29f4aaf01714778e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/blake2module.c.h b/Modules/clinic/blake2module.c.h index bb2e308574a50a..9e9cd56e569b24 100644 --- a/Modules/clinic/blake2module.c.h +++ b/Modules/clinic/blake2module.c.h @@ -10,20 +10,21 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(py_blake2b_new__doc__, -"blake2b(data=b\'\', /, *, digest_size=_blake2.blake2b.MAX_DIGEST_SIZE,\n" +"blake2b(data=b\'\', *, digest_size=_blake2.blake2b.MAX_DIGEST_SIZE,\n" " key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n" " node_offset=0, node_depth=0, inner_size=0, last_node=False,\n" -" usedforsecurity=True)\n" +" usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new BLAKE2b hash object."); static PyObject * -py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, +py_blake2b_new_impl(PyTypeObject *type, PyObject *data_obj, int digest_size, Py_buffer *key, Py_buffer *salt, Py_buffer *person, int fanout, int depth, unsigned long leaf_size, unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity); + int inner_size, int last_node, int usedforsecurity, + PyObject *string); static PyObject * py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -31,7 +32,7 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 12 + #define NUM_KEYWORDS 14 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -40,7 +41,7 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(digest_size), &_Py_ID(key), &_Py_ID(salt), &_Py_ID(person), &_Py_ID(fanout), &_Py_ID(depth), &_Py_ID(leaf_size), &_Py_ID(node_offset), &_Py_ID(node_depth), &_Py_ID(inner_size), &_Py_ID(last_node), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(digest_size), &_Py_ID(key), &_Py_ID(salt), &_Py_ID(person), &_Py_ID(fanout), &_Py_ID(depth), &_Py_ID(leaf_size), &_Py_ID(node_offset), &_Py_ID(node_depth), &_Py_ID(inner_size), &_Py_ID(last_node), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -49,18 +50,18 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "blake2b", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[13]; + PyObject *argsbuf[14]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; - PyObject *data = NULL; + PyObject *data_obj = NULL; int digest_size = HACL_HASH_BLAKE2B_OUT_BYTES; Py_buffer key = {NULL, NULL}; Py_buffer salt = {NULL, NULL}; @@ -73,18 +74,23 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int inner_size = 0; int last_node = 0; int usedforsecurity = 1; + PyObject *string = NULL; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } - if (nargs < 1) { - goto skip_optional_posonly; + if (!noptargs) { + goto skip_optional_pos; } - noptargs--; - data = fastargs[0]; -skip_optional_posonly: + if (fastargs[0]) { + data_obj = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } @@ -182,12 +188,18 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - usedforsecurity = PyObject_IsTrue(fastargs[12]); - if (usedforsecurity < 0) { - goto exit; + if (fastargs[12]) { + usedforsecurity = PyObject_IsTrue(fastargs[12]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = fastargs[13]; skip_optional_kwonly: - return_value = py_blake2b_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); + return_value = py_blake2b_new_impl(type, data_obj, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity, string); exit: /* Cleanup for key */ @@ -207,20 +219,21 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(py_blake2s_new__doc__, -"blake2s(data=b\'\', /, *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n" +"blake2s(data=b\'\', *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n" " key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n" " node_offset=0, node_depth=0, inner_size=0, last_node=False,\n" -" usedforsecurity=True)\n" +" usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new BLAKE2s hash object."); static PyObject * -py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, +py_blake2s_new_impl(PyTypeObject *type, PyObject *data_obj, int digest_size, Py_buffer *key, Py_buffer *salt, Py_buffer *person, int fanout, int depth, unsigned long leaf_size, unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity); + int inner_size, int last_node, int usedforsecurity, + PyObject *string); static PyObject * py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -228,7 +241,7 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 12 + #define NUM_KEYWORDS 14 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -237,7 +250,7 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(digest_size), &_Py_ID(key), &_Py_ID(salt), &_Py_ID(person), &_Py_ID(fanout), &_Py_ID(depth), &_Py_ID(leaf_size), &_Py_ID(node_offset), &_Py_ID(node_depth), &_Py_ID(inner_size), &_Py_ID(last_node), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(digest_size), &_Py_ID(key), &_Py_ID(salt), &_Py_ID(person), &_Py_ID(fanout), &_Py_ID(depth), &_Py_ID(leaf_size), &_Py_ID(node_offset), &_Py_ID(node_depth), &_Py_ID(inner_size), &_Py_ID(last_node), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -246,18 +259,18 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "blake2s", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[13]; + PyObject *argsbuf[14]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; - PyObject *data = NULL; + PyObject *data_obj = NULL; int digest_size = HACL_HASH_BLAKE2S_OUT_BYTES; Py_buffer key = {NULL, NULL}; Py_buffer salt = {NULL, NULL}; @@ -270,18 +283,23 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int inner_size = 0; int last_node = 0; int usedforsecurity = 1; + PyObject *string = NULL; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } - if (nargs < 1) { - goto skip_optional_posonly; + if (!noptargs) { + goto skip_optional_pos; } - noptargs--; - data = fastargs[0]; -skip_optional_posonly: + if (fastargs[0]) { + data_obj = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } @@ -379,12 +397,18 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - usedforsecurity = PyObject_IsTrue(fastargs[12]); - if (usedforsecurity < 0) { - goto exit; + if (fastargs[12]) { + usedforsecurity = PyObject_IsTrue(fastargs[12]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = fastargs[13]; skip_optional_kwonly: - return_value = py_blake2s_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); + return_value = py_blake2s_new_impl(type, data_obj, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity, string); exit: /* Cleanup for key */ @@ -478,4 +502,4 @@ _blake2_blake2b_hexdigest(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2b_hexdigest_impl((Blake2Object *)self); } -/*[clinic end generated code: output=d30e8293bd8e2950 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=eed18dcfaf6f7731 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/md5module.c.h b/Modules/clinic/md5module.c.h index 9ca4f6528ce8f5..f76902586dddb2 100644 --- a/Modules/clinic/md5module.c.h +++ b/Modules/clinic/md5module.c.h @@ -89,7 +89,7 @@ MD5Type_update(PyObject *self, PyObject *obj) } PyDoc_STRVAR(_md5_md5__doc__, -"md5($module, /, string=b\'\', *, usedforsecurity=True)\n" +"md5($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new MD5 hash object; optionally initialized with a string."); @@ -98,7 +98,8 @@ PyDoc_STRVAR(_md5_md5__doc__, {"md5", _PyCFunction_CAST(_md5_md5), METH_FASTCALL|METH_KEYWORDS, _md5_md5__doc__}, static PyObject * -_md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity); +_md5_md5_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -106,7 +107,7 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -115,7 +116,7 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -124,17 +125,18 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "md5", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -145,7 +147,7 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -154,14 +156,20 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _md5_md5_impl(module, string, usedforsecurity); + return_value = _md5_md5_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } -/*[clinic end generated code: output=73f4d2034d9fcc63 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=920fe54b9ed06f92 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha1module.c.h b/Modules/clinic/sha1module.c.h index 3e5fd1a41ce21f..4a58d0cd9b82a4 100644 --- a/Modules/clinic/sha1module.c.h +++ b/Modules/clinic/sha1module.c.h @@ -89,7 +89,7 @@ SHA1Type_update(PyObject *self, PyObject *obj) } PyDoc_STRVAR(_sha1_sha1__doc__, -"sha1($module, /, string=b\'\', *, usedforsecurity=True)\n" +"sha1($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA1 hash object; optionally initialized with a string."); @@ -98,7 +98,8 @@ PyDoc_STRVAR(_sha1_sha1__doc__, {"sha1", _PyCFunction_CAST(_sha1_sha1), METH_FASTCALL|METH_KEYWORDS, _sha1_sha1__doc__}, static PyObject * -_sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity); +_sha1_sha1_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -106,7 +107,7 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -115,7 +116,7 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -124,17 +125,18 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha1", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -145,7 +147,7 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -154,14 +156,20 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _sha1_sha1_impl(module, string, usedforsecurity); + return_value = _sha1_sha1_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } -/*[clinic end generated code: output=06161e87e2d645d4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fd5a917404b68c4f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha2module.c.h b/Modules/clinic/sha2module.c.h index 26612125e75df9..07be91e4f6c68f 100644 --- a/Modules/clinic/sha2module.c.h +++ b/Modules/clinic/sha2module.c.h @@ -169,7 +169,7 @@ SHA512Type_update(PyObject *self, PyObject *obj) } PyDoc_STRVAR(_sha2_sha256__doc__, -"sha256($module, /, string=b\'\', *, usedforsecurity=True)\n" +"sha256($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA-256 hash object; optionally initialized with a string."); @@ -178,7 +178,8 @@ PyDoc_STRVAR(_sha2_sha256__doc__, {"sha256", _PyCFunction_CAST(_sha2_sha256), METH_FASTCALL|METH_KEYWORDS, _sha2_sha256__doc__}, static PyObject * -_sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity); +_sha2_sha256_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -186,7 +187,7 @@ _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -195,7 +196,7 @@ _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -204,17 +205,18 @@ _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha256", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -225,7 +227,7 @@ _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -234,19 +236,25 @@ _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _sha2_sha256_impl(module, string, usedforsecurity); + return_value = _sha2_sha256_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } PyDoc_STRVAR(_sha2_sha224__doc__, -"sha224($module, /, string=b\'\', *, usedforsecurity=True)\n" +"sha224($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA-224 hash object; optionally initialized with a string."); @@ -255,7 +263,8 @@ PyDoc_STRVAR(_sha2_sha224__doc__, {"sha224", _PyCFunction_CAST(_sha2_sha224), METH_FASTCALL|METH_KEYWORDS, _sha2_sha224__doc__}, static PyObject * -_sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity); +_sha2_sha224_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -263,7 +272,7 @@ _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -272,7 +281,7 @@ _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -281,17 +290,18 @@ _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha224", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -302,7 +312,7 @@ _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -311,19 +321,25 @@ _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _sha2_sha224_impl(module, string, usedforsecurity); + return_value = _sha2_sha224_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } PyDoc_STRVAR(_sha2_sha512__doc__, -"sha512($module, /, string=b\'\', *, usedforsecurity=True)\n" +"sha512($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA-512 hash object; optionally initialized with a string."); @@ -332,7 +348,8 @@ PyDoc_STRVAR(_sha2_sha512__doc__, {"sha512", _PyCFunction_CAST(_sha2_sha512), METH_FASTCALL|METH_KEYWORDS, _sha2_sha512__doc__}, static PyObject * -_sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity); +_sha2_sha512_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -340,7 +357,7 @@ _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -349,7 +366,7 @@ _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -358,17 +375,18 @@ _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha512", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -379,7 +397,7 @@ _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -388,19 +406,25 @@ _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _sha2_sha512_impl(module, string, usedforsecurity); + return_value = _sha2_sha512_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } PyDoc_STRVAR(_sha2_sha384__doc__, -"sha384($module, /, string=b\'\', *, usedforsecurity=True)\n" +"sha384($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA-384 hash object; optionally initialized with a string."); @@ -409,7 +433,8 @@ PyDoc_STRVAR(_sha2_sha384__doc__, {"sha384", _PyCFunction_CAST(_sha2_sha384), METH_FASTCALL|METH_KEYWORDS, _sha2_sha384__doc__}, static PyObject * -_sha2_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity); +_sha2_sha384_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -417,7 +442,7 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -426,7 +451,7 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -435,17 +460,18 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha384", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -456,7 +482,7 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -465,14 +491,20 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _sha2_sha384_impl(module, string, usedforsecurity); + return_value = _sha2_sha384_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } -/*[clinic end generated code: output=af11090855b7c85a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=90625b237c774a9f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha3module.c.h b/Modules/clinic/sha3module.c.h index 25f72b74f801db..121be2c0758695 100644 --- a/Modules/clinic/sha3module.c.h +++ b/Modules/clinic/sha3module.c.h @@ -10,13 +10,14 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(py_sha3_new__doc__, -"sha3_224(data=b\'\', /, *, usedforsecurity=True)\n" +"sha3_224(data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA3 hash object."); static PyObject * -py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity); +py_sha3_new_impl(PyTypeObject *type, PyObject *data_obj, int usedforsecurity, + PyObject *string); static PyObject * py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -24,7 +25,7 @@ py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -33,7 +34,7 @@ py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -42,40 +43,51 @@ py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha3_224", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; - PyObject *data = NULL; + PyObject *data_obj = NULL; int usedforsecurity = 1; + PyObject *string = NULL; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } - if (nargs < 1) { - goto skip_optional_posonly; + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + data_obj = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } } - noptargs--; - data = fastargs[0]; -skip_optional_posonly: +skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(fastargs[1]); - if (usedforsecurity < 0) { - goto exit; + if (fastargs[1]) { + usedforsecurity = PyObject_IsTrue(fastargs[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = fastargs[2]; skip_optional_kwonly: - return_value = py_sha3_new_impl(type, data, usedforsecurity); + return_value = py_sha3_new_impl(type, data_obj, usedforsecurity, string); exit: return return_value; @@ -158,24 +170,57 @@ _sha3_sha3_224_update(PyObject *self, PyObject *data) } PyDoc_STRVAR(_sha3_shake_128_digest__doc__, -"digest($self, length, /)\n" +"digest($self, /, length)\n" "--\n" "\n" "Return the digest value as a bytes object."); #define _SHA3_SHAKE_128_DIGEST_METHODDEF \ - {"digest", (PyCFunction)_sha3_shake_128_digest, METH_O, _sha3_shake_128_digest__doc__}, + {"digest", _PyCFunction_CAST(_sha3_shake_128_digest), METH_FASTCALL|METH_KEYWORDS, _sha3_shake_128_digest__doc__}, static PyObject * _sha3_shake_128_digest_impl(SHA3object *self, unsigned long length); static PyObject * -_sha3_shake_128_digest(PyObject *self, PyObject *arg) +_sha3_shake_128_digest(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(length), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"length", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "digest", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; unsigned long length; - if (!_PyLong_UnsignedLong_Converter(arg, &length)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!_PyLong_UnsignedLong_Converter(args[0], &length)) { goto exit; } return_value = _sha3_shake_128_digest_impl((SHA3object *)self, length); @@ -185,24 +230,57 @@ _sha3_shake_128_digest(PyObject *self, PyObject *arg) } PyDoc_STRVAR(_sha3_shake_128_hexdigest__doc__, -"hexdigest($self, length, /)\n" +"hexdigest($self, /, length)\n" "--\n" "\n" "Return the digest value as a string of hexadecimal digits."); #define _SHA3_SHAKE_128_HEXDIGEST_METHODDEF \ - {"hexdigest", (PyCFunction)_sha3_shake_128_hexdigest, METH_O, _sha3_shake_128_hexdigest__doc__}, + {"hexdigest", _PyCFunction_CAST(_sha3_shake_128_hexdigest), METH_FASTCALL|METH_KEYWORDS, _sha3_shake_128_hexdigest__doc__}, static PyObject * _sha3_shake_128_hexdigest_impl(SHA3object *self, unsigned long length); static PyObject * -_sha3_shake_128_hexdigest(PyObject *self, PyObject *arg) +_sha3_shake_128_hexdigest(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(length), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"length", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "hexdigest", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; unsigned long length; - if (!_PyLong_UnsignedLong_Converter(arg, &length)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!_PyLong_UnsignedLong_Converter(args[0], &length)) { goto exit; } return_value = _sha3_shake_128_hexdigest_impl((SHA3object *)self, length); @@ -210,4 +288,4 @@ _sha3_shake_128_hexdigest(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=5b3ac1c06c6899ea input=a9049054013a1b77]*/ +/*[clinic end generated code: output=65e437799472b89f input=a9049054013a1b77]*/ diff --git a/Modules/hashlib.h b/Modules/hashlib.h index 7105e68af7b806..a80b195a765792 100644 --- a/Modules/hashlib.h +++ b/Modules/hashlib.h @@ -76,3 +76,32 @@ * to allow the user to optimize based on the platform they're using. */ #define HASHLIB_GIL_MINSIZE 2048 +static inline int +_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string) +{ + if (data != NULL && string == NULL) { + // called as H(data) or H(data=...) + *res = data; + return 1; + } + else if (data == NULL && string != NULL) { + // called as H(string=...) + *res = string; + return 1; + } + else if (data == NULL && string == NULL) { + // fast path when no data is given + assert(!PyErr_Occurred()); + *res = NULL; + return 0; + } + else { + // called as H(data=..., string) + *res = NULL; + PyErr_SetString(PyExc_TypeError, + "'data' and 'string' are mutually exclusive " + "and support for 'string' keyword parameter " + "is slated for removal in a future version."); + return -1; + } +} diff --git a/Modules/md5module.c b/Modules/md5module.c index c36eb41d4d201e..9b5ea2d6e02605 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -276,17 +276,24 @@ static PyType_Spec md5_type_spec = { /*[clinic input] _md5.md5 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new MD5 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=587071f76254a4ac input=7a144a1905636985]*/ +_md5_md5_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=d45e187d3d16f3a8 input=7ea5c5366dbb44bf]*/ { + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } + MD5object *new; Py_buffer buf; diff --git a/Modules/sha1module.c b/Modules/sha1module.c index f4a00cdb422156..a746bf74f8d4c1 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -272,19 +272,25 @@ static PyType_Spec sha1_type_spec = { /*[clinic input] _sha1.sha1 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new SHA1 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=6f8b3af05126e18e input=bd54b68e2bf36a8a]*/ +_sha1_sha1_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=0d453775924f88a7 input=807f25264e0ac656]*/ { SHA1object *new; Py_buffer buf; + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } if (string) { GET_BUFFER_VIEW_OR_ERROUT(string, &buf); diff --git a/Modules/sha2module.c b/Modules/sha2module.c index e88d7cb2d456bf..72931910c5d720 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -594,18 +594,24 @@ static PyType_Spec sha512_type_spec = { /*[clinic input] _sha2.sha256 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new SHA-256 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=243c9dd289931f87 input=6249da1de607280a]*/ +_sha2_sha256_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=49828a7bcd418f45 input=9ce1d70e669abc14]*/ { Py_buffer buf; + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } if (string) { GET_BUFFER_VIEW_OR_ERROUT(string, &buf); @@ -651,18 +657,25 @@ _sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity) /*[clinic input] _sha2.sha224 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new SHA-224 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=68191f232e4a3843 input=c42bcba47fd7d2b7]*/ +_sha2_sha224_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=2163cb03b6cf6157 input=612f7682a889bc2a]*/ { Py_buffer buf; + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } + if (string) { GET_BUFFER_VIEW_OR_ERROUT(string, &buf); } @@ -706,19 +719,25 @@ _sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity) /*[clinic input] _sha2.sha512 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new SHA-512 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=d55c8996eca214d7 input=0576ae2a6ebfad25]*/ +_sha2_sha512_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=cc3fcfce001a4538 input=19c9f2c06d59563a]*/ { SHA512object *new; Py_buffer buf; + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } sha2_state *state = sha2_get_state(module); @@ -763,19 +782,25 @@ _sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) /*[clinic input] _sha2.sha384 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new SHA-384 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_sha2_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=b29a0d81d51d1368 input=4e9199d8de0d2f9b]*/ +_sha2_sha384_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=b6e3db593b5a0330 input=9fd50c942ad9e0bf]*/ { SHA512object *new; Py_buffer buf; + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } sha2_state *state = sha2_get_state(module); diff --git a/Modules/sha3module.c b/Modules/sha3module.c index a7edf5c66a1e76..cfbf0cbcc042c5 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -105,18 +105,25 @@ sha3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *buf, Py_ssize_t len) /*[clinic input] @classmethod _sha3.sha3_224.__new__ as py_sha3_new - data: object(c_default="NULL") = b'' - / + + data as data_obj: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Return a new SHA3 hash object. [clinic start generated code]*/ static PyObject * -py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) -/*[clinic end generated code: output=90409addc5d5e8b0 input=637e5f8f6a93982a]*/ +py_sha3_new_impl(PyTypeObject *type, PyObject *data_obj, int usedforsecurity, + PyObject *string) +/*[clinic end generated code: output=dcec1eca20395f2a input=c106e0b4e2d67d58]*/ { + PyObject *data; + if (_Py_hashlib_data_argument(&data, data_obj, string) < 0) { + return NULL; + } + Py_buffer buf = {NULL, NULL}; SHA3State *state = _PyType_GetModuleState(type); SHA3object *self = newSHA3object(type); @@ -503,14 +510,13 @@ _SHAKE_digest(PyObject *op, unsigned long digestlen, int hex) _sha3.shake_128.digest length: unsigned_long - / Return the digest value as a bytes object. [clinic start generated code]*/ static PyObject * _sha3_shake_128_digest_impl(SHA3object *self, unsigned long length) -/*[clinic end generated code: output=2313605e2f87bb8f input=418ef6a36d2e6082]*/ +/*[clinic end generated code: output=2313605e2f87bb8f input=93d6d6ff32904f18]*/ { return _SHAKE_digest((PyObject *)self, length, 0); } @@ -520,14 +526,13 @@ _sha3_shake_128_digest_impl(SHA3object *self, unsigned long length) _sha3.shake_128.hexdigest length: unsigned_long - / Return the digest value as a string of hexadecimal digits. [clinic start generated code]*/ static PyObject * _sha3_shake_128_hexdigest_impl(SHA3object *self, unsigned long length) -/*[clinic end generated code: output=bf8e2f1e490944a8 input=69fb29b0926ae321]*/ +/*[clinic end generated code: output=bf8e2f1e490944a8 input=562d74e7060b56ab]*/ { return _SHAKE_digest((PyObject *)self, length, 1); } From 68784fed78aa297f0de0d038742495709185bef5 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 31 May 2025 11:23:01 +0300 Subject: [PATCH 033/116] gh-133489: Remove size restrictions on getrandbits() and randbytes() (GH-133658) random.getrandbits() can now generate more that 2**31 bits. random.randbytes() can now generate more that 256 MiB. --- Lib/test/test_random.py | 28 +++++++++++++++++++ ...-05-08-13-43-19.gh-issue-133489.9eGS1Z.rst | 2 ++ Modules/_randommodule.c | 20 ++++++------- Modules/clinic/_randommodule.c.h | 10 +++---- Objects/longobject.c | 13 ++------- 5 files changed, 47 insertions(+), 26 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-08-13-43-19.gh-issue-133489.9eGS1Z.rst diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index bd76d636e4f0fc..54910cd8054a1f 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -392,6 +392,8 @@ def test_getrandbits(self): self.assertRaises(TypeError, self.gen.getrandbits) self.assertRaises(TypeError, self.gen.getrandbits, 1, 2) self.assertRaises(ValueError, self.gen.getrandbits, -1) + self.assertRaises(OverflowError, self.gen.getrandbits, 1<<1000) + self.assertRaises(ValueError, self.gen.getrandbits, -1<<1000) self.assertRaises(TypeError, self.gen.getrandbits, 10.1) def test_pickling(self): @@ -435,6 +437,8 @@ def test_randbytes(self): self.assertRaises(TypeError, self.gen.randbytes) self.assertRaises(TypeError, self.gen.randbytes, 1, 2) self.assertRaises(ValueError, self.gen.randbytes, -1) + self.assertRaises(OverflowError, self.gen.randbytes, 1<<1000) + self.assertRaises((ValueError, OverflowError), self.gen.randbytes, -1<<1000) self.assertRaises(TypeError, self.gen.randbytes, 1.0) def test_mu_sigma_default_args(self): @@ -806,6 +810,22 @@ def test_getrandbits(self): self.assertEqual(self.gen.getrandbits(100), 97904845777343510404718956115) + def test_getrandbits_2G_bits(self): + size = 2**31 + self.gen.seed(1234567) + x = self.gen.getrandbits(size) + self.assertEqual(x.bit_length(), size) + self.assertEqual(x & (2**100-1), 890186470919986886340158459475) + self.assertEqual(x >> (size-100), 1226514312032729439655761284440) + + @support.bigmemtest(size=2**32, memuse=1/8+2/15, dry_run=False) + def test_getrandbits_4G_bits(self, size): + self.gen.seed(1234568) + x = self.gen.getrandbits(size) + self.assertEqual(x.bit_length(), size) + self.assertEqual(x & (2**100-1), 287241425661104632871036099814) + self.assertEqual(x >> (size-100), 739728759900339699429794460738) + def test_randrange_uses_getrandbits(self): # Verify use of getrandbits by randrange # Use same seed as in the cross-platform repeatability test @@ -962,6 +982,14 @@ def test_randbytes_getrandbits(self): self.assertEqual(self.gen.randbytes(n), gen2.getrandbits(n * 8).to_bytes(n, 'little')) + @support.bigmemtest(size=2**29, memuse=1+16/15, dry_run=False) + def test_randbytes_256M(self, size): + self.gen.seed(2849427419) + x = self.gen.randbytes(size) + self.assertEqual(len(x), size) + self.assertEqual(x[:12].hex(), 'f6fd9ae63855ab91ea238b4f') + self.assertEqual(x[-12:].hex(), '0e7af69a84ee99bf4a11becc') + def test_sample_counts_equivalence(self): # Test the documented strong equivalence to a sample with repeated elements. # We run this test on random.Random() which makes deterministic selections diff --git a/Misc/NEWS.d/next/Library/2025-05-08-13-43-19.gh-issue-133489.9eGS1Z.rst b/Misc/NEWS.d/next/Library/2025-05-08-13-43-19.gh-issue-133489.9eGS1Z.rst new file mode 100644 index 00000000000000..0c07beb76938f0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-08-13-43-19.gh-issue-133489.9eGS1Z.rst @@ -0,0 +1,2 @@ +:func:`random.getrandbits` can now generate more that 2\ :sup:`31` bits. +:func:`random.randbytes` can now generate more that 256 MiB. diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index d5bac2f5b78120..2f4f388ce1161a 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -497,34 +497,32 @@ _random_Random_setstate_impl(RandomObject *self, PyObject *state) _random.Random.getrandbits self: self(type="RandomObject *") - k: int + k: uint64 / getrandbits(k) -> x. Generates an int with k random bits. [clinic start generated code]*/ static PyObject * -_random_Random_getrandbits_impl(RandomObject *self, int k) -/*[clinic end generated code: output=b402f82a2158887f input=87603cd60f79f730]*/ +_random_Random_getrandbits_impl(RandomObject *self, uint64_t k) +/*[clinic end generated code: output=c30ef8435f3433cf input=64226ac13bb4d2a3]*/ { - int i, words; + Py_ssize_t i, words; uint32_t r; uint32_t *wordarray; PyObject *result; - if (k < 0) { - PyErr_SetString(PyExc_ValueError, - "number of bits must be non-negative"); - return NULL; - } - if (k == 0) return PyLong_FromLong(0); if (k <= 32) /* Fast path */ return PyLong_FromUnsignedLong(genrand_uint32(self) >> (32 - k)); - words = (k - 1) / 32 + 1; + if ((k - 1u) / 32u + 1u > PY_SSIZE_T_MAX / 4u) { + PyErr_NoMemory(); + return NULL; + } + words = (k - 1u) / 32u + 1u; wordarray = (uint32_t *)PyMem_Malloc(words * 4); if (wordarray == NULL) { PyErr_NoMemory(); diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h index 1e989e970c9de5..2563a16aea0b6f 100644 --- a/Modules/clinic/_randommodule.c.h +++ b/Modules/clinic/_randommodule.c.h @@ -3,6 +3,7 @@ preserve [clinic start generated code]*/ #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_long.h" // _PyLong_UInt64_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_random_Random_random__doc__, @@ -124,16 +125,15 @@ PyDoc_STRVAR(_random_Random_getrandbits__doc__, {"getrandbits", (PyCFunction)_random_Random_getrandbits, METH_O, _random_Random_getrandbits__doc__}, static PyObject * -_random_Random_getrandbits_impl(RandomObject *self, int k); +_random_Random_getrandbits_impl(RandomObject *self, uint64_t k); static PyObject * _random_Random_getrandbits(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; - int k; + uint64_t k; - k = PyLong_AsInt(arg); - if (k == -1 && PyErr_Occurred()) { + if (!_PyLong_UInt64_Converter(arg, &k)) { goto exit; } Py_BEGIN_CRITICAL_SECTION(self); @@ -143,4 +143,4 @@ _random_Random_getrandbits(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=4458b5a69201ebea input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7ce97b2194eecaf7 input=a9049054013a1b77]*/ diff --git a/Objects/longobject.c b/Objects/longobject.c index 0b2dfa003fac53..2b533312fee673 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -971,16 +971,9 @@ _PyLong_FromByteArray(const unsigned char* bytes, size_t n, ++numsignificantbytes; } - /* How many Python int digits do we need? We have - 8*numsignificantbytes bits, and each Python int digit has - PyLong_SHIFT bits, so it's the ceiling of the quotient. */ - /* catch overflow before it happens */ - if (numsignificantbytes > (PY_SSIZE_T_MAX - PyLong_SHIFT) / 8) { - PyErr_SetString(PyExc_OverflowError, - "byte array too long to convert to int"); - return NULL; - } - ndigits = (numsignificantbytes * 8 + PyLong_SHIFT - 1) / PyLong_SHIFT; + /* avoid integer overflow */ + ndigits = numsignificantbytes / PyLong_SHIFT * 8 + + (numsignificantbytes % PyLong_SHIFT * 8 + PyLong_SHIFT - 1) / PyLong_SHIFT; v = long_alloc(ndigits); if (v == NULL) return NULL; From 3c66e5976669a599adfb260514c03815b1a9e4e9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 31 May 2025 12:03:08 +0300 Subject: [PATCH 034/116] gh-134918: Fix and improve doctest's documentation (GH-134919) --- Doc/library/doctest.rst | 95 ++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index b86fef9fd6f310..8236d703fc1e45 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -174,7 +174,7 @@ with assorted summaries at the end. You can force verbose mode by passing ``verbose=True`` to :func:`testmod`, or prohibit it by passing ``verbose=False``. In either of those cases, -``sys.argv`` is not examined by :func:`testmod` (so passing ``-v`` or not +:data:`sys.argv` is not examined by :func:`testmod` (so passing ``-v`` or not has no effect). There is also a command line shortcut for running :func:`testmod`, see section @@ -231,7 +231,7 @@ documentation:: As with :func:`testmod`, :func:`testfile` won't display anything unless an example fails. If an example does fail, then the failing example(s) and the cause(s) of the failure(s) are printed to stdout, using the same format as -:func:`testmod`. +:func:`!testmod`. By default, :func:`testfile` looks for files in the calling module's directory. See section :ref:`doctest-basic-api` for a description of the optional arguments @@ -311,6 +311,9 @@ Which Docstrings Are Examined? The module docstring, and all function, class and method docstrings are searched. Objects imported into the module are not searched. +.. attribute:: module.__test__ + :no-typesetting: + In addition, there are cases when you want tests to be part of a module but not part of the help text, which requires that the tests not be included in the docstring. Doctest looks for a module-level variable called ``__test__`` and uses it to locate other @@ -533,7 +536,7 @@ Some details you should read once, but won't need to remember: * The interactive shell omits the traceback header line for some :exc:`SyntaxError`\ s. But doctest uses the traceback header line to distinguish exceptions from non-exceptions. So in the rare case where you need - to test a :exc:`SyntaxError` that omits the traceback header, you will need to + to test a :exc:`!SyntaxError` that omits the traceback header, you will need to manually add the traceback header line to your test example. .. index:: single: ^ (caret); marker @@ -860,15 +863,15 @@ The :const:`ELLIPSIS` directive gives a nice approach for the last example: Floating-point numbers are also subject to small output variations across -platforms, because Python defers to the platform C library for float formatting, -and C libraries vary widely in quality here. :: +platforms, because Python defers to the platform C library for some +floating-point calculations, and C libraries vary widely in quality here. :: - >>> 1./7 # risky - 0.14285714285714285 - >>> print(1./7) # safer - 0.142857142857 - >>> print(round(1./7, 6)) # much safer - 0.142857 + >>> 1000**0.1 # risky + 1.9952623149688797 + >>> round(1000**0.1, 9) # safer + 1.995262315 + >>> print(f'{1000**0.1:.4f}') # much safer + 1.9953 Numbers of the form ``I/2.**J`` are safe across all platforms, and I often contrive doctest examples to produce numbers of that form:: @@ -938,13 +941,13 @@ and :ref:`doctest-simple-testfile`. Optional argument *verbose* prints lots of stuff if true, and prints only failures if false; by default, or if ``None``, it's true if and only if ``'-v'`` - is in ``sys.argv``. + is in :data:`sys.argv`. Optional argument *report* prints a summary at the end when true, else prints nothing at the end. In verbose mode, the summary is detailed, else the summary is very brief (in fact, empty if all tests passed). - Optional argument *optionflags* (default value 0) takes the + Optional argument *optionflags* (default value ``0``) takes the :ref:`bitwise OR ` of option flags. See section :ref:`doctest-options`. @@ -1045,7 +1048,7 @@ from text files and modules with doctests: The returned :class:`unittest.TestSuite` is to be run by the unittest framework and runs the interactive examples in each file. If an example in any file - fails, then the synthesized unit test fails, and a :exc:`failureException` + fails, then the synthesized unit test fails, and a :exc:`~unittest.TestCase.failureException` exception is raised showing the name of the file containing the test and a (sometimes approximate) line number. If all the examples in a file are skipped, then the synthesized unit test is also marked as skipped. @@ -1078,13 +1081,14 @@ from text files and modules with doctests: Optional argument *setUp* specifies a set-up function for the test suite. This is called before running the tests in each file. The *setUp* function - will be passed a :class:`DocTest` object. The setUp function can access the - test globals as the *globs* attribute of the test passed. + will be passed a :class:`DocTest` object. The *setUp* function can access the + test globals as the :attr:`~DocTest.globs` attribute of the test passed. Optional argument *tearDown* specifies a tear-down function for the test suite. This is called after running the tests in each file. The *tearDown* - function will be passed a :class:`DocTest` object. The setUp function can - access the test globals as the *globs* attribute of the test passed. + function will be passed a :class:`DocTest` object. The *tearDown* function can + access the test globals as the :attr:`~DocTest.globs` attribute of the test + passed. Optional argument *globs* is a dictionary containing the initial global variables for the tests. A new copy of this dictionary is created for each @@ -1111,11 +1115,12 @@ from text files and modules with doctests: Convert doctest tests for a module to a :class:`unittest.TestSuite`. The returned :class:`unittest.TestSuite` is to be run by the unittest framework - and runs each doctest in the module. If any of the doctests fail, then the - synthesized unit test fails, and a :exc:`failureException` exception is raised + and runs each doctest in the module. + Each docstring is run as a separate unit test. + If any of the doctests fail, then the synthesized unit test fails, + and a :exc:`unittest.TestCase.failureException` exception is raised showing the name of the file containing the test and a (sometimes approximate) line number. If all the examples in a docstring are skipped, then the - synthesized unit test is also marked as skipped. Optional argument *module* provides the module to be tested. It can be a module object or a (possibly dotted) module name. If not specified, the module calling @@ -1123,7 +1128,7 @@ from text files and modules with doctests: Optional argument *globs* is a dictionary containing the initial global variables for the tests. A new copy of this dictionary is created for each - test. By default, *globs* is a new empty dictionary. + test. By default, *globs* is the module's :attr:`~module.__dict__`. Optional argument *extraglobs* specifies an extra set of global variables, which is merged into *globs*. By default, no extra globals are used. @@ -1132,7 +1137,7 @@ from text files and modules with doctests: drop-in replacement) that is used to extract doctests from the module. Optional arguments *setUp*, *tearDown*, and *optionflags* are the same as for - function :func:`DocFileSuite` above. + function :func:`DocFileSuite` above, but they are called for each docstring. This function uses the same search technique as :func:`testmod`. @@ -1140,12 +1145,6 @@ from text files and modules with doctests: :func:`DocTestSuite` returns an empty :class:`unittest.TestSuite` if *module* contains no docstrings instead of raising :exc:`ValueError`. -.. exception:: failureException - - When doctests which have been converted to unit tests by :func:`DocFileSuite` - or :func:`DocTestSuite` fail, this exception is raised showing the name of - the file containing the test and a (sometimes approximate) line number. - Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out of :class:`!doctest.DocTestCase` instances, and :class:`!DocTestCase` is a subclass of :class:`unittest.TestCase`. :class:`!DocTestCase` isn't documented @@ -1158,15 +1157,15 @@ of :class:`!DocTestCase`. So both ways of creating a :class:`unittest.TestSuite` run instances of :class:`!DocTestCase`. This is important for a subtle reason: when you run -:mod:`doctest` functions yourself, you can control the :mod:`doctest` options in -use directly, by passing option flags to :mod:`doctest` functions. However, if -you're writing a :mod:`unittest` framework, :mod:`unittest` ultimately controls +:mod:`doctest` functions yourself, you can control the :mod:`!doctest` options in +use directly, by passing option flags to :mod:`!doctest` functions. However, if +you're writing a :mod:`unittest` framework, :mod:`!unittest` ultimately controls when and how tests get run. The framework author typically wants to control -:mod:`doctest` reporting options (perhaps, e.g., specified by command line -options), but there's no way to pass options through :mod:`unittest` to -:mod:`doctest` test runners. +:mod:`!doctest` reporting options (perhaps, e.g., specified by command line +options), but there's no way to pass options through :mod:`!unittest` to +:mod:`!doctest` test runners. -For this reason, :mod:`doctest` also supports a notion of :mod:`doctest` +For this reason, :mod:`doctest` also supports a notion of :mod:`!doctest` reporting flags specific to :mod:`unittest` support, via this function: @@ -1181,12 +1180,12 @@ reporting flags specific to :mod:`unittest` support, via this function: :mod:`unittest`: the :meth:`!runTest` method of :class:`!DocTestCase` looks at the option flags specified for the test case when the :class:`!DocTestCase` instance was constructed. If no reporting flags were specified (which is the - typical and expected case), :mod:`!doctest`'s :mod:`unittest` reporting flags are + typical and expected case), :mod:`!doctest`'s :mod:`!unittest` reporting flags are :ref:`bitwise ORed ` into the option flags, and the option flags so augmented are passed to the :class:`DocTestRunner` instance created to run the doctest. If any reporting flags were specified when the :class:`!DocTestCase` instance was constructed, :mod:`!doctest`'s - :mod:`unittest` reporting flags are ignored. + :mod:`!unittest` reporting flags are ignored. The value of the :mod:`unittest` reporting flags in effect before the function was called is returned by the function. @@ -1279,7 +1278,7 @@ DocTest Objects .. attribute:: filename The name of the file that this :class:`DocTest` was extracted from; or - ``None`` if the filename is unknown, or if the :class:`DocTest` was not + ``None`` if the filename is unknown, or if the :class:`!DocTest` was not extracted from a file. @@ -1419,10 +1418,10 @@ DocTestFinder objects The globals for each :class:`DocTest` is formed by combining *globs* and *extraglobs* (bindings in *extraglobs* override bindings in *globs*). A new - shallow copy of the globals dictionary is created for each :class:`DocTest`. - If *globs* is not specified, then it defaults to the module's *__dict__*, if - specified, or ``{}`` otherwise. If *extraglobs* is not specified, then it - defaults to ``{}``. + shallow copy of the globals dictionary is created for each :class:`!DocTest`. + If *globs* is not specified, then it defaults to the module's + :attr:`~module.__dict__`, if specified, or ``{}`` otherwise. + If *extraglobs* is not specified, then it defaults to ``{}``. .. _doctest-doctestparser: @@ -1446,7 +1445,7 @@ DocTestParser objects :class:`DocTest` object. *globs*, *name*, *filename*, and *lineno* are attributes for the new - :class:`DocTest` object. See the documentation for :class:`DocTest` for more + :class:`!DocTest` object. See the documentation for :class:`DocTest` for more information. @@ -1461,7 +1460,7 @@ DocTestParser objects Divide the given string into examples and intervening text, and return them as a list of alternating :class:`Example`\ s and strings. Line numbers for the - :class:`Example`\ s are 0-based. The optional argument *name* is a name + :class:`!Example`\ s are 0-based. The optional argument *name* is a name identifying this string, and is only used for error messages. @@ -1501,7 +1500,7 @@ DocTestRunner objects :class:`OutputChecker`. This comparison may be customized with a number of option flags; see section :ref:`doctest-options` for more information. If the option flags are insufficient, then the comparison may also be customized by - passing a subclass of :class:`OutputChecker` to the constructor. + passing a subclass of :class:`!OutputChecker` to the constructor. The test runner's display output can be controlled in two ways. First, an output function can be passed to :meth:`run`; this function will be called @@ -1540,7 +1539,7 @@ DocTestRunner objects output; it should not be called directly. *example* is the example about to be processed. *test* is the test - *containing example*. *out* is the output function that was passed to + containing *example*. *out* is the output function that was passed to :meth:`DocTestRunner.run`. @@ -1940,7 +1939,7 @@ several options for organizing tests: containing test cases for the named topics. These functions can be included in the same file as the module, or separated out into a separate test file. -* Define a ``__test__`` dictionary mapping from regression test topics to +* Define a :attr:`~module.__test__` dictionary mapping from regression test topics to docstrings containing test cases. When you have placed your tests in a module, the module can itself be the test From ad39f017881e0bd8ffd809755ebf76380b928ad3 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 31 May 2025 13:01:46 +0300 Subject: [PATCH 035/116] gh-108885: Use subtests for doctest examples run by unittest (GH-134890) Run each example as a subtest in unit tests synthesized by doctest.DocFileSuite() and doctest.DocTestSuite(). Add the doctest.DocTestRunner.report_skip() method. --- Doc/library/doctest.rst | 52 ++- Lib/doctest.py | 95 +++-- Lib/test/test_doctest/test_doctest.py | 338 +++++++++--------- Lib/test/test_regrtest.py | 2 +- ...-05-29-17-39-13.gh-issue-108885.MegCRA.rst | 3 + 5 files changed, 279 insertions(+), 211 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-29-17-39-13.gh-issue-108885.MegCRA.rst diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 8236d703fc1e45..fb43cf918b84dd 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1046,12 +1046,15 @@ from text files and modules with doctests: Convert doctest tests from one or more text files to a :class:`unittest.TestSuite`. - The returned :class:`unittest.TestSuite` is to be run by the unittest framework - and runs the interactive examples in each file. If an example in any file - fails, then the synthesized unit test fails, and a :exc:`~unittest.TestCase.failureException` - exception is raised showing the name of the file containing the test and a - (sometimes approximate) line number. If all the examples in a file are - skipped, then the synthesized unit test is also marked as skipped. + The returned :class:`unittest.TestSuite` is to be run by the unittest + framework and runs the interactive examples in each file. + Each file is run as a separate unit test, and each example in a file + is run as a :ref:`subtest `. + If any example in a file fails, then the synthesized unit test fails. + The traceback for failure or error contains the name of the file + containing the test and a (sometimes approximate) line number. + If all the examples in a file are skipped, then the synthesized unit + test is also marked as skipped. Pass one or more paths (as strings) to text files to be examined. @@ -1109,18 +1112,23 @@ from text files and modules with doctests: The global ``__file__`` is added to the globals provided to doctests loaded from a text file using :func:`DocFileSuite`. + .. versionchanged:: next + Run each example as a :ref:`subtest `. + .. function:: DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, setUp=None, tearDown=None, optionflags=0, checker=None) Convert doctest tests for a module to a :class:`unittest.TestSuite`. - The returned :class:`unittest.TestSuite` is to be run by the unittest framework - and runs each doctest in the module. - Each docstring is run as a separate unit test. - If any of the doctests fail, then the synthesized unit test fails, - and a :exc:`unittest.TestCase.failureException` exception is raised - showing the name of the file containing the test and a (sometimes approximate) - line number. If all the examples in a docstring are skipped, then the + The returned :class:`unittest.TestSuite` is to be run by the unittest + framework and runs each doctest in the module. + Each docstring is run as a separate unit test, and each example in + a docstring is run as a :ref:`subtest `. + If any of the doctests fail, then the synthesized unit test fails. + The traceback for failure or error contains the name of the file + containing the test and a (sometimes approximate) line number. + If all the examples in a docstring are skipped, then the + synthesized unit test is also marked as skipped. Optional argument *module* provides the module to be tested. It can be a module object or a (possibly dotted) module name. If not specified, the module calling @@ -1145,6 +1153,9 @@ from text files and modules with doctests: :func:`DocTestSuite` returns an empty :class:`unittest.TestSuite` if *module* contains no docstrings instead of raising :exc:`ValueError`. + .. versionchanged:: next + Run each example as a :ref:`subtest `. + Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out of :class:`!doctest.DocTestCase` instances, and :class:`!DocTestCase` is a subclass of :class:`unittest.TestCase`. :class:`!DocTestCase` isn't documented @@ -1507,7 +1518,7 @@ DocTestRunner objects with strings that should be displayed. It defaults to ``sys.stdout.write``. If capturing the output is not sufficient, then the display output can be also customized by subclassing DocTestRunner, and overriding the methods - :meth:`report_start`, :meth:`report_success`, + :meth:`report_skip`, :meth:`report_start`, :meth:`report_success`, :meth:`report_unexpected_exception`, and :meth:`report_failure`. The optional keyword argument *checker* specifies the :class:`OutputChecker` @@ -1532,6 +1543,19 @@ DocTestRunner objects :class:`DocTestRunner` defines the following methods: + .. method:: report_skip(out, test, example) + + Report that the given example was skipped. This method is provided to + allow subclasses of :class:`DocTestRunner` to customize their output; it + should not be called directly. + + *example* is the example about to be processed. *test* is the test + containing *example*. *out* is the output function that was passed to + :meth:`DocTestRunner.run`. + + .. versionadded:: next + + .. method:: report_start(out, test, example) Report that the test runner is about to process the given example. This method diff --git a/Lib/doctest.py b/Lib/doctest.py index dec10a345165da..c8c95ecbb273b2 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -101,6 +101,7 @@ def _test(): import re import sys import traceback +import types import unittest from io import StringIO, IncrementalNewlineDecoder from collections import namedtuple @@ -108,8 +109,6 @@ def _test(): from _colorize import ANSIColors, can_colorize -__unittest = True - class TestResults(namedtuple('TestResults', 'failed attempted')): def __new__(cls, failed, attempted, *, skipped=0): results = super().__new__(cls, failed, attempted) @@ -387,7 +386,7 @@ def __init__(self, out): self.__out = out self.__debugger_used = False # do not play signal games in the pdb - pdb.Pdb.__init__(self, stdout=out, nosigint=True) + super().__init__(stdout=out, nosigint=True) # still use input() to get user input self.use_rawinput = 1 @@ -1280,6 +1279,11 @@ def __init__(self, checker=None, verbose=None, optionflags=0): # Reporting methods #///////////////////////////////////////////////////////////////// + def report_skip(self, out, test, example): + """ + Report that the given example was skipped. + """ + def report_start(self, out, test, example): """ Report that the test runner is about to process the given @@ -1377,6 +1381,8 @@ def __run(self, test, compileflags, out): # If 'SKIP' is set, then skip this example. if self.optionflags & SKIP: + if not quiet: + self.report_skip(out, test, example) skips += 1 continue @@ -2274,12 +2280,63 @@ def set_unittest_reportflags(flags): return old +class _DocTestCaseRunner(DocTestRunner): + + def __init__(self, *args, test_case, test_result, **kwargs): + super().__init__(*args, **kwargs) + self._test_case = test_case + self._test_result = test_result + self._examplenum = 0 + + def _subTest(self): + subtest = unittest.case._SubTest(self._test_case, str(self._examplenum), {}) + self._examplenum += 1 + return subtest + + def report_skip(self, out, test, example): + unittest.case._addSkip(self._test_result, self._subTest(), '') + + def report_success(self, out, test, example, got): + self._test_result.addSubTest(self._test_case, self._subTest(), None) + + def report_unexpected_exception(self, out, test, example, exc_info): + tb = self._add_traceback(exc_info[2], test, example) + exc_info = (*exc_info[:2], tb) + self._test_result.addSubTest(self._test_case, self._subTest(), exc_info) + + def report_failure(self, out, test, example, got): + msg = ('Failed example:\n' + _indent(example.source) + + self._checker.output_difference(example, got, self.optionflags).rstrip('\n')) + exc = self._test_case.failureException(msg) + tb = self._add_traceback(None, test, example) + exc_info = (type(exc), exc, tb) + self._test_result.addSubTest(self._test_case, self._subTest(), exc_info) + + def _add_traceback(self, traceback, test, example): + if test.lineno is None or example.lineno is None: + lineno = None + else: + lineno = test.lineno + example.lineno + 1 + return types.SimpleNamespace( + tb_frame = types.SimpleNamespace( + f_globals=test.globs, + f_code=types.SimpleNamespace( + co_filename=test.filename, + co_name=test.name, + ), + ), + tb_next = traceback, + tb_lasti = -1, + tb_lineno = lineno, + ) + + class DocTestCase(unittest.TestCase): def __init__(self, test, optionflags=0, setUp=None, tearDown=None, checker=None): - unittest.TestCase.__init__(self) + super().__init__() self._dt_optionflags = optionflags self._dt_checker = checker self._dt_test = test @@ -2303,30 +2360,28 @@ def tearDown(self): test.globs.clear() test.globs.update(self._dt_globs) + def run(self, result=None): + self._test_result = result + return super().run(result) + def runTest(self): test = self._dt_test - old = sys.stdout - new = StringIO() optionflags = self._dt_optionflags + result = self._test_result if not (optionflags & REPORTING_FLAGS): # The option flags don't include any reporting flags, # so add the default reporting flags optionflags |= _unittest_reportflags + if getattr(result, 'failfast', False): + optionflags |= FAIL_FAST - runner = DocTestRunner(optionflags=optionflags, - checker=self._dt_checker, verbose=False) - - try: - runner.DIVIDER = "-"*70 - results = runner.run(test, out=new.write, clear_globs=False) - if results.skipped == results.attempted: - raise unittest.SkipTest("all examples were skipped") - finally: - sys.stdout = old - - if results.failed: - raise self.failureException(self.format_failure(new.getvalue().rstrip('\n'))) + runner = _DocTestCaseRunner(optionflags=optionflags, + checker=self._dt_checker, verbose=False, + test_case=self, test_result=result) + results = runner.run(test, clear_globs=False) + if results.skipped == results.attempted: + raise unittest.SkipTest("all examples were skipped") def format_failure(self, err): test = self._dt_test @@ -2441,7 +2496,7 @@ def shortDescription(self): class SkipDocTestCase(DocTestCase): def __init__(self, module): self.module = module - DocTestCase.__init__(self, None) + super().__init__(None) def setUp(self): self.skipTest("DocTestSuite will not work with -O2 and above") diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 2bfaa6c599cd47..72763d4a0132d0 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -2269,20 +2269,22 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite(test.test_doctest.sample_doctest) >>> result = suite.run(unittest.TestResult()) >>> result - + >>> for tst, _ in result.failures: ... print(tst) - bad (test.test_doctest.sample_doctest.__test__) - foo (test.test_doctest.sample_doctest) - test_silly_setup (test.test_doctest.sample_doctest) - y_is_one (test.test_doctest.sample_doctest) + bad (test.test_doctest.sample_doctest.__test__) [0] + foo (test.test_doctest.sample_doctest) [0] + >>> for tst, _ in result.errors: + ... print(tst) + test_silly_setup (test.test_doctest.sample_doctest) [1] + y_is_one (test.test_doctest.sample_doctest) [0] We can also supply the module by name: >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest') >>> result = suite.run(unittest.TestResult()) >>> result - + The module need not contain any doctest examples: @@ -2304,21 +2306,26 @@ def test_DocTestSuite(): >>> result >>> len(result.skipped) - 2 + 7 >>> for tst, _ in result.skipped: ... print(tst) + double_skip (test.test_doctest.sample_doctest_skip) [0] + double_skip (test.test_doctest.sample_doctest_skip) [1] double_skip (test.test_doctest.sample_doctest_skip) + partial_skip_fail (test.test_doctest.sample_doctest_skip) [0] + partial_skip_pass (test.test_doctest.sample_doctest_skip) [0] + single_skip (test.test_doctest.sample_doctest_skip) [0] single_skip (test.test_doctest.sample_doctest_skip) >>> for tst, _ in result.failures: ... print(tst) - no_skip_fail (test.test_doctest.sample_doctest_skip) - partial_skip_fail (test.test_doctest.sample_doctest_skip) + no_skip_fail (test.test_doctest.sample_doctest_skip) [0] + partial_skip_fail (test.test_doctest.sample_doctest_skip) [1] We can use the current module: >>> suite = test.test_doctest.sample_doctest.test_suite() >>> suite.run(unittest.TestResult()) - + We can also provide a DocTestFinder: @@ -2326,7 +2333,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... test_finder=finder) >>> suite.run(unittest.TestResult()) - + The DocTestFinder need not return any tests: @@ -2342,7 +2349,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', globs={}) >>> suite.run(unittest.TestResult()) - + Alternatively, we can provide extra globals. Here we'll make an error go away by providing an extra global variable: @@ -2350,7 +2357,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... extraglobs={'y': 1}) >>> suite.run(unittest.TestResult()) - + You can pass option flags. Here we'll cause an extra error by disabling the blank-line feature: @@ -2358,7 +2365,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... optionflags=doctest.DONT_ACCEPT_BLANKLINE) >>> suite.run(unittest.TestResult()) - + You can supply setUp and tearDown functions: @@ -2375,7 +2382,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... setUp=setUp, tearDown=tearDown) >>> suite.run(unittest.TestResult()) - + But the tearDown restores sanity: @@ -2393,7 +2400,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', setUp=setUp) >>> suite.run(unittest.TestResult()) - + Here, we didn't need to use a tearDown function because we modified the test globals, which are a copy of the @@ -2409,115 +2416,97 @@ def test_DocTestSuite_errors(): >>> suite = doctest.DocTestSuite(mod) >>> result = suite.run(unittest.TestResult()) >>> result - + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors - File "...sample_doctest_errors.py", line 0, in sample_doctest_errors - - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line 5, in test.test_doctest.sample_doctest_errors - Failed example: + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 5, in test.test_doctest.sample_doctest_errors + >...>> 2 + 2 + AssertionError: Failed example: 2 + 2 Expected: 5 Got: 4 - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line 7, in test.test_doctest.sample_doctest_errors - Failed example: - 1/0 - Exception raised: - Traceback (most recent call last): - File "", line 1, in - 1/0 - ~^~ - ZeroDivisionError: division by zero >>> print(result.failures[1][1]) # doctest: +ELLIPSIS - AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.__test__.bad - File "...sample_doctest_errors.py", line unknown line number, in bad - - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad - Failed example: + Traceback (most recent call last): + File "...sample_doctest_errors.py", line None, in test.test_doctest.sample_doctest_errors.__test__.bad + AssertionError: Failed example: 2 + 2 Expected: 5 Got: 4 - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad - Failed example: - 1/0 - Exception raised: - Traceback (most recent call last): - File "", line 1, in - 1/0 - ~^~ - ZeroDivisionError: division by zero >>> print(result.failures[2][1]) # doctest: +ELLIPSIS - AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.errors - File "...sample_doctest_errors.py", line 14, in errors - - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line 16, in test.test_doctest.sample_doctest_errors.errors - Failed example: + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 16, in test.test_doctest.sample_doctest_errors.errors + >...>> 2 + 2 + AssertionError: Failed example: 2 + 2 Expected: 5 Got: 4 - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line 18, in test.test_doctest.sample_doctest_errors.errors - Failed example: + + >>> print(result.errors[0][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 7, in test.test_doctest.sample_doctest_errors + >...>> 1/0 + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + >>> print(result.errors[1][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line None, in test.test_doctest.sample_doctest_errors.__test__.bad + File "", line 1, in 1/0 - Exception raised: - Traceback (most recent call last): - File "", line 1, in - 1/0 - ~^~ - ZeroDivisionError: division by zero - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line 23, in test.test_doctest.sample_doctest_errors.errors - Failed example: + ~^~ + ZeroDivisionError: division by zero + + >>> print(result.errors[2][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 18, in test.test_doctest.sample_doctest_errors.errors + >...>> 1/0 + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + >>> print(result.errors[3][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 23, in test.test_doctest.sample_doctest_errors.errors + >...>> f() + File "", line 1, in f() - Exception raised: - Traceback (most recent call last): - File "", line 1, in - f() - ~^^ - File "", line 2, in f - 2 + '2' - ~~^~~~~ - TypeError: ... - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line 25, in test.test_doctest.sample_doctest_errors.errors - Failed example: - g() - Exception raised: - Traceback (most recent call last): - File "", line 1, in - g() - ~^^ - File "...sample_doctest_errors.py", line 12, in g - [][0] # line 12 - ~~^^^ - IndexError: list index out of range + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... - >>> print(result.failures[3][1]) # doctest: +ELLIPSIS - AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.syntax_error - File "...sample_doctest_errors.py", line 29, in syntax_error + >>> print(result.errors[4][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 25, in test.test_doctest.sample_doctest_errors.errors + >...>> g() + File "", line 1, in + g() + ~^^ + File "...sample_doctest_errors.py", line 12, in g + [][0] # line 12 + ~~^^^ + IndexError: list index out of range - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line 31, in test.test_doctest.sample_doctest_errors.syntax_error - Failed example: + >>> print(result.errors[5][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 31, in test.test_doctest.sample_doctest_errors.syntax_error + >...>> 2+*3 + File "", line 1 2+*3 - Exception raised: - File "", line 1 - 2+*3 - ^ - SyntaxError: invalid syntax + ^ + SyntaxError: invalid syntax """ @@ -2532,7 +2521,7 @@ def test_DocFileSuite(): ... 'test_doctest2.txt', ... 'test_doctest4.txt') >>> suite.run(unittest.TestResult()) - + The test files are looked for in the directory containing the calling module. A package keyword argument can be provided to @@ -2544,14 +2533,14 @@ def test_DocFileSuite(): ... 'test_doctest4.txt', ... package='test.test_doctest') >>> suite.run(unittest.TestResult()) - + '/' should be used as a path separator. It will be converted to a native separator at run time: >>> suite = doctest.DocFileSuite('../test_doctest/test_doctest.txt') >>> suite.run(unittest.TestResult()) - + If DocFileSuite is used from an interactive session, then files are resolved relative to the directory of sys.argv[0]: @@ -2577,7 +2566,7 @@ def test_DocFileSuite(): >>> suite = doctest.DocFileSuite(test_file, module_relative=False) >>> suite.run(unittest.TestResult()) - + It is an error to specify `package` when `module_relative=False`: @@ -2595,12 +2584,15 @@ def test_DocFileSuite(): ... 'test_doctest_skip2.txt') >>> result = suite.run(unittest.TestResult()) >>> result - + >>> len(result.skipped) - 1 + 4 >>> for tst, _ in result.skipped: # doctest: +ELLIPSIS ... print('=', tst) + = ...test_doctest_skip.txt [0] + = ...test_doctest_skip.txt [1] = ...test_doctest_skip.txt + = ...test_doctest_skip2.txt [0] You can specify initial global variables: @@ -2609,7 +2601,7 @@ def test_DocFileSuite(): ... 'test_doctest4.txt', ... globs={'favorite_color': 'blue'}) >>> suite.run(unittest.TestResult()) - + In this case, we supplied a missing favorite color. You can provide doctest options: @@ -2620,7 +2612,7 @@ def test_DocFileSuite(): ... optionflags=doctest.DONT_ACCEPT_BLANKLINE, ... globs={'favorite_color': 'blue'}) >>> suite.run(unittest.TestResult()) - + And, you can provide setUp and tearDown functions: @@ -2639,7 +2631,7 @@ def test_DocFileSuite(): ... 'test_doctest4.txt', ... setUp=setUp, tearDown=tearDown) >>> suite.run(unittest.TestResult()) - + But the tearDown restores sanity: @@ -2681,7 +2673,7 @@ def test_DocFileSuite(): ... 'test_doctest4.txt', ... encoding='utf-8') >>> suite.run(unittest.TestResult()) - + """ def test_DocFileSuite_errors(): @@ -2691,52 +2683,49 @@ def test_DocFileSuite_errors(): >>> suite = doctest.DocFileSuite('test_doctest_errors.txt') >>> result = suite.run(unittest.TestResult()) >>> result - + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - AssertionError: Failed doctest test for test_doctest_errors.txt - File "...test_doctest_errors.txt", line 0 - - ---------------------------------------------------------------------- - File "...test_doctest_errors.txt", line 4, in test_doctest_errors.txt - Failed example: + Traceback (most recent call last): + File "...test_doctest_errors.txt", line 4, in test_doctest_errors.txt + >...>> 2 + 2 + AssertionError: Failed example: 2 + 2 Expected: 5 Got: 4 - ---------------------------------------------------------------------- - File "...test_doctest_errors.txt", line 6, in test_doctest_errors.txt - Failed example: + + >>> print(result.errors[0][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...test_doctest_errors.txt", line 6, in test_doctest_errors.txt + >...>> 1/0 + File "", line 1, in 1/0 - Exception raised: - Traceback (most recent call last): - File "", line 1, in - 1/0 - ~^~ - ZeroDivisionError: division by zero - ---------------------------------------------------------------------- - File "...test_doctest_errors.txt", line 11, in test_doctest_errors.txt - Failed example: + ~^~ + ZeroDivisionError: division by zero + + >>> print(result.errors[1][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...test_doctest_errors.txt", line 11, in test_doctest_errors.txt + >...>> f() + File "", line 1, in f() - Exception raised: - Traceback (most recent call last): - File "", line 1, in - f() - ~^^ - File "", line 2, in f - 2 + '2' - ~~^~~~~ - TypeError: ... - ---------------------------------------------------------------------- - File "...test_doctest_errors.txt", line 13, in test_doctest_errors.txt - Failed example: + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + + >>> print(result.errors[2][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...test_doctest_errors.txt", line 13, in test_doctest_errors.txt + >...>> 2+*3 + File "", line 1 2+*3 - Exception raised: - File "", line 1 - 2+*3 - ^ - SyntaxError: invalid syntax + ^ + SyntaxError: invalid syntax + """ def test_trailing_space_in_test(): @@ -2807,16 +2796,25 @@ def test_unittest_reportflags(): >>> import unittest >>> result = suite.run(unittest.TestResult()) >>> result - + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - AssertionError: Failed doctest test for test_doctest.txt - ... - Failed example: - favorite_color - ... - Failed example: + Traceback (most recent call last): + File ... + >...>> if 1: + AssertionError: Failed example: if 1: - ... + print('a') + print() + print('b') + Expected: + a + + b + Got: + a + + b + Note that we see both failures displayed. @@ -2825,18 +2823,8 @@ def test_unittest_reportflags(): Now, when we run the test: - >>> result = suite.run(unittest.TestResult()) - >>> result - - >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - AssertionError: Failed doctest test for test_doctest.txt - ... - Failed example: - favorite_color - Exception raised: - ... - NameError: name 'favorite_color' is not defined - + >>> suite.run(unittest.TestResult()) + We get only the first failure. @@ -2846,22 +2834,20 @@ def test_unittest_reportflags(): >>> suite = doctest.DocFileSuite('test_doctest.txt', ... optionflags=doctest.DONT_ACCEPT_BLANKLINE | doctest.REPORT_NDIFF) - Then the default eporting options are ignored: + Then the default reporting options are ignored: >>> result = suite.run(unittest.TestResult()) >>> result - + *NOTE*: These doctest are intentionally not placed in raw string to depict the trailing whitespace using `\x20` in the diff below. >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - AssertionError: Failed doctest test for test_doctest.txt - ... - Failed example: - favorite_color - ... - Failed example: + Traceback ... + File ... + >...>> if 1: + AssertionError: Failed example: if 1: print('a') print() @@ -3669,9 +3655,9 @@ def test_run_doctestsuite_multiple_times(): >>> import test.test_doctest.sample_doctest >>> suite = doctest.DocTestSuite(test.test_doctest.sample_doctest) >>> suite.run(unittest.TestResult()) - + >>> suite.run(unittest.TestResult()) - + """ diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 8f4fc09442e083..f3ac301686b9fc 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -2067,7 +2067,7 @@ def load_tests(loader, tests, pattern): self.check_executed_tests(output, [testname], failed=[testname], parallel=True, - stats=TestStats(1, 1, 0)) + stats=TestStats(1, 2, 1)) def _check_random_seed(self, run_workers: bool): # gh-109276: When -r/--randomize is used, random.seed() is called diff --git a/Misc/NEWS.d/next/Library/2025-05-29-17-39-13.gh-issue-108885.MegCRA.rst b/Misc/NEWS.d/next/Library/2025-05-29-17-39-13.gh-issue-108885.MegCRA.rst new file mode 100644 index 00000000000000..e37cf121f5f529 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-29-17-39-13.gh-issue-108885.MegCRA.rst @@ -0,0 +1,3 @@ +Run each example as a subtest in unit tests synthesized by +:func:`doctest.DocFileSuite` and :func:`doctest.DocTestSuite`. +Add the :meth:`doctest.DocTestRunner.report_skip` method. From 379d0bc95646dfe923e7ea05fb7f1befbd85572d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 31 May 2025 12:48:34 +0200 Subject: [PATCH 036/116] gh-134696: fix `hashlib` tests for FIPS-only BLAKE-2 buildbot (#134968) --- Lib/test/test_hashlib.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 51b82fe3b516b5..8244f7c7553a37 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -249,6 +249,7 @@ def test_usedforsecurity_false(self): self._hashlib.new("md5", usedforsecurity=False) self._hashlib.openssl_md5(usedforsecurity=False) + @unittest.skipIf(get_fips_mode(), "skip in FIPS mode") def test_clinic_signature(self): for constructor in self.hash_constructors: with self.subTest(constructor.__name__): @@ -256,6 +257,17 @@ def test_clinic_signature(self): constructor(data=b'') constructor(string=b'') # should be deprecated in the future + digest_name = constructor(b'').name + with self.subTest(digest_name): + hashlib.new(digest_name, b'') + hashlib.new(digest_name, data=b'') + hashlib.new(digest_name, string=b'') + if self._hashlib: + self._hashlib.new(digest_name, b'') + self._hashlib.new(digest_name, data=b'') + self._hashlib.new(digest_name, string=b'') + + @unittest.skipIf(get_fips_mode(), "skip in FIPS mode") def test_clinic_signature_errors(self): nomsg = b'' mymsg = b'msg' @@ -295,9 +307,16 @@ def test_clinic_signature_errors(self): [(), dict(data=nomsg, string=nomsg), conflicting_call], ]: for constructor in self.hash_constructors: + digest_name = constructor(b'').name with self.subTest(constructor.__name__, args=args, kwds=kwds): with self.assertRaisesRegex(TypeError, errmsg): constructor(*args, **kwds) + with self.subTest(digest_name, args=args, kwds=kwds): + with self.assertRaisesRegex(TypeError, errmsg): + hashlib.new(digest_name, *args, **kwds) + if self._hashlib: + with self.assertRaisesRegex(TypeError, errmsg): + self._hashlib.new(digest_name, *args, **kwds) def test_unknown_hash(self): self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam') From c81446af1dbf3c84bfd4ed604c245dd40463fd3a Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Sat, 31 May 2025 13:35:51 +0200 Subject: [PATCH 037/116] gh-133968: Create the Unicode writer on demand in json (#133832) --- Modules/_json.c | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/Modules/_json.c b/Modules/_json.c index 4aa6ae650651b3..57678ad595f928 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -360,13 +360,6 @@ _build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) { return tpl; } -static inline int -_PyUnicodeWriter_IsEmpty(PyUnicodeWriter *writer_pub) -{ - _PyUnicodeWriter *writer = (_PyUnicodeWriter*)writer_pub; - return (writer->pos == 0); -} - static PyObject * scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr) { @@ -385,10 +378,7 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next const void *buf; int kind; - PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); - if (writer == NULL) { - goto bail; - } + PyUnicodeWriter *writer = NULL; len = PyUnicode_GET_LENGTH(pystr); buf = PyUnicode_DATA(pystr); @@ -419,12 +409,11 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next if (c == '"') { // Fast path for simple case. - if (_PyUnicodeWriter_IsEmpty(writer)) { + if (writer == NULL) { PyObject *ret = PyUnicode_Substring(pystr, end, next); if (ret == NULL) { goto bail; } - PyUnicodeWriter_Discard(writer); *next_end_ptr = next + 1;; return ret; } @@ -432,6 +421,11 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next else if (c != '\\') { raise_errmsg("Unterminated string starting at", pystr, begin); goto bail; + } else if (writer == NULL) { + writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + goto bail; + } } /* Pick up this chunk if it's not zero length */ From af0d3268d9ae6090877c276c12ee6712b56578e7 Mon Sep 17 00:00:00 2001 From: CF Bolz-Tereick Date: Sat, 31 May 2025 13:38:05 +0200 Subject: [PATCH 038/116] Skip test as cpython_only that checks whether setattr interns the attribute or not (#134972) Skip test that checks whether setattr interns the attribute or not The details of when a string is being interned or not is implementation dependent. --- Lib/test/test_class.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 4c12d43556fc2a..8c7a62a74ba90e 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -652,6 +652,7 @@ class B(A): a = A(hash(A.f)^(-1)) hash(a.f) + @cpython_only def testSetattrWrapperNameIntern(self): # Issue #25794: __setattr__ should intern the attribute name class A: From 895119ec24589cbf522e375aa71f27b9b7383a8b Mon Sep 17 00:00:00 2001 From: CF Bolz-Tereick Date: Sat, 31 May 2025 13:46:22 +0200 Subject: [PATCH 039/116] skip test for sys._stdlib_dir if that is not present (#134973) --- Lib/test/test_sys.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 65d15610ed1505..83745f3d0ba46e 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1299,6 +1299,7 @@ def test_module_names(self): for name in sys.stdlib_module_names: self.assertIsInstance(name, str) + @unittest.skipUnless(hasattr(sys, '_stdlib_dir'), 'need sys._stdlib_dir') def test_stdlib_dir(self): os = import_helper.import_fresh_module('os') marker = getattr(os, '__file__', None) From 5507eff19c757a908a2ff29dfe423e35595fda00 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sat, 31 May 2025 14:56:33 +0300 Subject: [PATCH 040/116] Improve format of `InternalDocs/exception_handling.md` (#134969) --- InternalDocs/exception_handling.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/InternalDocs/exception_handling.md b/InternalDocs/exception_handling.md index 28589787e1fad7..9e38da4c862f16 100644 --- a/InternalDocs/exception_handling.md +++ b/InternalDocs/exception_handling.md @@ -8,7 +8,7 @@ The cost of raising an exception is increased, but not by much. The following code: -``` +```python try: g(0) except: @@ -18,7 +18,7 @@ except: compiles into intermediate code like the following: -``` +```python RESUME 0 1 SETUP_FINALLY 8 (to L1) @@ -118,13 +118,13 @@ All offsets and lengths are in code units, not bytes. We want the format to be compact, but quickly searchable. For it to be compact, it needs to have variable sized entries so that we can store common (small) offsets compactly, but handle large offsets if needed. -For it to be searchable quickly, we need to support binary search giving us log(n) performance in all cases. +For it to be searchable quickly, we need to support binary search giving us `log(n)` performance in all cases. Binary search typically assumes fixed size entries, but that is not necessary, as long as we can identify the start of an entry. It is worth noting that the size (end-start) is always smaller than the end, so we encode the entries as: `start, size, target, depth, push-lasti`. -Also, sizes are limited to 2**30 as the code length cannot exceed 2**31 and each code unit takes 2 bytes. +Also, sizes are limited to `2**30` as the code length cannot exceed `2**31` and each code unit takes 2 bytes. It also happens that depth is generally quite small. So, we need to encode: @@ -140,7 +140,7 @@ lasti (1 bit) We need a marker for the start of the entry, so the first byte of entry will have the most significant bit set. Since the most significant bit is reserved for marking the start of an entry, we have 7 bits per byte to encode offsets. Encoding uses a standard varint encoding, but with only 7 bits instead of the usual 8. -The 8 bits of a byte are (msb left) SXdddddd where S is the start bit. X is the extend bit meaning that the next byte is required to extend the offset. +The 8 bits of a byte are (msb left) `SXdddddd` where `S` is the start bit. `X` is the extend bit meaning that the next byte is required to extend the offset. In addition, we combine `depth` and `lasti` into a single value, `((depth<<1)+lasti)`, before encoding. From f58873e4b2b7aad8e3a08a6188c6eb08d0a3001b Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Sat, 31 May 2025 07:29:03 -0700 Subject: [PATCH 041/116] gh-134954: Hard-cap max file descriptors in subprocess test fd_status (#134955) * Hard-cap max file descriptors in subprocess test fd_status On some systems, `SC_OPEN_MAX` may return a very large value (i.e. 10**30), leading to the subprocess test timing out (or run forever). Prevent this situation by applying a hard cap on how many file descriptors are checked. * Fix typo in usage docstring s/fd_stats/fd_status/ --- Lib/test/subprocessdata/fd_status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/subprocessdata/fd_status.py b/Lib/test/subprocessdata/fd_status.py index d12bd95abee61c..90e785981aeab0 100644 --- a/Lib/test/subprocessdata/fd_status.py +++ b/Lib/test/subprocessdata/fd_status.py @@ -2,7 +2,7 @@ file descriptors on stdout. Usage: -fd_stats.py: check all file descriptors +fd_status.py: check all file descriptors (up to 255) fd_status.py fd1 fd2 ...: check only specified file descriptors """ @@ -18,7 +18,7 @@ _MAXFD = os.sysconf("SC_OPEN_MAX") except: _MAXFD = 256 - test_fds = range(0, _MAXFD) + test_fds = range(0, min(_MAXFD, 256)) else: test_fds = map(int, sys.argv[1:]) for fd in test_fds: From 3704171415c1ea6ebbeb2f992758b6565f42e378 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sat, 31 May 2025 18:48:51 +0100 Subject: [PATCH 042/116] gh-134835: Remove outdated list from `howto/urllib2.rst` (GH-134844) :teapot: --- Doc/howto/urllib2.rst | 86 ++++++++++--------------------------------- 1 file changed, 19 insertions(+), 67 deletions(-) diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 33a2a7ea89ea07..d79d1abe8d0577 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -245,75 +245,27 @@ codes in the 100--299 range indicate success, you will usually only see error codes in the 400--599 range. :attr:`http.server.BaseHTTPRequestHandler.responses` is a useful dictionary of -response codes in that shows all the response codes used by :rfc:`2616`. The -dictionary is reproduced here for convenience :: +response codes that shows all the response codes used by :rfc:`2616`. +An excerpt from the dictionary is shown below :: - # Table mapping response codes to messages; entries have the - # form {code: (shortmessage, longmessage)}. responses = { - 100: ('Continue', 'Request received, please continue'), - 101: ('Switching Protocols', - 'Switching to new protocol; obey Upgrade header'), - - 200: ('OK', 'Request fulfilled, document follows'), - 201: ('Created', 'Document created, URL follows'), - 202: ('Accepted', - 'Request accepted, processing continues off-line'), - 203: ('Non-Authoritative Information', 'Request fulfilled from cache'), - 204: ('No Content', 'Request fulfilled, nothing follows'), - 205: ('Reset Content', 'Clear input form for further input.'), - 206: ('Partial Content', 'Partial content follows.'), - - 300: ('Multiple Choices', - 'Object has several resources -- see URI list'), - 301: ('Moved Permanently', 'Object moved permanently -- see URI list'), - 302: ('Found', 'Object moved temporarily -- see URI list'), - 303: ('See Other', 'Object moved -- see Method and URL list'), - 304: ('Not Modified', - 'Document has not changed since given time'), - 305: ('Use Proxy', - 'You must use proxy specified in Location to access this ' - 'resource.'), - 307: ('Temporary Redirect', - 'Object moved temporarily -- see URI list'), - - 400: ('Bad Request', - 'Bad request syntax or unsupported method'), - 401: ('Unauthorized', - 'No permission -- see authorization schemes'), - 402: ('Payment Required', - 'No payment -- see charging schemes'), - 403: ('Forbidden', - 'Request forbidden -- authorization will not help'), - 404: ('Not Found', 'Nothing matches the given URI'), - 405: ('Method Not Allowed', - 'Specified method is invalid for this server.'), - 406: ('Not Acceptable', 'URI not available in preferred format.'), - 407: ('Proxy Authentication Required', 'You must authenticate with ' - 'this proxy before proceeding.'), - 408: ('Request Timeout', 'Request timed out; try again later.'), - 409: ('Conflict', 'Request conflict.'), - 410: ('Gone', - 'URI no longer exists and has been permanently removed.'), - 411: ('Length Required', 'Client must specify Content-Length.'), - 412: ('Precondition Failed', 'Precondition in headers is false.'), - 413: ('Request Entity Too Large', 'Entity is too large.'), - 414: ('Request-URI Too Long', 'URI is too long.'), - 415: ('Unsupported Media Type', 'Entity body in unsupported format.'), - 416: ('Requested Range Not Satisfiable', - 'Cannot satisfy request range.'), - 417: ('Expectation Failed', - 'Expect condition could not be satisfied.'), - - 500: ('Internal Server Error', 'Server got itself in trouble'), - 501: ('Not Implemented', - 'Server does not support this operation'), - 502: ('Bad Gateway', 'Invalid responses from another server/proxy.'), - 503: ('Service Unavailable', - 'The server cannot process the request due to a high load'), - 504: ('Gateway Timeout', - 'The gateway server did not receive a timely response'), - 505: ('HTTP Version Not Supported', 'Cannot fulfill request.'), + ... + : ('OK', 'Request fulfilled, document follows'), + ... + : ('Forbidden', + 'Request forbidden -- authorization will ' + 'not help'), + : ('Not Found', + 'Nothing matches the given URI'), + ... + : ("I'm a Teapot", + 'Server refuses to brew coffee because ' + 'it is a teapot'), + ... + : ('Service Unavailable', + 'The server cannot process the ' + 'request due to a high load'), + ... } When an error is raised the server responds by returning an HTTP error code From cebae977a63f32c3c03d14c040df3cea55b8f585 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sun, 1 Jun 2025 00:33:02 +0300 Subject: [PATCH 043/116] gh-133891: Add missing error check to `SET_COUNT` macro in `_testinternalcapi.c` (#133892) --- Modules/_testinternalcapi.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 136e6a7a015049..845c218e679ad2 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1045,6 +1045,9 @@ get_code_var_counts(PyObject *self, PyObject *_args, PyObject *_kwargs) #define SET_COUNT(DICT, STRUCT, NAME) \ do { \ PyObject *count = PyLong_FromLong(STRUCT.NAME); \ + if (count == NULL) { \ + goto error; \ + } \ int res = PyDict_SetItemString(DICT, #NAME, count); \ Py_DECREF(count); \ if (res < 0) { \ From 965c48056633d3f4b41520c8cd07f0275f00fb4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Sun, 1 Jun 2025 07:56:56 +0200 Subject: [PATCH 044/116] gh-134970: Fix exception message in argparse module (GH-134971) Fix the "unknown action" exception in argparse.ArgumentParser.add_argument_group() to correctly replace the action class. --- Lib/argparse.py | 2 +- .../Library/2025-05-31-12-08-12.gh-issue-134970.lgSaxq.rst | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-31-12-08-12.gh-issue-134970.lgSaxq.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index d1a6350c3fda6d..83258cf3e0f37d 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1534,7 +1534,7 @@ def add_argument(self, *args, **kwargs): action_name = kwargs.get('action') action_class = self._pop_action_class(kwargs) if not callable(action_class): - raise ValueError('unknown action {action_class!r}') + raise ValueError(f'unknown action {action_class!r}') action = action_class(**kwargs) # raise an error if action for positional argument does not diff --git a/Misc/NEWS.d/next/Library/2025-05-31-12-08-12.gh-issue-134970.lgSaxq.rst b/Misc/NEWS.d/next/Library/2025-05-31-12-08-12.gh-issue-134970.lgSaxq.rst new file mode 100644 index 00000000000000..20f53569ef4566 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-31-12-08-12.gh-issue-134970.lgSaxq.rst @@ -0,0 +1,3 @@ +Fix the "unknown action" exception in +:meth:`argparse.ArgumentParser.add_argument_group` to correctly replace the +action class. From fe6f8a3619242b287a793a5b5d8645f402482c71 Mon Sep 17 00:00:00 2001 From: Rihaan Meher Date: Sun, 1 Jun 2025 04:18:31 -0400 Subject: [PATCH 045/116] gh-133503: clarify `compileall -s/-p` docs (#134756) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/compileall.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index c42288419c4d2d..ebbbf857e717a4 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -56,11 +56,18 @@ compile Python sources. executed. .. option:: -s strip_prefix + + Remove the given prefix from paths recorded in the ``.pyc`` files. + Paths are made relative to the prefix. + + This option can be used with ``-p`` but not with ``-d``. + .. option:: -p prepend_prefix - Remove (``-s``) or append (``-p``) the given prefix of paths - recorded in the ``.pyc`` files. - Cannot be combined with ``-d``. + Prepend the given prefix to paths recorded in the ``.pyc`` files. + Use ``-p /`` to make the paths absolute. + + This option can be used with ``-s`` but not with ``-d``. .. option:: -x regex From b5952371668089299bc8472c1adb9f8a0e69b4a2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 1 Jun 2025 11:22:15 +0300 Subject: [PATCH 046/116] gh-132983: Minor fixes and clean up for the _zstd module (GH-134930) --- Lib/test/test_zstd.py | 92 ++++++++++++++++++++++++++++++------ Modules/_zstd/_zstdmodule.c | 87 +++++++++++++++++++++++----------- Modules/_zstd/_zstdmodule.h | 10 +++- Modules/_zstd/compressor.c | 69 +++++---------------------- Modules/_zstd/decompressor.c | 67 ++++---------------------- Modules/_zstd/zstddict.c | 1 - 6 files changed, 166 insertions(+), 160 deletions(-) diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index 014634e450e449..e475d9346b9594 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -1138,27 +1138,41 @@ def test_invalid_dict(self): ZstdDecompressor(zd) # wrong type - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): - ZstdCompressor(zstd_dict=(zd, b'123')) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=[zd, 1]) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=(zd, 1.0)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=(zd,)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdCompressor(zstd_dict=(zd, 1, 2)) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdCompressor(zstd_dict=(zd, -1)) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdCompressor(zstd_dict=(zd, 3)) - - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): - ZstdDecompressor(zstd_dict=(zd, b'123')) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaises(OverflowError): + ZstdCompressor(zstd_dict=(zd, 2**1000)) + with self.assertRaises(OverflowError): + ZstdCompressor(zstd_dict=(zd, -2**1000)) + + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor(zstd_dict=[zd, 1]) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor(zstd_dict=(zd, 1.0)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor((zd,)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdDecompressor((zd, 1, 2)) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdDecompressor((zd, -1)) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdDecompressor((zd, 3)) + with self.assertRaises(OverflowError): + ZstdDecompressor((zd, 2**1000)) + with self.assertRaises(OverflowError): + ZstdDecompressor((zd, -2**1000)) def test_train_dict(self): - - TRAINED_DICT = train_dict(SAMPLES, DICT_SIZE1) ZstdDict(TRAINED_DICT.dict_content, is_raw=False) @@ -1239,18 +1253,37 @@ def test_train_dict_c(self): # argument wrong type with self.assertRaises(TypeError): _zstd.train_dict({}, (), 100) + with self.assertRaises(TypeError): + _zstd.train_dict(bytearray(), (), 100) with self.assertRaises(TypeError): _zstd.train_dict(b'', 99, 100) + with self.assertRaises(TypeError): + _zstd.train_dict(b'', [], 100) with self.assertRaises(TypeError): _zstd.train_dict(b'', (), 100.1) + with self.assertRaises(TypeError): + _zstd.train_dict(b'', (99.1,), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'abc', (4, -1), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'abc', (2,), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (99,), 100) # size > size_t with self.assertRaises(ValueError): - _zstd.train_dict(b'', (2**64+1,), 100) + _zstd.train_dict(b'', (2**1000,), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (-2**1000,), 100) # dict_size <= 0 with self.assertRaises(ValueError): _zstd.train_dict(b'', (), 0) + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (), -1) + + with self.assertRaises(ZstdError): + _zstd.train_dict(b'', (), 1) def test_finalize_dict_c(self): with self.assertRaises(TypeError): @@ -1259,22 +1292,51 @@ def test_finalize_dict_c(self): # argument wrong type with self.assertRaises(TypeError): _zstd.finalize_dict({}, b'', (), 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(bytearray(TRAINED_DICT.dict_content), b'', (), 100, 5) with self.assertRaises(TypeError): _zstd.finalize_dict(TRAINED_DICT.dict_content, {}, (), 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, bytearray(), (), 100, 5) with self.assertRaises(TypeError): _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', 99, 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', [], 100, 5) with self.assertRaises(TypeError): _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100.1, 5) with self.assertRaises(TypeError): _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5.1) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'abc', (4, -1), 100, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'abc', (2,), 100, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (99,), 100, 5) + # size > size_t with self.assertRaises(ValueError): - _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (2**64+1,), 100, 5) + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (2**1000,), 100, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (-2**1000,), 100, 5) # dict_size <= 0 with self.assertRaises(ValueError): _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 0, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), -1, 5) + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 2**1000, 5) + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), -2**1000, 5) + + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 2**1000) + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, -2**1000) + + with self.assertRaises(ZstdError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5) def test_train_buffer_protocol_samples(self): def _nbytes(dat): diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 986b3579479f0f..b0e50f873f4ca6 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -7,7 +7,6 @@ #include "Python.h" #include "_zstdmodule.h" -#include "zstddict.h" #include // ZSTD_*() #include // ZDICT_*() @@ -20,14 +19,52 @@ module _zstd #include "clinic/_zstdmodule.c.h" +ZstdDict * +_Py_parse_zstd_dict(const _zstd_state *state, PyObject *dict, int *ptype) +{ + if (state == NULL) { + return NULL; + } + + /* Check ZstdDict */ + if (PyObject_TypeCheck(dict, state->ZstdDict_type)) { + return (ZstdDict*)dict; + } + + /* Check (ZstdDict, type) */ + if (PyTuple_CheckExact(dict) && PyTuple_GET_SIZE(dict) == 2 + && PyObject_TypeCheck(PyTuple_GET_ITEM(dict, 0), state->ZstdDict_type) + && PyLong_Check(PyTuple_GET_ITEM(dict, 1))) + { + int type = PyLong_AsInt(PyTuple_GET_ITEM(dict, 1)); + if (type == -1 && PyErr_Occurred()) { + return NULL; + } + if (type == DICT_TYPE_DIGESTED + || type == DICT_TYPE_UNDIGESTED + || type == DICT_TYPE_PREFIX) + { + *ptype = type; + return (ZstdDict*)PyTuple_GET_ITEM(dict, 0); + } + } + + /* Wrong type */ + PyErr_SetString(PyExc_TypeError, + "zstd_dict argument should be a ZstdDict object."); + return NULL; +} + /* Format error message and set ZstdError. */ void -set_zstd_error(const _zstd_state* const state, - error_type type, size_t zstd_ret) +set_zstd_error(const _zstd_state *state, error_type type, size_t zstd_ret) { - char *msg; + const char *msg; assert(ZSTD_isError(zstd_ret)); + if (state == NULL) { + return; + } switch (type) { case ERR_DECOMPRESS: msg = "Unable to decompress Zstandard data: %s"; @@ -174,7 +211,7 @@ calculate_samples_stats(PyBytesObject *samples_bytes, PyObject *samples_sizes, Py_ssize_t sizes_sum; Py_ssize_t i; - chunks_number = Py_SIZE(samples_sizes); + chunks_number = PyTuple_GET_SIZE(samples_sizes); if ((size_t) chunks_number > UINT32_MAX) { PyErr_Format(PyExc_ValueError, "The number of samples should be <= %u.", UINT32_MAX); @@ -188,20 +225,24 @@ calculate_samples_stats(PyBytesObject *samples_bytes, PyObject *samples_sizes, return -1; } - sizes_sum = 0; + sizes_sum = PyBytes_GET_SIZE(samples_bytes); for (i = 0; i < chunks_number; i++) { - PyObject *size = PyTuple_GetItem(samples_sizes, i); - (*chunk_sizes)[i] = PyLong_AsSize_t(size); - if ((*chunk_sizes)[i] == (size_t)-1 && PyErr_Occurred()) { - PyErr_Format(PyExc_ValueError, - "Items in samples_sizes should be an int " - "object, with a value between 0 and %u.", SIZE_MAX); + size_t size = PyLong_AsSize_t(PyTuple_GET_ITEM(samples_sizes, i)); + (*chunk_sizes)[i] = size; + if (size == (size_t)-1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + goto sum_error; + } return -1; } - sizes_sum += (*chunk_sizes)[i]; + if ((size_t)sizes_sum < size) { + goto sum_error; + } + sizes_sum -= size; } - if (sizes_sum != Py_SIZE(samples_bytes)) { + if (sizes_sum != 0) { +sum_error: PyErr_SetString(PyExc_ValueError, "The samples size tuple doesn't match the " "concatenation's size."); @@ -257,7 +298,7 @@ _zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, /* Train the dictionary */ char *dst_dict_buffer = PyBytes_AS_STRING(dst_dict_bytes); - char *samples_buffer = PyBytes_AS_STRING(samples_bytes); + const char *samples_buffer = PyBytes_AS_STRING(samples_bytes); Py_BEGIN_ALLOW_THREADS zstd_ret = ZDICT_trainFromBuffer(dst_dict_buffer, dict_size, samples_buffer, @@ -507,17 +548,10 @@ _zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, { _zstd_state* mod_state = get_zstd_state(module); - if (!PyType_Check(c_parameter_type) || !PyType_Check(d_parameter_type)) { - PyErr_SetString(PyExc_ValueError, - "The two arguments should be CompressionParameter and " - "DecompressionParameter types."); - return NULL; - } - - Py_XSETREF( - mod_state->CParameter_type, (PyTypeObject*)Py_NewRef(c_parameter_type)); - Py_XSETREF( - mod_state->DParameter_type, (PyTypeObject*)Py_NewRef(d_parameter_type)); + Py_INCREF(c_parameter_type); + Py_XSETREF(mod_state->CParameter_type, (PyTypeObject*)c_parameter_type); + Py_INCREF(d_parameter_type); + Py_XSETREF(mod_state->DParameter_type, (PyTypeObject*)d_parameter_type); Py_RETURN_NONE; } @@ -580,7 +614,6 @@ do { \ return -1; } if (PyModule_AddType(m, (PyTypeObject *)mod_state->ZstdError) < 0) { - Py_DECREF(mod_state->ZstdError); return -1; } diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index 1f4160f474f0b0..c73f15b3c5299b 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -5,6 +5,8 @@ #ifndef ZSTD_MODULE_H #define ZSTD_MODULE_H +#include "zstddict.h" + /* Type specs */ extern PyType_Spec zstd_dict_type_spec; extern PyType_Spec zstd_compressor_type_spec; @@ -43,10 +45,14 @@ typedef enum { DICT_TYPE_PREFIX = 2 } dictionary_type; +extern ZstdDict * +_Py_parse_zstd_dict(const _zstd_state *state, + PyObject *dict, int *type); + /* Format error message and set ZstdError. */ extern void -set_zstd_error(const _zstd_state* const state, - const error_type type, size_t zstd_ret); +set_zstd_error(const _zstd_state *state, + error_type type, size_t zstd_ret); extern void set_parameter_error(int is_compress, int key_v, int value_v); diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 8ff2a3aadc1cd6..e1217635f60cb0 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -16,7 +16,6 @@ class _zstd.ZstdCompressor "ZstdCompressor *" "&zstd_compressor_type_spec" #include "_zstdmodule.h" #include "buffer.h" -#include "zstddict.h" #include "internal/pycore_lock.h" // PyMutex_IsLocked #include // offsetof() @@ -71,9 +70,6 @@ _zstd_set_c_level(ZstdCompressor *self, int level) /* Check error */ if (ZSTD_isError(zstd_ret)) { _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state == NULL) { - return -1; - } set_zstd_error(mod_state, ERR_SET_C_LEVEL, zstd_ret); return -1; } @@ -265,56 +261,17 @@ static int _zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) { _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state == NULL) { - return -1; - } - ZstdDict *zd; - int type, ret; - - /* Check ZstdDict */ - ret = PyObject_IsInstance(dict, (PyObject*)mod_state->ZstdDict_type); - if (ret < 0) { + /* When compressing, use undigested dictionary by default. */ + int type = DICT_TYPE_UNDIGESTED; + ZstdDict *zd = _Py_parse_zstd_dict(mod_state, dict, &type); + if (zd == NULL) { return -1; } - else if (ret > 0) { - /* When compressing, use undigested dictionary by default. */ - zd = (ZstdDict*)dict; - type = DICT_TYPE_UNDIGESTED; - PyMutex_Lock(&zd->lock); - ret = _zstd_load_impl(self, zd, mod_state, type); - PyMutex_Unlock(&zd->lock); - return ret; - } - - /* Check (ZstdDict, type) */ - if (PyTuple_CheckExact(dict) && PyTuple_GET_SIZE(dict) == 2) { - /* Check ZstdDict */ - ret = PyObject_IsInstance(PyTuple_GET_ITEM(dict, 0), - (PyObject*)mod_state->ZstdDict_type); - if (ret < 0) { - return -1; - } - else if (ret > 0) { - /* type == -1 may indicate an error. */ - type = PyLong_AsInt(PyTuple_GET_ITEM(dict, 1)); - if (type == DICT_TYPE_DIGESTED - || type == DICT_TYPE_UNDIGESTED - || type == DICT_TYPE_PREFIX) - { - assert(type >= 0); - zd = (ZstdDict*)PyTuple_GET_ITEM(dict, 0); - PyMutex_Lock(&zd->lock); - ret = _zstd_load_impl(self, zd, mod_state, type); - PyMutex_Unlock(&zd->lock); - return ret; - } - } - } - - /* Wrong type */ - PyErr_SetString(PyExc_TypeError, - "zstd_dict argument should be ZstdDict object."); - return -1; + int ret; + PyMutex_Lock(&zd->lock); + ret = _zstd_load_impl(self, zd, mod_state, type); + PyMutex_Unlock(&zd->lock); + return ret; } /*[clinic input] @@ -481,9 +438,7 @@ compress_lock_held(ZstdCompressor *self, Py_buffer *data, /* Check error */ if (ZSTD_isError(zstd_ret)) { _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state != NULL) { - set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret); - } + set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret); goto error; } @@ -553,9 +508,7 @@ compress_mt_continue_lock_held(ZstdCompressor *self, Py_buffer *data) /* Check error */ if (ZSTD_isError(zstd_ret)) { _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state != NULL) { - set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret); - } + set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret); goto error; } diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 26e568cf433308..c53d6e4cb05cf0 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -16,7 +16,6 @@ class _zstd.ZstdDecompressor "ZstdDecompressor *" "&zstd_decompressor_type_spec" #include "_zstdmodule.h" #include "buffer.h" -#include "zstddict.h" #include "internal/pycore_lock.h" // PyMutex_IsLocked #include // bool @@ -61,11 +60,6 @@ _get_DDict(ZstdDict *self) assert(PyMutex_IsLocked(&self->lock)); ZSTD_DDict *ret; - /* Already created */ - if (self->d_dict != NULL) { - return self->d_dict; - } - if (self->d_dict == NULL) { /* Create ZSTD_DDict instance from dictionary content */ Py_BEGIN_ALLOW_THREADS @@ -182,56 +176,17 @@ static int _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) { _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state == NULL) { - return -1; - } - ZstdDict *zd; - int type, ret; - - /* Check ZstdDict */ - ret = PyObject_IsInstance(dict, (PyObject*)mod_state->ZstdDict_type); - if (ret < 0) { + /* When decompressing, use digested dictionary by default. */ + int type = DICT_TYPE_DIGESTED; + ZstdDict *zd = _Py_parse_zstd_dict(mod_state, dict, &type); + if (zd == NULL) { return -1; } - else if (ret > 0) { - /* When decompressing, use digested dictionary by default. */ - zd = (ZstdDict*)dict; - type = DICT_TYPE_DIGESTED; - PyMutex_Lock(&zd->lock); - ret = _zstd_load_impl(self, zd, mod_state, type); - PyMutex_Unlock(&zd->lock); - return ret; - } - - /* Check (ZstdDict, type) */ - if (PyTuple_CheckExact(dict) && PyTuple_GET_SIZE(dict) == 2) { - /* Check ZstdDict */ - ret = PyObject_IsInstance(PyTuple_GET_ITEM(dict, 0), - (PyObject*)mod_state->ZstdDict_type); - if (ret < 0) { - return -1; - } - else if (ret > 0) { - /* type == -1 may indicate an error. */ - type = PyLong_AsInt(PyTuple_GET_ITEM(dict, 1)); - if (type == DICT_TYPE_DIGESTED - || type == DICT_TYPE_UNDIGESTED - || type == DICT_TYPE_PREFIX) - { - assert(type >= 0); - zd = (ZstdDict*)PyTuple_GET_ITEM(dict, 0); - PyMutex_Lock(&zd->lock); - ret = _zstd_load_impl(self, zd, mod_state, type); - PyMutex_Unlock(&zd->lock); - return ret; - } - } - } - - /* Wrong type */ - PyErr_SetString(PyExc_TypeError, - "zstd_dict argument should be ZstdDict object."); - return -1; + int ret; + PyMutex_Lock(&zd->lock); + ret = _zstd_load_impl(self, zd, mod_state, type); + PyMutex_Unlock(&zd->lock); + return ret; } /* @@ -282,9 +237,7 @@ decompress_lock_held(ZstdDecompressor *self, ZSTD_inBuffer *in, /* Check error */ if (ZSTD_isError(zstd_ret)) { _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state != NULL) { - set_zstd_error(mod_state, ERR_DECOMPRESS, zstd_ret); - } + set_zstd_error(mod_state, ERR_DECOMPRESS, zstd_ret); goto error; } diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index afc58b42e893d3..14f74aaed46ec5 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -15,7 +15,6 @@ class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec" #include "Python.h" #include "_zstdmodule.h" -#include "zstddict.h" #include "clinic/zstddict.c.h" #include "internal/pycore_lock.h" // PyMutex_IsLocked From f806463e16428ea4b379bf547bafa11f43a480ef Mon Sep 17 00:00:00 2001 From: Andrea-Oliveri <47737621+Andrea-Oliveri@users.noreply.github.com> Date: Sun, 1 Jun 2025 14:30:04 +0200 Subject: [PATCH 047/116] gh-134004: Added the reorganize() methods to dbm.sqlite, dbm.dumb and shelve (GH-134028) They are similar to the same named method in dbm.gnu. --- Doc/library/dbm.rst | 40 +++++++++++- Doc/library/shelve.rst | 16 ++++- Doc/whatsnew/3.15.rst | 17 ++++++ Lib/dbm/dumb.py | 32 +++++++++- Lib/dbm/sqlite3.py | 4 ++ Lib/shelve.py | 5 ++ Lib/test/test_dbm.py | 61 +++++++++++++++++++ Misc/ACKS | 1 + ...-05-15-00-27-09.gh-issue-134004.e8k4-R.rst | 2 + 9 files changed, 172 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-15-00-27-09.gh-issue-134004.e8k4-R.rst diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 36221c026d6d4b..6f548fbb1b39d8 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -15,10 +15,16 @@ * :mod:`dbm.ndbm` If none of these modules are installed, the -slow-but-simple implementation in module :mod:`dbm.dumb` will be used. There +slow-but-simple implementation in module :mod:`dbm.dumb` will be used. There is a `third party interface `_ to the Oracle Berkeley DB. +.. note:: + None of the underlying modules will automatically shrink the disk space used by + the database file. However, :mod:`dbm.sqlite3`, :mod:`dbm.gnu` and :mod:`dbm.dumb` + provide a :meth:`!reorganize` method that can be used for this purpose. + + .. exception:: error A tuple containing the exceptions that can be raised by each of the supported @@ -186,6 +192,17 @@ or any other SQLite browser, including the SQLite CLI. The Unix file access mode of the file (default: octal ``0o666``), used only when the database has to be created. + .. method:: sqlite3.reorganize() + + If you have carried out a lot of deletions and would like to shrink the space + used on disk, this method will reorganize the database; otherwise, deleted file + space will be kept and reused as new (key, value) pairs are added. + + .. note:: + While reorganizing, as much as two times the size of the original database is required + in free disk space. However, be aware that this factor changes for each :mod:`dbm` submodule. + + .. versionadded:: next :mod:`dbm.gnu` --- GNU database manager --------------------------------------- @@ -284,6 +301,10 @@ functionality like crash tolerance. reorganization; otherwise, deleted file space will be kept and reused as new (key, value) pairs are added. + .. note:: + While reorganizing, as much as one time the size of the original database is required + in free disk space. However, be aware that this factor changes for each :mod:`dbm` submodule. + .. method:: gdbm.sync() When the database has been opened in fast mode, this method forces any @@ -438,6 +459,11 @@ The :mod:`!dbm.dumb` module defines the following: with a sufficiently large/complex entry due to stack depth limitations in Python's AST compiler. + .. warning:: + :mod:`dbm.dumb` does not support concurrent read/write access. (Multiple + simultaneous read accesses are safe.) When a program has the database open + for writing, no other program should have it open for reading or writing. + .. versionchanged:: 3.5 :func:`~dbm.dumb.open` always creates a new database when *flag* is ``'n'``. @@ -460,3 +486,15 @@ The :mod:`!dbm.dumb` module defines the following: .. method:: dumbdbm.close() Close the database. + + .. method:: dumbdbm.reorganize() + + If you have carried out a lot of deletions and would like to shrink the space + used on disk, this method will reorganize the database; otherwise, deleted file + space will not be reused. + + .. note:: + While reorganizing, no additional free disk space is required. However, be aware + that this factor changes for each :mod:`dbm` submodule. + + .. versionadded:: next diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 6e74a59b82b8ec..23a2e0c3d0c758 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -75,8 +75,15 @@ Two additional methods are supported: Write back all entries in the cache if the shelf was opened with *writeback* set to :const:`True`. Also empty the cache and synchronize the persistent - dictionary on disk, if feasible. This is called automatically when the shelf - is closed with :meth:`close`. + dictionary on disk, if feasible. This is called automatically when + :meth:`reorganize` is called or the shelf is closed with :meth:`close`. + +.. method:: Shelf.reorganize() + + Calls :meth:`sync` and attempts to shrink space used on disk by removing empty + space resulting from deletions. + + .. versionadded:: next .. method:: Shelf.close() @@ -116,6 +123,11 @@ Restrictions * On macOS :mod:`dbm.ndbm` can silently corrupt the database file on updates, which can cause hard crashes when trying to read from the database. +* :meth:`Shelf.reorganize` may not be available for all database packages and + may temporarely increase resource usage (especially disk space) when called. + Additionally, it will never run automatically and instead needs to be called + explicitly. + .. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8') diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 6d1f653f086a15..244ce327763f57 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -89,6 +89,14 @@ New modules Improved modules ================ +dbm +--- + +* Added new :meth:`!reorganize` methods to :mod:`dbm.dumb` and :mod:`dbm.sqlite3` + which allow to recover unused free space previously occupied by deleted entries. + (Contributed by Andrea Oliveri in :gh:`134004`.) + + difflib ------- @@ -96,6 +104,15 @@ difflib class, and migrated the output to the HTML5 standard. (Contributed by Jiahao Li in :gh:`134580`.) + +shelve +------ + +* Added new :meth:`!reorganize` method to :mod:`shelve` used to recover unused free + space previously occupied by deleted entries. + (Contributed by Andrea Oliveri in :gh:`134004`.) + + ssl --- diff --git a/Lib/dbm/dumb.py b/Lib/dbm/dumb.py index def120ffc3778b..1bc239a84fff83 100644 --- a/Lib/dbm/dumb.py +++ b/Lib/dbm/dumb.py @@ -9,7 +9,7 @@ - seems to contain a bug when updating... - reclaim free space (currently, space once occupied by deleted or expanded -items is never reused) +items is not reused exept if .reorganize() is called) - support concurrent access (currently, if two processes take turns making updates, they can mess up the index) @@ -17,8 +17,6 @@ - support efficient access to large databases (currently, the whole index is read when the database is opened, and some updates rewrite the whole index) -- support opening for read-only (flag = 'm') - """ import ast as _ast @@ -289,6 +287,34 @@ def __enter__(self): def __exit__(self, *args): self.close() + def reorganize(self): + if self._readonly: + raise error('The database is opened for reading only') + self._verify_open() + # Ensure all changes are committed before reorganizing. + self._commit() + # Open file in r+ to allow changing in-place. + with _io.open(self._datfile, 'rb+') as f: + reorganize_pos = 0 + + # Iterate over existing keys, sorted by starting byte. + for key in sorted(self._index, key = lambda k: self._index[k][0]): + pos, siz = self._index[key] + f.seek(pos) + val = f.read(siz) + + f.seek(reorganize_pos) + f.write(val) + self._index[key] = (reorganize_pos, siz) + + blocks_occupied = (siz + _BLOCKSIZE - 1) // _BLOCKSIZE + reorganize_pos += blocks_occupied * _BLOCKSIZE + + f.truncate(reorganize_pos) + # Commit changes to index, which were not in-place. + self._commit() + + def open(file, flag='c', mode=0o666): """Open the database file, filename, and return corresponding object. diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 7e0ae2a29e3a64..b296a1bcd1bbfa 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -15,6 +15,7 @@ STORE_KV = "REPLACE INTO Dict (key, value) VALUES (CAST(? AS BLOB), CAST(? AS BLOB))" DELETE_KEY = "DELETE FROM Dict WHERE key = CAST(? AS BLOB)" ITER_KEYS = "SELECT key FROM Dict" +REORGANIZE = "VACUUM" class error(OSError): @@ -122,6 +123,9 @@ def __enter__(self): def __exit__(self, *args): self.close() + def reorganize(self): + self._execute(REORGANIZE) + def open(filename, /, flag="r", mode=0o666): """Open a dbm.sqlite3 database and return the dbm object. diff --git a/Lib/shelve.py b/Lib/shelve.py index 50584716e9ea64..b53dc8b7a8ece9 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -171,6 +171,11 @@ def sync(self): if hasattr(self.dict, 'sync'): self.dict.sync() + def reorganize(self): + self.sync() + if hasattr(self.dict, 'reorganize'): + self.dict.reorganize() + class BsdDbShelf(Shelf): """Shelf implementation using the "BSD" db interface. diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index a10922a403ec38..7e8d78b89405ab 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -135,6 +135,67 @@ def test_anydbm_access(self): assert(f[key] == b"Python:") f.close() + def test_anydbm_readonly_reorganize(self): + self.init_db() + with dbm.open(_fname, 'r') as d: + # Early stopping. + if not hasattr(d, 'reorganize'): + self.skipTest("method reorganize not available this dbm submodule") + + self.assertRaises(dbm.error, lambda: d.reorganize()) + + def test_anydbm_reorganize_not_changed_content(self): + self.init_db() + with dbm.open(_fname, 'c') as d: + # Early stopping. + if not hasattr(d, 'reorganize'): + self.skipTest("method reorganize not available this dbm submodule") + + keys_before = sorted(d.keys()) + values_before = [d[k] for k in keys_before] + d.reorganize() + keys_after = sorted(d.keys()) + values_after = [d[k] for k in keys_before] + self.assertEqual(keys_before, keys_after) + self.assertEqual(values_before, values_after) + + def test_anydbm_reorganize_decreased_size(self): + + def _calculate_db_size(db_path): + if os.path.isfile(db_path): + return os.path.getsize(db_path) + total_size = 0 + for root, _, filenames in os.walk(db_path): + for filename in filenames: + file_path = os.path.join(root, filename) + total_size += os.path.getsize(file_path) + return total_size + + # This test requires relatively large databases to reliably show difference in size before and after reorganizing. + with dbm.open(_fname, 'n') as f: + # Early stopping. + if not hasattr(f, 'reorganize'): + self.skipTest("method reorganize not available this dbm submodule") + + for k in self._dict: + f[k.encode('ascii')] = self._dict[k] * 100000 + db_keys = list(f.keys()) + + # Make sure to calculate size of database only after file is closed to ensure file content are flushed to disk. + size_before = _calculate_db_size(os.path.dirname(_fname)) + + # Delete some elements from the start of the database. + keys_to_delete = db_keys[:len(db_keys) // 2] + with dbm.open(_fname, 'c') as f: + for k in keys_to_delete: + del f[k] + f.reorganize() + + # Make sure to calculate size of database only after file is closed to ensure file content are flushed to disk. + size_after = _calculate_db_size(os.path.dirname(_fname)) + + self.assertLess(size_after, size_before) + def test_open_with_bytes(self): dbm.open(os.fsencode(_fname), "c").close() diff --git a/Misc/ACKS b/Misc/ACKS index 571142e7e49763..2435943f1bb2bd 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1365,6 +1365,7 @@ Milan Oberkirch Pascal Oberndoerfer Géry Ogam Seonkyo Ok +Andrea Oliveri Jeffrey Ollie Adam Olsen Bryan Olson diff --git a/Misc/NEWS.d/next/Library/2025-05-15-00-27-09.gh-issue-134004.e8k4-R.rst b/Misc/NEWS.d/next/Library/2025-05-15-00-27-09.gh-issue-134004.e8k4-R.rst new file mode 100644 index 00000000000000..a9a56d9239b305 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-15-00-27-09.gh-issue-134004.e8k4-R.rst @@ -0,0 +1,2 @@ +:mod:`shelve` as well as underlying :mod:`!dbm.dumb` and :mod:`!dbm.sqlite` now have :meth:`!reorganize` methods to +recover unused free space previously occupied by deleted entries. From ac7511062bf8e16ad489b17990d99abd3b4351f5 Mon Sep 17 00:00:00 2001 From: Sam Ng Date: Sun, 1 Jun 2025 08:35:58 -0700 Subject: [PATCH 048/116] gh-130478: fix HACL* build for macOS Silicon (#134188) --- configure | 12 ++++++++++-- configure.ac | 13 +++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/configure b/configure index c9ce9b4a73527d..b0dc18d5cea749 100755 --- a/configure +++ b/configure @@ -32557,6 +32557,14 @@ LIBHACL_CFLAGS="${LIBHACL_FLAG_I} ${LIBHACL_FLAG_D} \$(PY_STDMODULE_CFLAGS) \$(C LIBHACL_LDFLAGS= # for now, no specific linker flags are needed +if test "$UNIVERSAL_ARCHS" = "universal2" -o \ + \( "$build_cpu" = "aarch64" -a "$build_vendor" = "apple" \) +then + use_hacl_universal2_impl=yes +else + use_hacl_universal2_impl=no +fi + # The SIMD files use aligned_alloc, which is not available on older versions of # Android. # The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. @@ -32610,7 +32618,7 @@ printf "%s\n" "#define HACL_CAN_COMPILE_SIMD128 1" >>confdefs.h # isn't great, so it's disabled on ARM64. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* SIMD128 implementation" >&5 printf %s "checking for HACL* SIMD128 implementation... " >&6; } - if test "$UNIVERSAL_ARCHS" == "universal2"; then + if test "$use_hacl_universal2_impl" = "yes"; then LIBHACL_BLAKE2_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: universal2" >&5 printf "%s\n" "universal2" >&6; } @@ -32687,7 +32695,7 @@ printf "%s\n" "#define HACL_CAN_COMPILE_SIMD256 1" >>confdefs.h # wrapped implementation if we're building for universal2. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* SIMD256 implementation" >&5 printf %s "checking for HACL* SIMD256 implementation... " >&6; } - if test "$UNIVERSAL_ARCHS" == "universal2"; then + if test "$use_hacl_universal2_impl" = "yes"; then LIBHACL_BLAKE2_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: universal2" >&5 printf "%s\n" "universal2" >&6; } diff --git a/configure.ac b/configure.ac index eb52365d95be21..70ad6da86719c6 100644 --- a/configure.ac +++ b/configure.ac @@ -8001,6 +8001,15 @@ AC_SUBST([LIBHACL_CFLAGS]) LIBHACL_LDFLAGS= # for now, no specific linker flags are needed AC_SUBST([LIBHACL_LDFLAGS]) +dnl Check if universal2 HACL* implementation should be used. +if test "$UNIVERSAL_ARCHS" = "universal2" -o \ + \( "$build_cpu" = "aarch64" -a "$build_vendor" = "apple" \) +then + use_hacl_universal2_impl=yes +else + use_hacl_universal2_impl=no +fi + # The SIMD files use aligned_alloc, which is not available on older versions of # Android. # The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. @@ -8017,7 +8026,7 @@ then # available on x86_64. However, performance of the HACL SIMD128 implementation # isn't great, so it's disabled on ARM64. AC_MSG_CHECKING([for HACL* SIMD128 implementation]) - if test "$UNIVERSAL_ARCHS" == "universal2"; then + if test "$use_hacl_universal2_impl" = "yes"; then [LIBHACL_BLAKE2_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o"] AC_MSG_RESULT([universal2]) else @@ -8049,7 +8058,7 @@ then # implementation requires symbols that aren't available on ARM64. Use a # wrapped implementation if we're building for universal2. AC_MSG_CHECKING([for HACL* SIMD256 implementation]) - if test "$UNIVERSAL_ARCHS" == "universal2"; then + if test "$use_hacl_universal2_impl" = "yes"; then [LIBHACL_BLAKE2_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o"] AC_MSG_RESULT([universal2]) else From ee65ebdb50005655d75aca1618d3994a7b7ed869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 2 Jun 2025 10:25:50 +0200 Subject: [PATCH 049/116] gh-134978: deprecate `string` keyword parameter for hash function constructors (#134979) --- Doc/deprecations/pending-removal-in-3.19.rst | 16 +++++++++++++++ Doc/library/hashlib.rst | 7 +++++++ Doc/whatsnew/3.15.rst | 16 +++++++++++++-- Lib/test/test_hashlib.py | 20 ++++++++++++++++--- ...-05-31-15-49-46.gh-issue-134978.mXXuvW.rst | 7 +++++++ Modules/hashlib.h | 9 +++++++++ 6 files changed, 70 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-31-15-49-46.gh-issue-134978.mXXuvW.rst diff --git a/Doc/deprecations/pending-removal-in-3.19.rst b/Doc/deprecations/pending-removal-in-3.19.rst index 3936f63ca5b5af..25f9cba390de68 100644 --- a/Doc/deprecations/pending-removal-in-3.19.rst +++ b/Doc/deprecations/pending-removal-in-3.19.rst @@ -6,3 +6,19 @@ Pending removal in Python 3.19 * Implicitly switching to the MSVC-compatible struct layout by setting :attr:`~ctypes.Structure._pack_` but not :attr:`~ctypes.Structure._layout_` on non-Windows platforms. + +* :mod:`hashlib`: + + - In hash function constructors such as :func:`~hashlib.new` or the + direct hash-named constructors such as :func:`~hashlib.md5` and + :func:`~hashlib.sha256`, their optional initial data parameter could + also be passed a keyword argument named ``data=`` or ``string=`` in + various :mod:`!hashlib` implementations. + + Support for the ``string`` keyword argument name is now deprecated + and slated for removal in Python 3.19. + + Before Python 3.13, the ``string`` keyword parameter was not correctly + supported depending on the backend implementation of hash functions. + Prefer passing the initial data as a positional argument for maximum + backwards compatibility. diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 4818a4944a512a..8bba6700930bf4 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -94,6 +94,13 @@ accessible by name via :func:`new`. See :data:`algorithms_available`. OpenSSL does not provide we fall back to a verified implementation from the `HACL\* project`_. +.. deprecated-removed:: 3.15 3.19 + The undocumented ``string`` keyword parameter in :func:`!_hashlib.new` + and hash-named constructors such as :func:`!_md5.md5` is deprecated. + Prefer passing the initial data as a positional argument for maximum + backwards compatibility. + + Usage ----- diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 244ce327763f57..b342939f70577f 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -146,8 +146,20 @@ module_name Deprecated ========== -* module_name: - TODO +hashlib +------- + +* In hash function constructors such as :func:`~hashlib.new` or the + direct hash-named constructors such as :func:`~hashlib.md5` and + :func:`~hashlib.sha256`, their optional initial data parameter could + also be passed a keyword argument named ``data=`` or ``string=`` in + various :mod:`hashlib` implementations. + + Support for the ``string`` keyword argument name is now deprecated and + is slated for removal in Python 3.19. Prefer passing the initial data as + a positional argument for maximum backwards compatibility. + + (Contributed by Bénédikt Tran in :gh:`134978`.) .. Add deprecations above alphabetically, not here at the end. diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 8244f7c7553a37..b83ae181718b7a 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -98,6 +98,14 @@ def read_vectors(hash_name): yield parts +DEPRECATED_STRING_PARAMETER = re.escape( + "the 'string' keyword parameter is deprecated since " + "Python 3.15 and slated for removal in Python 3.19; " + "use the 'data' keyword parameter or pass the data " + "to hash as a positional argument instead" +) + + class HashLibTestCase(unittest.TestCase): supported_hash_names = ( 'md5', 'MD5', 'sha1', 'SHA1', 'sha224', 'SHA224', 'sha256', 'SHA256', @@ -255,17 +263,23 @@ def test_clinic_signature(self): with self.subTest(constructor.__name__): constructor(b'') constructor(data=b'') - constructor(string=b'') # should be deprecated in the future + with self.assertWarnsRegex(DeprecationWarning, + DEPRECATED_STRING_PARAMETER): + constructor(string=b'') digest_name = constructor(b'').name with self.subTest(digest_name): hashlib.new(digest_name, b'') hashlib.new(digest_name, data=b'') - hashlib.new(digest_name, string=b'') + with self.assertWarnsRegex(DeprecationWarning, + DEPRECATED_STRING_PARAMETER): + hashlib.new(digest_name, string=b'') if self._hashlib: self._hashlib.new(digest_name, b'') self._hashlib.new(digest_name, data=b'') - self._hashlib.new(digest_name, string=b'') + with self.assertWarnsRegex(DeprecationWarning, + DEPRECATED_STRING_PARAMETER): + self._hashlib.new(digest_name, string=b'') @unittest.skipIf(get_fips_mode(), "skip in FIPS mode") def test_clinic_signature_errors(self): diff --git a/Misc/NEWS.d/next/Library/2025-05-31-15-49-46.gh-issue-134978.mXXuvW.rst b/Misc/NEWS.d/next/Library/2025-05-31-15-49-46.gh-issue-134978.mXXuvW.rst new file mode 100644 index 00000000000000..e75ce1622d6396 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-31-15-49-46.gh-issue-134978.mXXuvW.rst @@ -0,0 +1,7 @@ +:mod:`hashlib`: Supporting the ``string`` keyword parameter in hash function +constructors such as :func:`~hashlib.new` or the direct hash-named constructors +such as :func:`~hashlib.md5` and :func:`~hashlib.sha256` is now deprecated and +slated for removal in Python 3.19. +Prefer passing the initial data as a positional argument for maximum backwards +compatibility. +Patch by Bénédikt Tran. diff --git a/Modules/hashlib.h b/Modules/hashlib.h index a80b195a765792..e82ec92be25c57 100644 --- a/Modules/hashlib.h +++ b/Modules/hashlib.h @@ -86,6 +86,15 @@ _Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string) } else if (data == NULL && string != NULL) { // called as H(string=...) + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "the 'string' keyword parameter is deprecated since " + "Python 3.15 and slated for removal in Python 3.19; " + "use the 'data' keyword parameter or pass the data " + "to hash as a positional argument instead", 1) < 0) + { + *res = NULL; + return -1; + } *res = string; return 1; } From 7828d52680907d1661ff6993e540f7026461c390 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Mon, 2 Jun 2025 12:45:48 +0300 Subject: [PATCH 050/116] gh-134449: fix grammar for `precision_with_grouping` in format description (#134608) This amends commit f39a07be47cd9219eaf0e538ae32ad8239c88e66. --- Doc/library/string.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index c4012483a52f73..23e15780075435 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -328,7 +328,7 @@ The general form of a *standard format specifier* is: sign: "+" | "-" | " " width_and_precision: [`width_with_grouping`][`precision_with_grouping`] width_with_grouping: [`width`][`grouping`] - precision_with_grouping: "." [`precision`][`grouping`] + precision_with_grouping: "." [`precision`][`grouping`] | "." `grouping` width: `~python-grammar:digit`+ precision: `~python-grammar:digit`+ grouping: "," | "_" From 128195e12eb6d5b9542558453df7045dd7aa1e15 Mon Sep 17 00:00:00 2001 From: GalaxySnail Date: Mon, 2 Jun 2025 18:23:52 +0800 Subject: [PATCH 051/116] gh-122153: indicate that Windows does not support `socket.{send,recv}_fds` (#134960) This amends commit e3b6ff19aaa318a813130ba9ad2ab0a332f27feb. --- Doc/library/socket.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 75fd637045d7d8..bc89a3228f0ed9 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -1492,7 +1492,7 @@ The :mod:`socket` module also offers various network-related services: The *fds* parameter is a sequence of file descriptors. Consult :meth:`~socket.sendmsg` for the documentation of these parameters. - .. availability:: Unix, Windows, not WASI. + .. availability:: Unix, not WASI. Unix platforms supporting :meth:`~socket.sendmsg` and :const:`SCM_RIGHTS` mechanism. @@ -1506,9 +1506,9 @@ The :mod:`socket` module also offers various network-related services: Return ``(msg, list(fds), flags, addr)``. Consult :meth:`~socket.recvmsg` for the documentation of these parameters. - .. availability:: Unix, Windows, not WASI. + .. availability:: Unix, not WASI. - Unix platforms supporting :meth:`~socket.sendmsg` + Unix platforms supporting :meth:`~socket.recvmsg` and :const:`SCM_RIGHTS` mechanism. .. versionadded:: 3.9 From 5f61cde80a9b33c8e118b1c009fe2aaa4bb87356 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Mon, 2 Jun 2025 13:38:05 +0300 Subject: [PATCH 052/116] gh-132908: Add math.isnormal/issubnormal() functions (GH132935) --- Doc/library/math.rst | 20 ++++++ Doc/whatsnew/3.14.rst | 2 +- Doc/whatsnew/3.15.rst | 7 ++ Lib/test/test_math.py | 22 ++++++ ...-04-25-16-06-53.gh-issue-132908.wV5rja.rst | 2 + Modules/clinic/mathmodule.c.h | 70 ++++++++++++++++++- Modules/mathmodule.c | 40 +++++++++++ 7 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-04-25-16-06-53.gh-issue-132908.wV5rja.rst diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 11d3b756e21322..c8061fb16380cd 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -53,6 +53,8 @@ noted otherwise, all return values are floats. :func:`frexp(x) ` Mantissa and exponent of *x* :func:`isclose(a, b, rel_tol, abs_tol) ` Check if the values *a* and *b* are close to each other :func:`isfinite(x) ` Check if *x* is neither an infinity nor a NaN +:func:`isnormal(x) ` Check if *x* is a normal number +:func:`issubnormal(x) ` Check if *x* is a subnormal number :func:`isinf(x) ` Check if *x* is a positive or negative infinity :func:`isnan(x) ` Check if *x* is a NaN (not a number) :func:`ldexp(x, i) ` ``x * (2**i)``, inverse of function :func:`frexp` @@ -373,6 +375,24 @@ Floating point manipulation functions .. versionadded:: 3.2 +.. function:: isnormal(x) + + Return ``True`` if *x* is a normal number, that is a finite + nonzero number that is not a subnormal (see :func:`issubnormal`). + Return ``False`` otherwise. + + .. versionadded:: next + + +.. function:: issubnormal(x) + + Return ``True`` if *x* is a subnormal number, that is a finite + nonzero number with a magnitude smaller than the smallest positive normal + number, see :data:`sys.float_info.min`. Return ``False`` otherwise. + + .. versionadded:: next + + .. function:: isinf(x) Return ``True`` if *x* is a positive or negative infinity, and diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 561d1a8914b50c..27dfc75c90fbe9 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1454,7 +1454,7 @@ math ---- * Added more detailed error messages for domain errors in the module. - (Contributed by by Charlie Zhao and Sergey B Kirpichev in :gh:`101410`.) + (Contributed by Charlie Zhao and Sergey B Kirpichev in :gh:`101410`.) mimetypes diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index b342939f70577f..a27a17afdba2a8 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -105,6 +105,13 @@ difflib (Contributed by Jiahao Li in :gh:`134580`.) +math +---- + +* Add :func:`math.isnormal` and :func:`math.issubnormal` functions. + (Contributed by Sergey B Kirpichev in :gh:`132908`.) + + shelve ------ diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index d14336f8bac498..384ad5c828d9b3 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -1973,6 +1973,28 @@ def testIsfinite(self): self.assertFalse(math.isfinite(float("inf"))) self.assertFalse(math.isfinite(float("-inf"))) + def testIsnormal(self): + self.assertTrue(math.isnormal(1.25)) + self.assertTrue(math.isnormal(-1.0)) + self.assertFalse(math.isnormal(0.0)) + self.assertFalse(math.isnormal(-0.0)) + self.assertFalse(math.isnormal(INF)) + self.assertFalse(math.isnormal(NINF)) + self.assertFalse(math.isnormal(NAN)) + self.assertFalse(math.isnormal(FLOAT_MIN/2)) + self.assertFalse(math.isnormal(-FLOAT_MIN/2)) + + def testIssubnormal(self): + self.assertFalse(math.issubnormal(1.25)) + self.assertFalse(math.issubnormal(-1.0)) + self.assertFalse(math.issubnormal(0.0)) + self.assertFalse(math.issubnormal(-0.0)) + self.assertFalse(math.issubnormal(INF)) + self.assertFalse(math.issubnormal(NINF)) + self.assertFalse(math.issubnormal(NAN)) + self.assertTrue(math.issubnormal(FLOAT_MIN/2)) + self.assertTrue(math.issubnormal(-FLOAT_MIN/2)) + def testIsnan(self): self.assertTrue(math.isnan(float("nan"))) self.assertTrue(math.isnan(float("-nan"))) diff --git a/Misc/NEWS.d/next/Library/2025-04-25-16-06-53.gh-issue-132908.wV5rja.rst b/Misc/NEWS.d/next/Library/2025-04-25-16-06-53.gh-issue-132908.wV5rja.rst new file mode 100644 index 00000000000000..e33b061bb9ba1f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-25-16-06-53.gh-issue-132908.wV5rja.rst @@ -0,0 +1,2 @@ +Add :func:`math.isnormal` and :func:`math.issubnormal` functions. Patch by +Sergey B Kirpichev. diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index 9df73b187bb827..fbb012fb6dd9e1 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -628,6 +628,74 @@ math_isfinite(PyObject *module, PyObject *arg) return return_value; } +PyDoc_STRVAR(math_isnormal__doc__, +"isnormal($module, x, /)\n" +"--\n" +"\n" +"Return True if x is normal, and False otherwise."); + +#define MATH_ISNORMAL_METHODDEF \ + {"isnormal", (PyCFunction)math_isnormal, METH_O, math_isnormal__doc__}, + +static PyObject * +math_isnormal_impl(PyObject *module, double x); + +static PyObject * +math_isnormal(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + double x; + + if (PyFloat_CheckExact(arg)) { + x = PyFloat_AS_DOUBLE(arg); + } + else + { + x = PyFloat_AsDouble(arg); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + return_value = math_isnormal_impl(module, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(math_issubnormal__doc__, +"issubnormal($module, x, /)\n" +"--\n" +"\n" +"Return True if x is subnormal, and False otherwise."); + +#define MATH_ISSUBNORMAL_METHODDEF \ + {"issubnormal", (PyCFunction)math_issubnormal, METH_O, math_issubnormal__doc__}, + +static PyObject * +math_issubnormal_impl(PyObject *module, double x); + +static PyObject * +math_issubnormal(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + double x; + + if (PyFloat_CheckExact(arg)) { + x = PyFloat_AS_DOUBLE(arg); + } + else + { + x = PyFloat_AsDouble(arg); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + return_value = math_issubnormal_impl(module, x); + +exit: + return return_value; +} + PyDoc_STRVAR(math_isnan__doc__, "isnan($module, x, /)\n" "--\n" @@ -1110,4 +1178,4 @@ math_ulp(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=77e7b8c161c39843 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=44bba3a0a052a364 input=a9049054013a1b77]*/ diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 71d9c1387f5780..bbbb49115681de 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -3118,6 +3118,44 @@ math_isfinite_impl(PyObject *module, double x) } +/*[clinic input] +math.isnormal + + x: double + / + +Return True if x is normal, and False otherwise. +[clinic start generated code]*/ + +static PyObject * +math_isnormal_impl(PyObject *module, double x) +/*[clinic end generated code: output=c7b302b5b89c3541 input=fdaa00c58aa7bc17]*/ +{ + return PyBool_FromLong(isnormal(x)); +} + + +/*[clinic input] +math.issubnormal + + x: double + / + +Return True if x is subnormal, and False otherwise. +[clinic start generated code]*/ + +static PyObject * +math_issubnormal_impl(PyObject *module, double x) +/*[clinic end generated code: output=4e76ac98ddcae761 input=9a20aba7107d0d95]*/ +{ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L + return PyBool_FromLong(issubnormal(x)); +#else + return PyBool_FromLong(isfinite(x) && x && !isnormal(x)); +#endif +} + + /*[clinic input] math.isnan @@ -4145,6 +4183,8 @@ static PyMethodDef math_methods[] = { MATH_HYPOT_METHODDEF MATH_ISCLOSE_METHODDEF MATH_ISFINITE_METHODDEF + MATH_ISNORMAL_METHODDEF + MATH_ISSUBNORMAL_METHODDEF MATH_ISINF_METHODDEF MATH_ISNAN_METHODDEF MATH_ISQRT_METHODDEF From baccfdb3d4d004cfb5308674e5e6ea6e598abcd7 Mon Sep 17 00:00:00 2001 From: devdanzin <74280297+devdanzin@users.noreply.github.com> Date: Mon, 2 Jun 2025 08:04:59 -0300 Subject: [PATCH 053/116] gh-130999: Avoid exiting the new REPL when there are non-string candidates for suggestions (gh-131001) --- Lib/test/test_pyrepl/test_pyrepl.py | 11 +++++++++ Lib/test/test_traceback.py | 24 +++++++++++++++++++ Lib/traceback.py | 13 ++++++++-- ...-03-09-03-13-41.gh-issue-130999.tBRBVB.rst | 2 ++ 4 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-03-09-03-13-41.gh-issue-130999.tBRBVB.rst diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index aa3a592766d6d1..98bae7dd703fd9 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1672,6 +1672,17 @@ def test_null_byte(self): self.assertEqual(exit_code, 0) self.assertNotIn("TypeError", output) + @force_not_colorized + def test_non_string_suggestion_candidates(self): + commands = ("import runpy\n" + "runpy._run_module_code('blech', {0: '', 'bluch': ''}, '')\n" + "exit()\n") + + output, exit_code = self.run_repl(commands) + self.assertEqual(exit_code, 0) + self.assertNotIn("all elements in 'candidates' must be strings", output) + self.assertIn("bluch", output) + def test_readline_history_file(self): # skip, if readline module is not available readline = import_module('readline') diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index b9be87f357ffdd..6b2271f5d5ba8d 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -4188,6 +4188,15 @@ def __dir__(self): self.assertNotIn("blech", actual) self.assertNotIn("oh no!", actual) + def test_attribute_error_with_non_string_candidates(self): + class T: + bluch = 1 + + instance = T() + instance.__dict__[0] = 1 + actual = self.get_suggestion(instance, 'blich') + self.assertIn("bluch", actual) + def test_attribute_error_with_bad_name(self): def raise_attribute_error_with_bad_name(): raise AttributeError(name=12, obj=23) @@ -4301,6 +4310,13 @@ def test_import_from_suggestions_underscored(self): self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_luch')) self.assertNotIn("'_bluch'", self.get_import_from_suggestion(code, 'bluch')) + def test_import_from_suggestions_non_string(self): + modWithNonStringAttr = textwrap.dedent("""\ + globals()[0] = 1 + bluch = 1 + """) + self.assertIn("'bluch'", self.get_import_from_suggestion(modWithNonStringAttr, 'blech')) + def test_import_from_suggestions_do_not_trigger_for_long_attributes(self): code = "blech = None" @@ -4397,6 +4413,14 @@ def func(): actual = self.get_suggestion(func) self.assertIn("'ZeroDivisionError'?", actual) + def test_name_error_suggestions_with_non_string_candidates(self): + def func(): + abc = 1 + globals()[0] = 1 + abv + actual = self.get_suggestion(func) + self.assertIn("abc", actual) + def test_name_error_suggestions_do_not_trigger_for_long_names(self): def func(): somethingverywronghehehehehehe = None diff --git a/Lib/traceback.py b/Lib/traceback.py index 17b082eced6f05..a1f175dbbaa421 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1595,7 +1595,11 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): if isinstance(exc_value, AttributeError): obj = exc_value.obj try: - d = dir(obj) + try: + d = dir(obj) + except TypeError: # Attributes are unsortable, e.g. int and str + d = list(obj.__class__.__dict__.keys()) + list(obj.__dict__.keys()) + d = sorted([x for x in d if isinstance(x, str)]) hide_underscored = (wrong_name[:1] != '_') if hide_underscored and tb is not None: while tb.tb_next is not None: @@ -1610,7 +1614,11 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): elif isinstance(exc_value, ImportError): try: mod = __import__(exc_value.name) - d = dir(mod) + try: + d = dir(mod) + except TypeError: # Attributes are unsortable, e.g. int and str + d = list(mod.__dict__.keys()) + d = sorted([x for x in d if isinstance(x, str)]) if wrong_name[:1] != '_': d = [x for x in d if x[:1] != '_'] except Exception: @@ -1628,6 +1636,7 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): + list(frame.f_globals) + list(frame.f_builtins) ) + d = [x for x in d if isinstance(x, str)] # Check first if we are in a method and the instance # has the wrong name as attribute diff --git a/Misc/NEWS.d/next/Library/2025-03-09-03-13-41.gh-issue-130999.tBRBVB.rst b/Misc/NEWS.d/next/Library/2025-03-09-03-13-41.gh-issue-130999.tBRBVB.rst new file mode 100644 index 00000000000000..157522f9aab1b6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-09-03-13-41.gh-issue-130999.tBRBVB.rst @@ -0,0 +1,2 @@ +Avoid exiting the new REPL and offer suggestions even if there are non-string +candidates when errors occur. From 5bc2d99126e40cb727a249939a4f905a51e8c920 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Mon, 2 Jun 2025 16:28:20 +0300 Subject: [PATCH 054/116] gh-130662: Accept leading zeros in precision/width for Fraction's formatting (#130663) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/fractions.py | 4 ++-- Lib/test/test_fractions.py | 8 ++------ .../2025-06-02-14-28-30.gh-issue-130662.EIgIR8.rst | 3 +++ 3 files changed, 7 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-06-02-14-28-30.gh-issue-130662.EIgIR8.rst diff --git a/Lib/fractions.py b/Lib/fractions.py index 063f28478c7338..cb05ae7c2003f0 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -168,9 +168,9 @@ def _round_to_figures(n, d, figures): # A '0' that's *not* followed by another digit is parsed as a minimum width # rather than a zeropad flag. (?P0(?=[0-9]))? - (?P0|[1-9][0-9]*)? + (?P[0-9]+)? (?P[,_])? - (?:\.(?P0|[1-9][0-9]*))? + (?:\.(?P[0-9]+))? (?P[eEfFgG%]) """, re.DOTALL | re.VERBOSE).fullmatch diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 96b3f30519459b..d1d2739856c092 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -1518,6 +1518,8 @@ def test_format_f_presentation_type(self): (F(51, 1000), '.1f', '0.1'), (F(149, 1000), '.1f', '0.1'), (F(151, 1000), '.1f', '0.2'), + (F(22, 7), '.02f', '3.14'), # issue gh-130662 + (F(22, 7), '005.02f', '03.14'), ] for fraction, spec, expected in testcases: with self.subTest(fraction=fraction, spec=spec): @@ -1616,12 +1618,6 @@ def test_invalid_formats(self): '=010%', '>00.2f', '>00f', - # Too many zeros - minimum width should not have leading zeros - '006f', - # Leading zeros in precision - '.010f', - '.02f', - '.000f', # Missing precision '.e', '.f', diff --git a/Misc/NEWS.d/next/Library/2025-06-02-14-28-30.gh-issue-130662.EIgIR8.rst b/Misc/NEWS.d/next/Library/2025-06-02-14-28-30.gh-issue-130662.EIgIR8.rst new file mode 100644 index 00000000000000..e07200f9a3fbbd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-02-14-28-30.gh-issue-130662.EIgIR8.rst @@ -0,0 +1,3 @@ +Accept leading zeros in precision and width fields for +:class:`~fractions.Fraction` formatting, for example ``format(Fraction(1, +3), '.016f')``. From 9c72658e49806eae52346a0905c1c176d3d49a2f Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Mon, 2 Jun 2025 16:30:52 +0300 Subject: [PATCH 055/116] gh-130662: Accept leading zeros in precision/width for Decimal's formatting (#132549) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/_pydecimal.py | 4 ++-- Lib/test/test_decimal.py | 2 ++ .../Library/2025-06-02-14-36-28.gh-issue-130662.Gpr2GB.rst | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-06-02-14-36-28.gh-issue-130662.Gpr2GB.rst diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 46fa9ffcb1e056..781b38ec26ba33 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -6120,9 +6120,9 @@ def _convert_for_comparison(self, other, equality_op=False): (?Pz)? (?P\#)? (?P0)? -(?P(?!0)\d+)? +(?P\d+)? (?P[,_])? -(?:\.(?P0|(?!0)\d+))? +(?:\.(?P\d+))? (?P[eEfFgGn%])? \z """, re.VERBOSE|re.DOTALL) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index c0a1e378583ba8..ef64b878805d77 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -981,6 +981,7 @@ def test_formatting(self): ('.0f', '0e-2', '0'), ('.0f', '3.14159265', '3'), ('.1f', '3.14159265', '3.1'), + ('.01f', '3.14159265', '3.1'), # leading zero in precision ('.4f', '3.14159265', '3.1416'), ('.6f', '3.14159265', '3.141593'), ('.7f', '3.14159265', '3.1415926'), # round-half-even! @@ -1066,6 +1067,7 @@ def test_formatting(self): ('8,', '123456', ' 123,456'), ('08,', '123456', '0,123,456'), # special case: extra 0 needed ('+08,', '123456', '+123,456'), # but not if there's a sign + ('008,', '123456', '0,123,456'), # leading zero in width (' 08,', '123456', ' 123,456'), ('08,', '-123456', '-123,456'), ('+09,', '123456', '+0,123,456'), diff --git a/Misc/NEWS.d/next/Library/2025-06-02-14-36-28.gh-issue-130662.Gpr2GB.rst b/Misc/NEWS.d/next/Library/2025-06-02-14-36-28.gh-issue-130662.Gpr2GB.rst new file mode 100644 index 00000000000000..d97d937376a9d0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-02-14-36-28.gh-issue-130662.Gpr2GB.rst @@ -0,0 +1,3 @@ ++Accept leading zeros in precision and width fields for ++:class:`~decimal.Decimal` formatting, for example ``format(Decimal(1.25), +'.016f')``. From b6237c3602075294a18dec821773429a51fa7e0d Mon Sep 17 00:00:00 2001 From: Duane Griffin Date: Tue, 3 Jun 2025 02:34:26 +1200 Subject: [PATCH 056/116] gh-117852: fix argument checking of `async_generator.athrow` (#134868) Co-authored-by: Kumar Aditya --- Lib/test/test_asyncgen.py | 9 ++++ ...-05-28-23-58-50.gh-issue-117852.BO9g7z.rst | 1 + Objects/genobject.c | 49 +++++++++++-------- 3 files changed, 38 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-28-23-58-50.gh-issue-117852.BO9g7z.rst diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 2c44647bf3e2f9..636cb33dd9884a 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -2021,6 +2021,15 @@ async def gen(): g.athrow(RuntimeError) gc_collect() + def test_athrow_throws_immediately(self): + async def gen(): + yield 1 + + g = gen() + msg = "athrow expected at least 1 argument, got 0" + with self.assertRaisesRegex(TypeError, msg): + g.athrow() + def test_aclose(self): async def gen(): yield 1 diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-28-23-58-50.gh-issue-117852.BO9g7z.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-28-23-58-50.gh-issue-117852.BO9g7z.rst new file mode 100644 index 00000000000000..fc71cd21a367b2 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-28-23-58-50.gh-issue-117852.BO9g7z.rst @@ -0,0 +1 @@ +Fix argument checking of :meth:`~agen.athrow`. diff --git a/Objects/genobject.c b/Objects/genobject.c index 98b2c5004df8ac..da1462deaaa02c 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1451,7 +1451,9 @@ typedef struct PyAsyncGenAThrow { /* Can be NULL, when in the "aclose()" mode (equivalent of "athrow(GeneratorExit)") */ - PyObject *agt_args; + PyObject *agt_typ; + PyObject *agt_tb; + PyObject *agt_val; AwaitableState agt_state; } PyAsyncGenAThrow; @@ -2078,7 +2080,9 @@ async_gen_athrow_dealloc(PyObject *self) _PyObject_GC_UNTRACK(self); Py_CLEAR(agt->agt_gen); - Py_CLEAR(agt->agt_args); + Py_XDECREF(agt->agt_typ); + Py_XDECREF(agt->agt_tb); + Py_XDECREF(agt->agt_val); PyObject_GC_Del(self); } @@ -2088,7 +2092,9 @@ async_gen_athrow_traverse(PyObject *self, visitproc visit, void *arg) { PyAsyncGenAThrow *agt = _PyAsyncGenAThrow_CAST(self); Py_VISIT(agt->agt_gen); - Py_VISIT(agt->agt_args); + Py_VISIT(agt->agt_typ); + Py_VISIT(agt->agt_tb); + Py_VISIT(agt->agt_val); return 0; } @@ -2116,7 +2122,7 @@ async_gen_athrow_send(PyObject *self, PyObject *arg) if (o->agt_state == AWAITABLE_STATE_INIT) { if (o->agt_gen->ag_running_async) { o->agt_state = AWAITABLE_STATE_CLOSED; - if (o->agt_args == NULL) { + if (o->agt_typ == NULL) { PyErr_SetString( PyExc_RuntimeError, "aclose(): asynchronous generator is already running"); @@ -2143,7 +2149,7 @@ async_gen_athrow_send(PyObject *self, PyObject *arg) o->agt_state = AWAITABLE_STATE_ITER; o->agt_gen->ag_running_async = 1; - if (o->agt_args == NULL) { + if (o->agt_typ == NULL) { /* aclose() mode */ o->agt_gen->ag_closed = 1; @@ -2157,19 +2163,10 @@ async_gen_athrow_send(PyObject *self, PyObject *arg) goto yield_close; } } else { - PyObject *typ; - PyObject *tb = NULL; - PyObject *val = NULL; - - if (!PyArg_UnpackTuple(o->agt_args, "athrow", 1, 3, - &typ, &val, &tb)) { - return NULL; - } - retval = _gen_throw((PyGenObject *)gen, 0, /* Do not close generator when PyExc_GeneratorExit is passed */ - typ, val, tb); + o->agt_typ, o->agt_val, o->agt_tb); retval = async_gen_unwrap_value(o->agt_gen, retval); } if (retval == NULL) { @@ -2181,7 +2178,7 @@ async_gen_athrow_send(PyObject *self, PyObject *arg) assert(o->agt_state == AWAITABLE_STATE_ITER); retval = gen_send((PyObject *)gen, arg); - if (o->agt_args) { + if (o->agt_typ) { return async_gen_unwrap_value(o->agt_gen, retval); } else { /* aclose() mode */ @@ -2212,7 +2209,7 @@ async_gen_athrow_send(PyObject *self, PyObject *arg) if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || PyErr_ExceptionMatches(PyExc_GeneratorExit)) { - if (o->agt_args == NULL) { + if (o->agt_typ == NULL) { /* when aclose() is called we don't want to propagate StopAsyncIteration or GeneratorExit; just raise StopIteration, signalling that this 'aclose()' await @@ -2241,7 +2238,7 @@ async_gen_athrow_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (o->agt_state == AWAITABLE_STATE_INIT) { if (o->agt_gen->ag_running_async) { o->agt_state = AWAITABLE_STATE_CLOSED; - if (o->agt_args == NULL) { + if (o->agt_typ == NULL) { PyErr_SetString( PyExc_RuntimeError, "aclose(): asynchronous generator is already running"); @@ -2259,7 +2256,7 @@ async_gen_athrow_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } PyObject *retval = gen_throw((PyObject*)o->agt_gen, args, nargs); - if (o->agt_args) { + if (o->agt_typ) { retval = async_gen_unwrap_value(o->agt_gen, retval); if (retval == NULL) { o->agt_gen->ag_running_async = 0; @@ -2334,7 +2331,7 @@ async_gen_athrow_finalize(PyObject *op) { PyAsyncGenAThrow *o = (PyAsyncGenAThrow*)op; if (o->agt_state == AWAITABLE_STATE_INIT) { - PyObject *method = o->agt_args ? &_Py_ID(athrow) : &_Py_ID(aclose); + PyObject *method = o->agt_typ ? &_Py_ID(athrow) : &_Py_ID(aclose); _PyErr_WarnUnawaitedAgenMethod(o->agt_gen, method); } } @@ -2403,13 +2400,23 @@ PyTypeObject _PyAsyncGenAThrow_Type = { static PyObject * async_gen_athrow_new(PyAsyncGenObject *gen, PyObject *args) { + PyObject *typ = NULL; + PyObject *tb = NULL; + PyObject *val = NULL; + if (args && !PyArg_UnpackTuple(args, "athrow", 1, 3, &typ, &val, &tb)) { + return NULL; + } + PyAsyncGenAThrow *o; o = PyObject_GC_New(PyAsyncGenAThrow, &_PyAsyncGenAThrow_Type); if (o == NULL) { return NULL; } o->agt_gen = (PyAsyncGenObject*)Py_NewRef(gen); - o->agt_args = Py_XNewRef(args); + o->agt_typ = Py_XNewRef(typ); + o->agt_tb = Py_XNewRef(tb); + o->agt_val = Py_XNewRef(val); + o->agt_state = AWAITABLE_STATE_INIT; _PyObject_GC_TRACK((PyObject*)o); return (PyObject*)o; From 26a1cd4e8c0c9ea1a6683abd82547ddee656ff3d Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Mon, 2 Jun 2025 16:43:32 +0200 Subject: [PATCH 057/116] gh-123471: make concurrent iteration over `itertools.cycle` safe under free-threading (#131212) Co-authored-by: Kumar Aditya --- ...itertools_batched.py => test_itertools.py} | 32 +++++++++++++++++-- ...-03-13-20-48-58.gh-issue-123471.cM4w4f.rst | 1 + Modules/itertoolsmodule.c | 27 +++++++++------- 3 files changed, 46 insertions(+), 14 deletions(-) rename Lib/test/test_free_threading/{test_itertools_batched.py => test_itertools.py} (53%) create mode 100644 Misc/NEWS.d/next/Library/2025-03-13-20-48-58.gh-issue-123471.cM4w4f.rst diff --git a/Lib/test/test_free_threading/test_itertools_batched.py b/Lib/test/test_free_threading/test_itertools.py similarity index 53% rename from Lib/test/test_free_threading/test_itertools_batched.py rename to Lib/test/test_free_threading/test_itertools.py index a754b4f9ea9902..8360afbf78cadd 100644 --- a/Lib/test/test_free_threading/test_itertools_batched.py +++ b/Lib/test/test_free_threading/test_itertools.py @@ -1,15 +1,15 @@ import unittest from threading import Thread, Barrier -from itertools import batched +from itertools import batched, cycle from test.support import threading_helper threading_helper.requires_working_threading(module=True) -class EnumerateThreading(unittest.TestCase): +class ItertoolsThreading(unittest.TestCase): @threading_helper.reap_threads - def test_threading(self): + def test_batched(self): number_of_threads = 10 number_of_iterations = 20 barrier = Barrier(number_of_threads) @@ -34,5 +34,31 @@ def work(it): barrier.reset() + @threading_helper.reap_threads + def test_cycle(self): + number_of_threads = 6 + number_of_iterations = 10 + number_of_cycles = 400 + + barrier = Barrier(number_of_threads) + def work(it): + barrier.wait() + for _ in range(number_of_cycles): + _ = next(it) + + data = (1, 2, 3, 4) + for it in range(number_of_iterations): + cycle_iterator = cycle(data) + worker_threads = [] + for ii in range(number_of_threads): + worker_threads.append( + Thread(target=work, args=[cycle_iterator])) + + with threading_helper.start_threads(worker_threads): + pass + + barrier.reset() + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-03-13-20-48-58.gh-issue-123471.cM4w4f.rst b/Misc/NEWS.d/next/Library/2025-03-13-20-48-58.gh-issue-123471.cM4w4f.rst new file mode 100644 index 00000000000000..cfc783900de70f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-13-20-48-58.gh-issue-123471.cM4w4f.rst @@ -0,0 +1 @@ +Make concurrent iterations over :class:`itertools.cycle` safe under free-threading. diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 943c1e8607b38f..2003546ce84cef 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1124,7 +1124,6 @@ typedef struct { PyObject *it; PyObject *saved; Py_ssize_t index; - int firstpass; } cycleobject; #define cycleobject_CAST(op) ((cycleobject *)(op)) @@ -1165,8 +1164,7 @@ itertools_cycle_impl(PyTypeObject *type, PyObject *iterable) } lz->it = it; lz->saved = saved; - lz->index = 0; - lz->firstpass = 0; + lz->index = -1; return (PyObject *)lz; } @@ -1199,11 +1197,11 @@ cycle_next(PyObject *op) cycleobject *lz = cycleobject_CAST(op); PyObject *item; - if (lz->it != NULL) { + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(lz->index); + + if (index < 0) { item = PyIter_Next(lz->it); if (item != NULL) { - if (lz->firstpass) - return item; if (PyList_Append(lz->saved, item)) { Py_DECREF(item); return NULL; @@ -1213,15 +1211,22 @@ cycle_next(PyObject *op) /* Note: StopIteration is already cleared by PyIter_Next() */ if (PyErr_Occurred()) return NULL; + index = 0; + FT_ATOMIC_STORE_SSIZE_RELAXED(lz->index, 0); +#ifndef Py_GIL_DISABLED Py_CLEAR(lz->it); +#endif } if (PyList_GET_SIZE(lz->saved) == 0) return NULL; - item = PyList_GET_ITEM(lz->saved, lz->index); - lz->index++; - if (lz->index >= PyList_GET_SIZE(lz->saved)) - lz->index = 0; - return Py_NewRef(item); + item = PyList_GetItemRef(lz->saved, index); + assert(item); + index++; + if (index >= PyList_GET_SIZE(lz->saved)) { + index = 0; + } + FT_ATOMIC_STORE_SSIZE_RELAXED(lz->index, index); + return item; } static PyType_Slot cycle_slots[] = { From 055827528fa50c9a7707792f5fe264c4e20c07e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 2 Jun 2025 16:57:08 +0200 Subject: [PATCH 058/116] gh-130999: Fix globals() poisoning in test_traceback (gh-135030) --- Lib/test/test_traceback.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 6b2271f5d5ba8d..74b979d009664d 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -4232,8 +4232,8 @@ def make_module(self, code): return mod_name - def get_import_from_suggestion(self, mod_dict, name): - modname = self.make_module(mod_dict) + def get_import_from_suggestion(self, code, name): + modname = self.make_module(code) def callable(): try: @@ -4416,8 +4416,9 @@ def func(): def test_name_error_suggestions_with_non_string_candidates(self): def func(): abc = 1 - globals()[0] = 1 - abv + custom_globals = globals().copy() + custom_globals[0] = 1 + print(eval("abv", custom_globals, locals())) actual = self.get_suggestion(func) self.assertIn("abc", actual) From 44fb7c361cb24dcf9989a7a1cfee4f6aad5c81aa Mon Sep 17 00:00:00 2001 From: Duane Griffin Date: Tue, 3 Jun 2025 05:22:41 +1200 Subject: [PATCH 059/116] gh-134908: Protect `textiowrapper_iternext` with critical section (gh-134910) The `textiowrapper_iternext` function called `_textiowrapper_writeflush`, but did not use a critical section, making it racy in free-threaded builds. --- Lib/test/test_io.py | 31 +++++++++++++++++++ ...-05-30-15-56-19.gh-issue-134908.3a7PxM.rst | 1 + Modules/_io/textio.c | 15 ++++++++- 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-30-15-56-19.gh-issue-134908.3a7PxM.rst diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 168e66c5a3f0e0..0c921ffbc2576a 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -1062,6 +1062,37 @@ def flush(self): # Silence destructor error R.flush = lambda self: None + @threading_helper.requires_working_threading() + def test_write_readline_races(self): + # gh-134908: Concurrent iteration over a file caused races + thread_count = 2 + write_count = 100 + read_count = 100 + + def writer(file, barrier): + barrier.wait() + for _ in range(write_count): + file.write("x") + + def reader(file, barrier): + barrier.wait() + for _ in range(read_count): + for line in file: + self.assertEqual(line, "") + + with self.open(os_helper.TESTFN, "w+") as f: + barrier = threading.Barrier(thread_count + 1) + reader = threading.Thread(target=reader, args=(f, barrier)) + writers = [threading.Thread(target=writer, args=(f, barrier)) + for _ in range(thread_count)] + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads(writers + [reader]): + pass + self.assertIsNone(cm.exc_type) + + self.assertEqual(os.stat(os_helper.TESTFN).st_size, + write_count * thread_count) + class CIOTest(IOTest): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-30-15-56-19.gh-issue-134908.3a7PxM.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-30-15-56-19.gh-issue-134908.3a7PxM.rst new file mode 100644 index 00000000000000..3178f0aaf885f8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-30-15-56-19.gh-issue-134908.3a7PxM.rst @@ -0,0 +1 @@ +Fix crash when iterating over lines in a text file on the :term:`free threaded ` build. diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 86328e46a7b131..3808ecdceb9b70 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -1578,6 +1578,8 @@ _io_TextIOWrapper_detach_impl(textio *self) static int _textiowrapper_writeflush(textio *self) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + if (self->pending_bytes == NULL) return 0; @@ -3173,8 +3175,9 @@ _io_TextIOWrapper_close_impl(textio *self) } static PyObject * -textiowrapper_iternext(PyObject *op) +textiowrapper_iternext_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); PyObject *line; textio *self = textio_CAST(op); @@ -3210,6 +3213,16 @@ textiowrapper_iternext(PyObject *op) return line; } +static PyObject * +textiowrapper_iternext(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = textiowrapper_iternext_lock_held(op); + Py_END_CRITICAL_SECTION(); + return result; +} + /*[clinic input] @critical_section @getter From 0cec424af5904b3d23ad6e3c6d1a27f89d238d64 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 2 Jun 2025 21:08:26 +0300 Subject: [PATCH 060/116] gh-66234: Add flag to disable the use of mmap in dbm.gnu (GH-135005) This may harm performance, but improve crash tolerance. --- Doc/library/dbm.rst | 3 +++ Doc/whatsnew/3.15.rst | 4 +++ Lib/test/test_dbm_gnu.py | 27 +++++++++++++++++-- ...5-06-01-15-13-07.gh-issue-66234.Jw7OdC.rst | 3 +++ Modules/_gdbmmodule.c | 8 ++++++ 5 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-06-01-15-13-07.gh-issue-66234.Jw7OdC.rst diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 6f548fbb1b39d8..39e287b15214e4 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -254,6 +254,9 @@ functionality like crash tolerance. * ``'s'``: Synchronized mode. Changes to the database will be written immediately to the file. * ``'u'``: Do not lock database. + * ``'m'``: Do not use :manpage:`mmap(2)`. + This may harm performance, but improve crash tolerance. + .. versionadded:: next Not all flags are valid for all versions of GDBM. See the :data:`open_flags` member for a list of supported flag characters. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index a27a17afdba2a8..2fe33c4c535919 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -96,6 +96,10 @@ dbm which allow to recover unused free space previously occupied by deleted entries. (Contributed by Andrea Oliveri in :gh:`134004`.) +* Add the ``'m'`` flag for :func:`dbm.gnu.open` which allows to disable + the use of :manpage:`mmap(2)`. + This may harm performance, but improve crash tolerance. + (Contributed by Serhiy Storchaka in :gh:`66234`.) difflib ------- diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py index 66268c42a300b5..e0b988b7b95bbd 100644 --- a/Lib/test/test_dbm_gnu.py +++ b/Lib/test/test_dbm_gnu.py @@ -74,12 +74,12 @@ def test_flags(self): # Test the flag parameter open() by trying all supported flag modes. all = set(gdbm.open_flags) # Test standard flags (presumably "crwn"). - modes = all - set('fsu') + modes = all - set('fsum') for mode in sorted(modes): # put "c" mode first self.g = gdbm.open(filename, mode) self.g.close() - # Test additional flags (presumably "fsu"). + # Test additional flags (presumably "fsum"). flags = all - set('crwn') for mode in modes: for flag in flags: @@ -217,6 +217,29 @@ def test_localized_error(self): create_empty_file(os.path.join(d, 'test')) self.assertRaises(gdbm.error, gdbm.open, filename, 'r') + @unittest.skipUnless('m' in gdbm.open_flags, "requires 'm' in open_flags") + def test_nommap_no_crash(self): + self.g = g = gdbm.open(filename, 'nm') + os.truncate(filename, 0) + + g.get(b'a', b'c') + g.keys() + g.firstkey() + g.nextkey(b'a') + with self.assertRaises(KeyError): + g[b'a'] + with self.assertRaises(gdbm.error): + len(g) + + with self.assertRaises(gdbm.error): + g[b'a'] = b'c' + with self.assertRaises(gdbm.error): + del g[b'a'] + with self.assertRaises(gdbm.error): + g.setdefault(b'a', b'c') + with self.assertRaises(gdbm.error): + g.reorganize() + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-06-01-15-13-07.gh-issue-66234.Jw7OdC.rst b/Misc/NEWS.d/next/Library/2025-06-01-15-13-07.gh-issue-66234.Jw7OdC.rst new file mode 100644 index 00000000000000..1defb9a72e04e7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-01-15-13-07.gh-issue-66234.Jw7OdC.rst @@ -0,0 +1,3 @@ +Add the ``'m'`` flag for :func:`dbm.gnu.open` which allows to disable the +use of :manpage:`mmap(2)`. This may harm performance, but improve crash +tolerance. diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index 9c402e20e513b9..6a4939512b22fc 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -813,6 +813,11 @@ dbmopen_impl(PyObject *module, PyObject *filename, const char *flags, case 'u': iflags |= GDBM_NOLOCK; break; +#endif +#ifdef GDBM_NOMMAP + case 'm': + iflags |= GDBM_NOMMAP; + break; #endif default: PyErr_Format(state->gdbm_error, @@ -846,6 +851,9 @@ static const char gdbmmodule_open_flags[] = "rwcn" #endif #ifdef GDBM_NOLOCK "u" +#endif +#ifdef GDBM_NOMMAP + "m" #endif ; From c7051a366996dd2213decb8677c52d1eda0fde6a Mon Sep 17 00:00:00 2001 From: Rafael Fontenelle Date: Mon, 2 Jun 2025 15:13:08 -0300 Subject: [PATCH 061/116] Remove newline in Doc/c-api/lifecycle.rst for gettext builder (GH-135013) --- Doc/c-api/lifecycle.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Doc/c-api/lifecycle.rst b/Doc/c-api/lifecycle.rst index 0e2ffc096caba7..5a170862a26f44 100644 --- a/Doc/c-api/lifecycle.rst +++ b/Doc/c-api/lifecycle.rst @@ -55,16 +55,14 @@ that must be true for *B* to occur after *A*. .. image:: lifecycle.dot.svg :align: center :class: invert-in-dark-mode - :alt: Diagram showing events in an object's life. Explained in detail - below. + :alt: Diagram showing events in an object's life. Explained in detail below. .. only:: latex .. image:: lifecycle.dot.pdf :align: center :class: invert-in-dark-mode - :alt: Diagram showing events in an object's life. Explained in detail - below. + :alt: Diagram showing events in an object's life. Explained in detail below. .. container:: :name: life-events-graph-description From 7a79f52d83c22f5a9787e590f267325c1175d389 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 2 Jun 2025 23:25:32 +0300 Subject: [PATCH 062/116] gh-133454: Mark test_queue tests with many threads as bigmem (gh-134575) 50 producer and 50 consumer threads need more than 5GB of memory. --- Lib/test/test_queue.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 7f4fe357034b71..c855fb8fe2b05a 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -6,7 +6,7 @@ import time import unittest import weakref -from test.support import gc_collect +from test.support import gc_collect, bigmemtest from test.support import import_helper from test.support import threading_helper @@ -963,33 +963,33 @@ def test_order(self): # One producer, one consumer => results appended in well-defined order self.assertEqual(results, inputs) - def test_many_threads(self): + @bigmemtest(size=50, memuse=100*2**20, dry_run=False) + def test_many_threads(self, size): # Test multiple concurrent put() and get() - N = 50 q = self.q inputs = list(range(10000)) - results = self.run_threads(N, q, inputs, self.feed, self.consume) + results = self.run_threads(size, q, inputs, self.feed, self.consume) # Multiple consumers without synchronization append the # results in random order self.assertEqual(sorted(results), inputs) - def test_many_threads_nonblock(self): + @bigmemtest(size=50, memuse=100*2**20, dry_run=False) + def test_many_threads_nonblock(self, size): # Test multiple concurrent put() and get(block=False) - N = 50 q = self.q inputs = list(range(10000)) - results = self.run_threads(N, q, inputs, + results = self.run_threads(size, q, inputs, self.feed, self.consume_nonblock) self.assertEqual(sorted(results), inputs) - def test_many_threads_timeout(self): + @bigmemtest(size=50, memuse=100*2**20, dry_run=False) + def test_many_threads_timeout(self, size): # Test multiple concurrent put() and get(timeout=...) - N = 50 q = self.q inputs = list(range(1000)) - results = self.run_threads(N, q, inputs, + results = self.run_threads(size, q, inputs, self.feed, self.consume_timeout) self.assertEqual(sorted(results), inputs) From e814f43f2c655b931af8ee9e1c128bd1027549fb Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 2 Jun 2025 23:31:06 +0300 Subject: [PATCH 063/116] gh-74232: Add a note about roundtrip of non-float numerics in CSV (GH-134963) --- Doc/library/csv.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 533cdf13974be6..5297be17bd708e 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -70,7 +70,7 @@ The :mod:`csv` module defines the following functions: section :ref:`csv-fmt-params`. Each row read from the csv file is returned as a list of strings. No - automatic data type conversion is performed unless the ``QUOTE_NONNUMERIC`` format + automatic data type conversion is performed unless the :data:`QUOTE_NONNUMERIC` format option is specified (in which case unquoted fields are transformed into floats). A short usage example:: @@ -331,8 +331,14 @@ The :mod:`csv` module defines the following constants: Instructs :class:`writer` objects to quote all non-numeric fields. - Instructs :class:`reader` objects to convert all non-quoted fields to type *float*. + Instructs :class:`reader` objects to convert all non-quoted fields to type :class:`float`. + .. note:: + Some numeric types, such as :class:`bool`, :class:`~fractions.Fraction`, + or :class:`~enum.IntEnum`, have a string representation that cannot be + converted to :class:`float`. + They cannot be read in the :data:`QUOTE_NONNUMERIC` and + :data:`QUOTE_STRINGS` modes. .. data:: QUOTE_NONE From df98a47a61a274eb7427c6201ddabec9ffd30b0a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 2 Jun 2025 23:35:41 +0300 Subject: [PATCH 064/116] gh-132813: Improve error messages for incorrect types and values of csv.Dialog attributes (GH-133241) Make them similar to PyArg_Parse error messages, mention None as a possible value, show a wrong type and the string length. --- Lib/test/test_csv.py | 48 ++++++++---- ...-05-01-10-56-44.gh-issue-132813.rKurvp.rst | 2 + Modules/_csv.c | 73 ++++++++----------- 3 files changed, 69 insertions(+), 54 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-01-10-56-44.gh-issue-132813.rKurvp.rst diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 9aace57633b0c6..60feab225a107c 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1122,19 +1122,22 @@ class mydialect(csv.Dialect): with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"quotechar" must be a 1-character string') + '"quotechar" must be a unicode character or None, ' + 'not a string of length 0') mydialect.quotechar = "''" with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"quotechar" must be a 1-character string') + '"quotechar" must be a unicode character or None, ' + 'not a string of length 2') mydialect.quotechar = 4 with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"quotechar" must be string or None, not int') + '"quotechar" must be a unicode character or None, ' + 'not int') def test_delimiter(self): class mydialect(csv.Dialect): @@ -1151,31 +1154,32 @@ class mydialect(csv.Dialect): with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be a 1-character string') + '"delimiter" must be a unicode character, ' + 'not a string of length 3') mydialect.delimiter = "" with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be a 1-character string') + '"delimiter" must be a unicode character, not a string of length 0') mydialect.delimiter = b"," with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be string, not bytes') + '"delimiter" must be a unicode character, not bytes') mydialect.delimiter = 4 with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be string, not int') + '"delimiter" must be a unicode character, not int') mydialect.delimiter = None with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be string, not NoneType') + '"delimiter" must be a unicode character, not NoneType') def test_escapechar(self): class mydialect(csv.Dialect): @@ -1189,20 +1193,32 @@ class mydialect(csv.Dialect): self.assertEqual(d.escapechar, "\\") mydialect.escapechar = "" - with self.assertRaisesRegex(csv.Error, '"escapechar" must be a 1-character string'): + with self.assertRaises(csv.Error) as cm: mydialect() + self.assertEqual(str(cm.exception), + '"escapechar" must be a unicode character or None, ' + 'not a string of length 0') mydialect.escapechar = "**" - with self.assertRaisesRegex(csv.Error, '"escapechar" must be a 1-character string'): + with self.assertRaises(csv.Error) as cm: mydialect() + self.assertEqual(str(cm.exception), + '"escapechar" must be a unicode character or None, ' + 'not a string of length 2') mydialect.escapechar = b"*" - with self.assertRaisesRegex(csv.Error, '"escapechar" must be string or None, not bytes'): + with self.assertRaises(csv.Error) as cm: mydialect() + self.assertEqual(str(cm.exception), + '"escapechar" must be a unicode character or None, ' + 'not bytes') mydialect.escapechar = 4 - with self.assertRaisesRegex(csv.Error, '"escapechar" must be string or None, not int'): + with self.assertRaises(csv.Error) as cm: mydialect() + self.assertEqual(str(cm.exception), + '"escapechar" must be a unicode character or None, ' + 'not int') def test_lineterminator(self): class mydialect(csv.Dialect): @@ -1223,7 +1239,13 @@ class mydialect(csv.Dialect): with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"lineterminator" must be a string') + '"lineterminator" must be a string, not int') + + mydialect.lineterminator = None + with self.assertRaises(csv.Error) as cm: + mydialect() + self.assertEqual(str(cm.exception), + '"lineterminator" must be a string, not NoneType') def test_invalid_chars(self): def create_invalid(field_name, value, **kwargs): diff --git a/Misc/NEWS.d/next/Library/2025-05-01-10-56-44.gh-issue-132813.rKurvp.rst b/Misc/NEWS.d/next/Library/2025-05-01-10-56-44.gh-issue-132813.rKurvp.rst new file mode 100644 index 00000000000000..55608528a4564b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-01-10-56-44.gh-issue-132813.rKurvp.rst @@ -0,0 +1,2 @@ +Improve error messages for incorrect types and values of :class:`csv.Dialect` +attributes. diff --git a/Modules/_csv.c b/Modules/_csv.c index e5ae853590bf2c..2e04136e0ac657 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -237,7 +237,7 @@ _set_int(const char *name, int *target, PyObject *src, int dflt) int value; if (!PyLong_CheckExact(src)) { PyErr_Format(PyExc_TypeError, - "\"%s\" must be an integer", name); + "\"%s\" must be an integer, not %T", name, src); return -1; } value = PyLong_AsInt(src); @@ -255,27 +255,29 @@ _set_char_or_none(const char *name, Py_UCS4 *target, PyObject *src, Py_UCS4 dflt if (src == NULL) { *target = dflt; } - else { + else if (src == Py_None) { *target = NOT_SET; - if (src != Py_None) { - if (!PyUnicode_Check(src)) { - PyErr_Format(PyExc_TypeError, - "\"%s\" must be string or None, not %.200s", name, - Py_TYPE(src)->tp_name); - return -1; - } - Py_ssize_t len = PyUnicode_GetLength(src); - if (len < 0) { - return -1; - } - if (len != 1) { - PyErr_Format(PyExc_TypeError, - "\"%s\" must be a 1-character string", - name); - return -1; - } - *target = PyUnicode_READ_CHAR(src, 0); + } + else { + // similar to PyArg_Parse("C?") + if (!PyUnicode_Check(src)) { + PyErr_Format(PyExc_TypeError, + "\"%s\" must be a unicode character or None, not %T", + name, src); + return -1; + } + Py_ssize_t len = PyUnicode_GetLength(src); + if (len < 0) { + return -1; } + if (len != 1) { + PyErr_Format(PyExc_TypeError, + "\"%s\" must be a unicode character or None, " + "not a string of length %zd", + name, len); + return -1; + } + *target = PyUnicode_READ_CHAR(src, 0); } return 0; } @@ -287,11 +289,12 @@ _set_char(const char *name, Py_UCS4 *target, PyObject *src, Py_UCS4 dflt) *target = dflt; } else { + // similar to PyArg_Parse("C") if (!PyUnicode_Check(src)) { PyErr_Format(PyExc_TypeError, - "\"%s\" must be string, not %.200s", name, - Py_TYPE(src)->tp_name); - return -1; + "\"%s\" must be a unicode character, not %T", + name, src); + return -1; } Py_ssize_t len = PyUnicode_GetLength(src); if (len < 0) { @@ -299,8 +302,9 @@ _set_char(const char *name, Py_UCS4 *target, PyObject *src, Py_UCS4 dflt) } if (len != 1) { PyErr_Format(PyExc_TypeError, - "\"%s\" must be a 1-character string", - name); + "\"%s\" must be a unicode character, " + "not a string of length %zd", + name, len); return -1; } *target = PyUnicode_READ_CHAR(src, 0); @@ -314,16 +318,12 @@ _set_str(const char *name, PyObject **target, PyObject *src, const char *dflt) if (src == NULL) *target = PyUnicode_DecodeASCII(dflt, strlen(dflt), NULL); else { - if (src == Py_None) - *target = NULL; - else if (!PyUnicode_Check(src)) { + if (!PyUnicode_Check(src)) { PyErr_Format(PyExc_TypeError, - "\"%s\" must be a string", name); + "\"%s\" must be a string, not %T", name, src); return -1; } - else { - Py_XSETREF(*target, Py_NewRef(src)); - } + Py_XSETREF(*target, Py_NewRef(src)); } return 0; } @@ -533,11 +533,6 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) /* validate options */ if (dialect_check_quoting(self->quoting)) goto err; - if (self->delimiter == NOT_SET) { - PyErr_SetString(PyExc_TypeError, - "\"delimiter\" must be a 1-character string"); - goto err; - } if (quotechar == Py_None && quoting == NULL) self->quoting = QUOTE_NONE; if (self->quoting != QUOTE_NONE && self->quotechar == NOT_SET) { @@ -545,10 +540,6 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) "quotechar must be set if quoting enabled"); goto err; } - if (self->lineterminator == NULL) { - PyErr_SetString(PyExc_TypeError, "lineterminator must be set"); - goto err; - } if (dialect_check_char("delimiter", self->delimiter, self, true) || dialect_check_char("escapechar", self->escapechar, self, !self->skipinitialspace) || From 0ac9e17fb47075c9446b99da4dffe4cad993b97a Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 2 Jun 2025 23:08:20 +0100 Subject: [PATCH 065/116] gh-134830: Fix reference in `Doc/extending/windows.rst` (GH-134831) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Miro Hrončok --- Doc/c-api/stable.rst | 1 + Doc/extending/windows.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 124e58cf950b7a..9b65e0b8d23d93 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -51,6 +51,7 @@ It is generally intended for specialized, low-level tools like debuggers. Projects that use this API are expected to follow CPython development and spend extra effort adjusting to changes. +.. _stable-application-binary-interface: Stable Application Binary Interface =================================== diff --git a/Doc/extending/windows.rst b/Doc/extending/windows.rst index 56aa44e4e58c83..a97c6182553c30 100644 --- a/Doc/extending/windows.rst +++ b/Doc/extending/windows.rst @@ -121,7 +121,7 @@ When creating DLLs in Windows, you can use the CPython library in two ways: :file:`Python.h` triggers an implicit, configure-aware link with the library. The header file chooses :file:`pythonXY_d.lib` for Debug, :file:`pythonXY.lib` for Release, and :file:`pythonX.lib` for Release with - the `Limited API `_ enabled. + the :ref:`Limited API ` enabled. To build two DLLs, spam and ni (which uses C functions found in spam), you could use these commands:: From b525e31b7fc50e7a498f8b9b16437cb7b9656f6f Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 3 Jun 2025 08:40:40 +0900 Subject: [PATCH 066/116] gh-134875: Fix mimallc build error for the old compilers (gh-134994) --- Include/internal/mimalloc/mimalloc/internal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/internal/mimalloc/mimalloc/internal.h b/Include/internal/mimalloc/mimalloc/internal.h index 71b7ea702d6c5e..a7daa3a40a4c0b 100644 --- a/Include/internal/mimalloc/mimalloc/internal.h +++ b/Include/internal/mimalloc/mimalloc/internal.h @@ -634,10 +634,10 @@ static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* bl mi_track_mem_defined(block,sizeof(mi_block_t)); mi_block_t* next; #ifdef MI_ENCODE_FREELIST - next = (mi_block_t*)mi_ptr_decode(null, mi_atomic_load_relaxed(&block->next), keys); + next = (mi_block_t*)mi_ptr_decode(null, mi_atomic_load_relaxed((_Atomic(mi_encoded_t)*)&block->next), keys); #else MI_UNUSED(keys); MI_UNUSED(null); - next = (mi_block_t*)mi_atomic_load_relaxed(&block->next); + next = (mi_block_t*)mi_atomic_load_relaxed((_Atomic(mi_encoded_t)*)&block->next); #endif mi_track_mem_noaccess(block,sizeof(mi_block_t)); return next; From 6e80f11eb5eba360334b4ace105eb7d73394baf7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 3 Jun 2025 08:40:45 +0200 Subject: [PATCH 067/116] gh-135028: Increase parser MAXSTACK for nested parenthesis (#135031) --- Lib/test/test_grammar.py | 14 +++++++++++++- Parser/parser.c | 2 +- Tools/peg_generator/pegen/c_generator.py | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index c39565144bf7f4..7f5d48b9c63ab7 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -1,7 +1,7 @@ # Python test set -- part 1, grammar. # This just tests whether the parser accepts them all. -from test.support import check_syntax_error +from test.support import check_syntax_error, skip_wasi_stack_overflow from test.support import import_helper import annotationlib import inspect @@ -249,6 +249,18 @@ def test_eof_error(self): compile(s, "", "exec") self.assertIn("was never closed", str(cm.exception)) + @skip_wasi_stack_overflow() + def test_max_level(self): + # Macro defined in Parser/lexer/state.h + MAXLEVEL = 200 + + result = eval("(" * MAXLEVEL + ")" * MAXLEVEL) + self.assertEqual(result, ()) + + with self.assertRaises(SyntaxError) as cm: + eval("(" * (MAXLEVEL + 1) + ")" * (MAXLEVEL + 1)) + self.assertStartsWith(str(cm.exception), 'too many nested parentheses') + var_annot_global: int # a global annotated is necessary for test_var_annot diff --git a/Parser/parser.c b/Parser/parser.c index 84a293cddffde6..d5aafef826ed3a 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -14,7 +14,7 @@ # define MAXSTACK 4000 # endif #else -# define MAXSTACK 4000 +# define MAXSTACK 6000 #endif static const int n_keyword_lists = 9; static KeywordToken *reserved_keywords[] = { diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index 2be85a163b4043..09c5651f24a3bb 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -44,7 +44,7 @@ # define MAXSTACK 4000 # endif #else -# define MAXSTACK 4000 +# define MAXSTACK 6000 #endif """ From 485b499610eefe362faf171f258b3a3588378ff1 Mon Sep 17 00:00:00 2001 From: stratakis Date: Tue, 3 Jun 2025 09:09:43 +0200 Subject: [PATCH 068/116] gh-128605: Add branch protections for x86_64 in asm_trampoline.S (#128606) Apply Intel Control-flow Technology for x86-64 on asm_trampoline.S. Required for mitigation against return-oriented programming (ROP) and Call or Jump Oriented Programming (COP/JOP) attacks. Manual application is required for the assembly files. See also: https://sourceware.org/annobin/annobin.html/Test-cf-protection.html --- Python/asm_trampoline.S | 22 ++++++++++++++++++++++ Python/perf_jit_trampoline.c | 4 ++++ 2 files changed, 26 insertions(+) diff --git a/Python/asm_trampoline.S b/Python/asm_trampoline.S index 0a3265dfeee204..616752459ba4d9 100644 --- a/Python/asm_trampoline.S +++ b/Python/asm_trampoline.S @@ -9,6 +9,9 @@ # } _Py_trampoline_func_start: #ifdef __x86_64__ +#if defined(__CET__) && (__CET__ & 1) + endbr64 +#endif sub $8, %rsp call *%rcx add $8, %rsp @@ -34,3 +37,22 @@ _Py_trampoline_func_start: .globl _Py_trampoline_func_end _Py_trampoline_func_end: .section .note.GNU-stack,"",@progbits +# Note for indicating the assembly code supports CET +#if defined(__x86_64__) && defined(__CET__) && (__CET__ & 1) + .section .note.gnu.property,"a" + .align 8 + .long 1f - 0f + .long 4f - 1f + .long 5 +0: + .string "GNU" +1: + .align 8 + .long 0xc0000002 + .long 3f - 2f +2: + .long 0x3 +3: + .align 8 +4: +#endif // __x86_64__ diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 5c7cb5b0a9913c..2ca18c23593547 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -869,7 +869,11 @@ static void elf_init_ehframe(ELFObjectContext* ctx) { */ #ifdef __x86_64__ /* x86_64 calling convention unwinding rules */ +# if defined(__CET__) && (__CET__ & 1) + DWRF_U8(DWRF_CFA_advance_loc | 8); // Advance location by 8 bytes when CET protection is enabled +# else DWRF_U8(DWRF_CFA_advance_loc | 4); // Advance location by 4 bytes +# endif DWRF_U8(DWRF_CFA_def_cfa_offset); // Redefine CFA offset DWRF_UV(16); // New offset: SP + 16 DWRF_U8(DWRF_CFA_advance_loc | 6); // Advance location by 6 bytes From ec12559ebafca01ded22c9013de64abe535c838d Mon Sep 17 00:00:00 2001 From: Roei Ben Artzi <155478676+roeibenartzi@users.noreply.github.com> Date: Tue, 3 Jun 2025 10:40:25 +0300 Subject: [PATCH 069/116] gh-131884: Fix incorrect formatting in json.dumps() when using indent and skipkeys=True (GH-132200) --- Lib/json/encoder.py | 5 +++-- Lib/test/test_json/test_dump.py | 8 ++++++++ .../2025-04-07-06-41-54.gh-issue-131884.ym9BJN.rst | 1 + Modules/_json.c | 13 ++++++++----- 4 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-04-07-06-41-54.gh-issue-131884.ym9BJN.rst diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 016638549aa59b..bc446e0f377a11 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -348,7 +348,6 @@ def _iterencode_dict(dct, _current_indent_level): _current_indent_level += 1 newline_indent = '\n' + _indent * _current_indent_level item_separator = _item_separator + newline_indent - yield newline_indent else: newline_indent = None item_separator = _item_separator @@ -381,6 +380,8 @@ def _iterencode_dict(dct, _current_indent_level): f'not {key.__class__.__name__}') if first: first = False + if newline_indent is not None: + yield newline_indent else: yield item_separator yield _encoder(key) @@ -413,7 +414,7 @@ def _iterencode_dict(dct, _current_indent_level): except BaseException as exc: exc.add_note(f'when serializing {type(dct).__name__} item {key!r}') raise - if newline_indent is not None: + if not first and newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level yield '}' diff --git a/Lib/test/test_json/test_dump.py b/Lib/test/test_json/test_dump.py index 13b40020781bae..39470754003bb6 100644 --- a/Lib/test/test_json/test_dump.py +++ b/Lib/test/test_json/test_dump.py @@ -22,6 +22,14 @@ def test_dump_skipkeys(self): self.assertIn('valid_key', o) self.assertNotIn(b'invalid_key', o) + def test_dump_skipkeys_indent_empty(self): + v = {b'invalid_key': False} + self.assertEqual(self.json.dumps(v, skipkeys=True, indent=4), '{}') + + def test_skipkeys_indent(self): + v = {b'invalid_key': False, 'valid_key': True} + self.assertEqual(self.json.dumps(v, skipkeys=True, indent=4), '{\n "valid_key": true\n}') + def test_encode_truefalse(self): self.assertEqual(self.dumps( {True: False, False: True}, sort_keys=True), diff --git a/Misc/NEWS.d/next/Library/2025-04-07-06-41-54.gh-issue-131884.ym9BJN.rst b/Misc/NEWS.d/next/Library/2025-04-07-06-41-54.gh-issue-131884.ym9BJN.rst new file mode 100644 index 00000000000000..d9e2eae02dce1f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-07-06-41-54.gh-issue-131884.ym9BJN.rst @@ -0,0 +1 @@ +Fix formatting issues in :func:`json.dump` when both *indent* and *skipkeys* are used. diff --git a/Modules/_json.c b/Modules/_json.c index 57678ad595f928..6b5f6ea42df4d1 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1603,6 +1603,12 @@ encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *firs if (*first) { *first = false; + if (s->indent != Py_None) { + if (write_newline_indent(writer, indent_level, indent_cache) < 0) { + Py_DECREF(keystr); + return -1; + } + } } else { if (PyUnicodeWriter_WriteStr(writer, item_separator) < 0) { @@ -1670,11 +1676,8 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, if (s->indent != Py_None) { indent_level++; separator = get_item_separator(s, indent_level, indent_cache); - if (separator == NULL || - write_newline_indent(writer, indent_level, indent_cache) < 0) - { + if (separator == NULL) goto bail; - } } if (s->sort_keys || !PyDict_CheckExact(dct)) { @@ -1714,7 +1717,7 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, goto bail; Py_CLEAR(ident); } - if (s->indent != Py_None) { + if (s->indent != Py_None && !first) { indent_level--; if (write_newline_indent(writer, indent_level, indent_cache) < 0) { goto bail; From 3612d8f51741b11f36f8fb0494d79086bac9390a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 3 Jun 2025 12:42:11 +0200 Subject: [PATCH 070/116] gh-135034: Normalize link targets in tarfile, add `os.path.realpath(strict='allow_missing')` (#135037) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses CVEs 2024-12718, 2025-4138, 2025-4330, and 2025-4517. Signed-off-by: Łukasz Langa Co-authored-by: Petr Viktorin Co-authored-by: Seth Michael Larson Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Serhiy Storchaka --- Doc/library/os.path.rst | 32 +- Doc/library/tarfile.rst | 20 ++ Doc/whatsnew/3.15.rst | 32 ++ Lib/genericpath.py | 11 +- Lib/ntpath.py | 38 ++- Lib/posixpath.py | 57 ++-- Lib/tarfile.py | 163 +++++++-- Lib/test/test_ntpath.py | 214 ++++++++++-- Lib/test/test_posixpath.py | 252 +++++++++++--- Lib/test/test_tarfile.py | 310 +++++++++++++++++- ...-06-02-11-32-23.gh-issue-135034.RLGjbp.rst | 6 + 11 files changed, 966 insertions(+), 169 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-06-02-11-32-23.gh-issue-135034.RLGjbp.rst diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index ecbbc1d7605f9f..f72aee19d8f332 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -408,9 +408,26 @@ the :mod:`glob` module.) system). On Windows, this function will also resolve MS-DOS (also called 8.3) style names such as ``C:\\PROGRA~1`` to ``C:\\Program Files``. - If a path doesn't exist or a symlink loop is encountered, and *strict* is - ``True``, :exc:`OSError` is raised. If *strict* is ``False`` these errors - are ignored, and so the result might be missing or otherwise inaccessible. + By default, the path is evaluated up to the first component that does not + exist, is a symlink loop, or whose evaluation raises :exc:`OSError`. + All such components are appended unchanged to the existing part of the path. + + Some errors that are handled this way include "access denied", "not a + directory", or "bad argument to internal function". Thus, the + resulting path may be missing or inaccessible, may still contain + links or loops, and may traverse non-directories. + + This behavior can be modified by keyword arguments: + + If *strict* is ``True``, the first error encountered when evaluating the path is + re-raised. + In particular, :exc:`FileNotFoundError` is raised if *path* does not exist, + or another :exc:`OSError` if it is otherwise inaccessible. + + If *strict* is :py:data:`os.path.ALLOW_MISSING`, errors other than + :exc:`FileNotFoundError` are re-raised (as with ``strict=True``). + Thus, the returned path will not contain any symbolic links, but the named + file and some of its parent directories may be missing. .. note:: This function emulates the operating system's procedure for making a path @@ -429,6 +446,15 @@ the :mod:`glob` module.) .. versionchanged:: 3.10 The *strict* parameter was added. + .. versionchanged:: next + The :py:data:`~os.path.ALLOW_MISSING` value for the *strict* parameter + was added. + +.. data:: ALLOW_MISSING + + Special value used for the *strict* argument in :func:`realpath`. + + .. versionadded:: next .. function:: relpath(path, start=os.curdir) diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index f9cb5495e60cd2..7cec108a5bd41d 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -255,6 +255,15 @@ The :mod:`tarfile` module defines the following exceptions: Raised to refuse extracting a symbolic link pointing outside the destination directory. +.. exception:: LinkFallbackError + + Raised to refuse emulating a link (hard or symbolic) by extracting another + archive member, when that member would be rejected by the filter location. + The exception that was raised to reject the replacement member is available + as :attr:`!BaseException.__context__`. + + .. versionadded:: next + The following constants are available at the module level: @@ -1068,6 +1077,12 @@ reused in custom filters: Implements the ``'data'`` filter. In addition to what ``tar_filter`` does: + - Normalize link targets (:attr:`TarInfo.linkname`) using + :func:`os.path.normpath`. + Note that this removes internal ``..`` components, which may change the + meaning of the link if the path in :attr:`!TarInfo.linkname` traverses + symbolic links. + - :ref:`Refuse ` to extract links (hard or soft) that link to absolute paths, or ones that link outside the destination. @@ -1099,6 +1114,10 @@ reused in custom filters: Note that this filter does not block *all* dangerous archive features. See :ref:`tarfile-further-verification` for details. + .. versionchanged:: next + + Link targets are now normalized. + .. _tarfile-extraction-refuse: @@ -1127,6 +1146,7 @@ Here is an incomplete list of things to consider: * Extract to a :func:`new temporary directory ` to prevent e.g. exploiting pre-existing links, and to make it easier to clean up after a failed extraction. +* Disallow symbolic links if you do not need the functionality. * When working with untrusted data, use external (e.g. OS-level) limits on disk, memory and CPU usage. * Check filenames against an allow-list of characters diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 2fe33c4c535919..daf3e8fb6c2c2b 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -116,6 +116,16 @@ math (Contributed by Sergey B Kirpichev in :gh:`132908`.) +os.path +------- + +* The *strict* parameter to :func:`os.path.realpath` accepts a new value, + :data:`os.path.ALLOW_MISSING`. + If used, errors other than :exc:`FileNotFoundError` will be re-raised; + the resulting path can be missing but it will be free of symlinks. + (Contributed by Petr Viktorin for :cve:`2025-4517`.) + + shelve ------ @@ -132,6 +142,28 @@ ssl (Contributed by Will Childs-Klein in :gh:`133624`.) +tarfile +------- + +* :func:`~tarfile.data_filter` now normalizes symbolic link targets in order to + avoid path traversal attacks. + (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2025-4138`.) +* :func:`~tarfile.TarFile.extractall` now skips fixing up directory attributes + when a directory was removed or replaced by another kind of file. + (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2024-12718`.) +* :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` + now (re-)apply the extraction filter when substituting a link (hard or + symbolic) with a copy of another archive member, and when fixing up + directory attributes. + The former raises a new exception, :exc:`~tarfile.LinkFallbackError`. + (Contributed by Petr Viktorin for :cve:`2025-4330` and :cve:`2024-12718`.) +* :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` + no longer extract rejected members when + :func:`~tarfile.TarFile.errorlevel` is zero. + (Contributed by Matt Prodani and Petr Viktorin in :gh:`112887` + and :cve:`2025-4435`.) + + zlib ---- diff --git a/Lib/genericpath.py b/Lib/genericpath.py index ba7b0a13c7f81d..9363f564aab7a6 100644 --- a/Lib/genericpath.py +++ b/Lib/genericpath.py @@ -8,7 +8,7 @@ __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', 'getsize', 'isdevdrive', 'isdir', 'isfile', 'isjunction', 'islink', - 'lexists', 'samefile', 'sameopenfile', 'samestat'] + 'lexists', 'samefile', 'sameopenfile', 'samestat', 'ALLOW_MISSING'] # Does a path exist? @@ -189,3 +189,12 @@ def _check_arg_types(funcname, *args): f'os.PathLike object, not {s.__class__.__name__!r}') from None if hasstr and hasbytes: raise TypeError("Can't mix strings and bytes in path components") from None + +# A singleton with a true boolean value. +@object.__new__ +class ALLOW_MISSING: + """Special value for use in realpath().""" + def __repr__(self): + return 'os.path.ALLOW_MISSING' + def __reduce__(self): + return self.__class__.__name__ diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 52ff2af743af6c..9cdc16480f9afe 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -29,7 +29,7 @@ "abspath","curdir","pardir","sep","pathsep","defpath","altsep", "extsep","devnull","realpath","supports_unicode_filenames","relpath", "samefile", "sameopenfile", "samestat", "commonpath", "isjunction", - "isdevdrive"] + "isdevdrive", "ALLOW_MISSING"] def _get_bothseps(path): if isinstance(path, bytes): @@ -601,9 +601,10 @@ def abspath(path): from nt import _findfirstfile, _getfinalpathname, readlink as _nt_readlink except ImportError: # realpath is a no-op on systems without _getfinalpathname support. - realpath = abspath + def realpath(path, *, strict=False): + return abspath(path) else: - def _readlink_deep(path): + def _readlink_deep(path, ignored_error=OSError): # These error codes indicate that we should stop reading links and # return the path we currently have. # 1: ERROR_INVALID_FUNCTION @@ -636,7 +637,7 @@ def _readlink_deep(path): path = old_path break path = normpath(join(dirname(old_path), path)) - except OSError as ex: + except ignored_error as ex: if ex.winerror in allowed_winerror: break raise @@ -645,7 +646,7 @@ def _readlink_deep(path): break return path - def _getfinalpathname_nonstrict(path): + def _getfinalpathname_nonstrict(path, ignored_error=OSError): # These error codes indicate that we should stop resolving the path # and return the value we currently have. # 1: ERROR_INVALID_FUNCTION @@ -673,17 +674,18 @@ def _getfinalpathname_nonstrict(path): try: path = _getfinalpathname(path) return join(path, tail) if tail else path - except OSError as ex: + except ignored_error as ex: if ex.winerror not in allowed_winerror: raise try: # The OS could not resolve this path fully, so we attempt # to follow the link ourselves. If we succeed, join the tail # and return. - new_path = _readlink_deep(path) + new_path = _readlink_deep(path, + ignored_error=ignored_error) if new_path != path: return join(new_path, tail) if tail else new_path - except OSError: + except ignored_error: # If we fail to readlink(), let's keep traversing pass # If we get these errors, try to get the real name of the file without accessing it. @@ -691,7 +693,7 @@ def _getfinalpathname_nonstrict(path): try: name = _findfirstfile(path) path, _ = split(path) - except OSError: + except ignored_error: path, name = split(path) else: path, name = split(path) @@ -721,6 +723,15 @@ def realpath(path, *, strict=False): if normcase(path) == devnull: return '\\\\.\\NUL' had_prefix = path.startswith(prefix) + + if strict is ALLOW_MISSING: + ignored_error = FileNotFoundError + strict = True + elif strict: + ignored_error = () + else: + ignored_error = OSError + if not had_prefix and not isabs(path): path = join(cwd, path) try: @@ -728,17 +739,16 @@ def realpath(path, *, strict=False): initial_winerror = 0 except ValueError as ex: # gh-106242: Raised for embedded null characters - # In strict mode, we convert into an OSError. + # In strict modes, we convert into an OSError. # Non-strict mode returns the path as-is, since we've already # made it absolute. if strict: raise OSError(str(ex)) from None path = normpath(path) - except OSError as ex: - if strict: - raise + except ignored_error as ex: initial_winerror = ex.winerror - path = _getfinalpathname_nonstrict(path) + path = _getfinalpathname_nonstrict(path, + ignored_error=ignored_error) # The path returned by _getfinalpathname will always start with \\?\ - # strip off that prefix unless it was already provided on the original # path. diff --git a/Lib/posixpath.py b/Lib/posixpath.py index db72ded8826056..d38f3bd5872bcd 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -36,7 +36,7 @@ "samefile","sameopenfile","samestat", "curdir","pardir","sep","pathsep","defpath","altsep","extsep", "devnull","realpath","supports_unicode_filenames","relpath", - "commonpath", "isjunction","isdevdrive"] + "commonpath", "isjunction","isdevdrive","ALLOW_MISSING"] def _get_sep(path): @@ -402,10 +402,18 @@ def realpath(filename, *, strict=False): curdir = '.' pardir = '..' getcwd = os.getcwd - return _realpath(filename, strict, sep, curdir, pardir, getcwd) + if strict is ALLOW_MISSING: + ignored_error = FileNotFoundError + strict = True + elif strict: + ignored_error = () + else: + ignored_error = OSError + + lstat = os.lstat + readlink = os.readlink + maxlinks = None -def _realpath(filename, strict=False, sep=sep, curdir=curdir, pardir=pardir, - getcwd=os.getcwd, lstat=os.lstat, readlink=os.readlink, maxlinks=None): # The stack of unresolved path parts. When popped, a special value of None # indicates that a symlink target has been resolved, and that the original # symlink path can be retrieved by popping again. The [::-1] slice is a @@ -477,27 +485,28 @@ def _realpath(filename, strict=False, sep=sep, curdir=curdir, pardir=pardir, path = newpath continue target = readlink(newpath) - except OSError: - if strict: - raise - path = newpath + except ignored_error: + pass + else: + # Resolve the symbolic link + if target.startswith(sep): + # Symlink target is absolute; reset resolved path. + path = sep + if maxlinks is None: + # Mark this symlink as seen but not fully resolved. + seen[newpath] = None + # Push the symlink path onto the stack, and signal its specialness + # by also pushing None. When these entries are popped, we'll + # record the fully-resolved symlink target in the 'seen' mapping. + rest.append(newpath) + rest.append(None) + # Push the unresolved symlink target parts onto the stack. + target_parts = target.split(sep)[::-1] + rest.extend(target_parts) + part_count += len(target_parts) continue - # Resolve the symbolic link - if target.startswith(sep): - # Symlink target is absolute; reset resolved path. - path = sep - if maxlinks is None: - # Mark this symlink as seen but not fully resolved. - seen[newpath] = None - # Push the symlink path onto the stack, and signal its specialness - # by also pushing None. When these entries are popped, we'll - # record the fully-resolved symlink target in the 'seen' mapping. - rest.append(newpath) - rest.append(None) - # Push the unresolved symlink target parts onto the stack. - target_parts = target.split(sep)[::-1] - rest.extend(target_parts) - part_count += len(target_parts) + # An error occurred and was ignored. + path = newpath return path diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 212b71f6509740..068aa13ed70356 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -67,7 +67,7 @@ "DEFAULT_FORMAT", "open","fully_trusted_filter", "data_filter", "tar_filter", "FilterError", "AbsoluteLinkError", "OutsideDestinationError", "SpecialFileError", "AbsolutePathError", - "LinkOutsideDestinationError"] + "LinkOutsideDestinationError", "LinkFallbackError"] #--------------------------------------------------------- @@ -766,10 +766,22 @@ def __init__(self, tarinfo, path): super().__init__(f'{tarinfo.name!r} would link to {path!r}, ' + 'which is outside the destination') +class LinkFallbackError(FilterError): + def __init__(self, tarinfo, path): + self.tarinfo = tarinfo + self._path = path + super().__init__(f'link {tarinfo.name!r} would be extracted as a ' + + f'copy of {path!r}, which was rejected') + +# Errors caused by filters -- both "fatal" and "non-fatal" -- that +# we consider to be issues with the argument, rather than a bug in the +# filter function +_FILTER_ERRORS = (FilterError, OSError, ExtractError) + def _get_filtered_attrs(member, dest_path, for_data=True): new_attrs = {} name = member.name - dest_path = os.path.realpath(dest_path) + dest_path = os.path.realpath(dest_path, strict=os.path.ALLOW_MISSING) # Strip leading / (tar's directory separator) from filenames. # Include os.sep (target OS directory separator) as well. if name.startswith(('/', os.sep)): @@ -779,7 +791,8 @@ def _get_filtered_attrs(member, dest_path, for_data=True): # For example, 'C:/foo' on Windows. raise AbsolutePathError(member) # Ensure we stay in the destination - target_path = os.path.realpath(os.path.join(dest_path, name)) + target_path = os.path.realpath(os.path.join(dest_path, name), + strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: raise OutsideDestinationError(member, target_path) # Limit permissions (no high bits, and go-w) @@ -817,6 +830,9 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) + normalized = os.path.normpath(member.linkname) + if normalized != member.linkname: + new_attrs['linkname'] = normalized if member.issym(): target_path = os.path.join(dest_path, os.path.dirname(name), @@ -824,7 +840,8 @@ def _get_filtered_attrs(member, dest_path, for_data=True): else: target_path = os.path.join(dest_path, member.linkname) - target_path = os.path.realpath(target_path) + target_path = os.path.realpath(target_path, + strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: raise LinkOutsideDestinationError(member, target_path) return new_attrs @@ -2386,30 +2403,58 @@ def extractall(self, path=".", members=None, *, numeric_owner=False, members = self for member in members: - tarinfo = self._get_extract_tarinfo(member, filter_function, path) + tarinfo, unfiltered = self._get_extract_tarinfo( + member, filter_function, path) if tarinfo is None: continue if tarinfo.isdir(): # For directories, delay setting attributes until later, # since permissions can interfere with extraction and # extracting contents can reset mtime. - directories.append(tarinfo) + directories.append(unfiltered) self._extract_one(tarinfo, path, set_attrs=not tarinfo.isdir(), - numeric_owner=numeric_owner) + numeric_owner=numeric_owner, + filter_function=filter_function) # Reverse sort directories. directories.sort(key=lambda a: a.name, reverse=True) + # Set correct owner, mtime and filemode on directories. - for tarinfo in directories: - dirpath = os.path.join(path, tarinfo.name) + for unfiltered in directories: try: + # Need to re-apply any filter, to take the *current* filesystem + # state into account. + try: + tarinfo = filter_function(unfiltered, path) + except _FILTER_ERRORS as exc: + self._log_no_directory_fixup(unfiltered, repr(exc)) + continue + if tarinfo is None: + self._log_no_directory_fixup(unfiltered, + 'excluded by filter') + continue + dirpath = os.path.join(path, tarinfo.name) + try: + lstat = os.lstat(dirpath) + except FileNotFoundError: + self._log_no_directory_fixup(tarinfo, 'missing') + continue + if not stat.S_ISDIR(lstat.st_mode): + # This is no longer a directory; presumably a later + # member overwrote the entry. + self._log_no_directory_fixup(tarinfo, 'not a directory') + continue self.chown(tarinfo, dirpath, numeric_owner=numeric_owner) self.utime(tarinfo, dirpath) self.chmod(tarinfo, dirpath) except ExtractError as e: self._handle_nonfatal_error(e) + def _log_no_directory_fixup(self, member, reason): + self._dbg(2, "tarfile: Not fixing up directory %r (%s)" % + (member.name, reason)) + def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, filter=None): """Extract a member from the archive to the current working directory, @@ -2425,41 +2470,56 @@ def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, String names of common filters are accepted. """ filter_function = self._get_filter_function(filter) - tarinfo = self._get_extract_tarinfo(member, filter_function, path) + tarinfo, unfiltered = self._get_extract_tarinfo( + member, filter_function, path) if tarinfo is not None: self._extract_one(tarinfo, path, set_attrs, numeric_owner) def _get_extract_tarinfo(self, member, filter_function, path): - """Get filtered TarInfo (or None) from member, which might be a str""" + """Get (filtered, unfiltered) TarInfos from *member* + + *member* might be a string. + + Return (None, None) if not found. + """ + if isinstance(member, str): - tarinfo = self.getmember(member) + unfiltered = self.getmember(member) else: - tarinfo = member + unfiltered = member - unfiltered = tarinfo + filtered = None try: - tarinfo = filter_function(tarinfo, path) + filtered = filter_function(unfiltered, path) except (OSError, UnicodeEncodeError, FilterError) as e: self._handle_fatal_error(e) except ExtractError as e: self._handle_nonfatal_error(e) - if tarinfo is None: + if filtered is None: self._dbg(2, "tarfile: Excluded %r" % unfiltered.name) - return None + return None, None + # Prepare the link target for makelink(). - if tarinfo.islnk(): - tarinfo = copy.copy(tarinfo) - tarinfo._link_target = os.path.join(path, tarinfo.linkname) - return tarinfo + if filtered.islnk(): + filtered = copy.copy(filtered) + filtered._link_target = os.path.join(path, filtered.linkname) + return filtered, unfiltered + + def _extract_one(self, tarinfo, path, set_attrs, numeric_owner, + filter_function=None): + """Extract from filtered tarinfo to disk. - def _extract_one(self, tarinfo, path, set_attrs, numeric_owner): - """Extract from filtered tarinfo to disk""" + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a symlink) + """ self._check("r") try: self._extract_member(tarinfo, os.path.join(path, tarinfo.name), set_attrs=set_attrs, - numeric_owner=numeric_owner) + numeric_owner=numeric_owner, + filter_function=filter_function, + extraction_root=path) except (OSError, UnicodeEncodeError) as e: self._handle_fatal_error(e) except ExtractError as e: @@ -2517,9 +2577,13 @@ def extractfile(self, member): return None def _extract_member(self, tarinfo, targetpath, set_attrs=True, - numeric_owner=False): - """Extract the TarInfo object tarinfo to a physical + numeric_owner=False, *, filter_function=None, + extraction_root=None): + """Extract the filtered TarInfo object tarinfo to a physical file called targetpath. + + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a symlink) """ # Fetch the TarInfo object for the given name # and build the destination pathname, replacing @@ -2548,7 +2612,10 @@ def _extract_member(self, tarinfo, targetpath, set_attrs=True, elif tarinfo.ischr() or tarinfo.isblk(): self.makedev(tarinfo, targetpath) elif tarinfo.islnk() or tarinfo.issym(): - self.makelink(tarinfo, targetpath) + self.makelink_with_filter( + tarinfo, targetpath, + filter_function=filter_function, + extraction_root=extraction_root) elif tarinfo.type not in SUPPORTED_TYPES: self.makeunknown(tarinfo, targetpath) else: @@ -2631,10 +2698,18 @@ def makedev(self, tarinfo, targetpath): os.makedev(tarinfo.devmajor, tarinfo.devminor)) def makelink(self, tarinfo, targetpath): + return self.makelink_with_filter(tarinfo, targetpath, None, None) + + def makelink_with_filter(self, tarinfo, targetpath, + filter_function, extraction_root): """Make a (symbolic) link called targetpath. If it cannot be created (platform limitation), we try to make a copy of the referenced file instead of a link. + + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a link). """ + keyerror_to_extracterror = False try: # For systems that support symbolic and hard links. if tarinfo.issym(): @@ -2642,18 +2717,38 @@ def makelink(self, tarinfo, targetpath): # Avoid FileExistsError on following os.symlink. os.unlink(targetpath) os.symlink(tarinfo.linkname, targetpath) + return else: if os.path.exists(tarinfo._link_target): os.link(tarinfo._link_target, targetpath) - else: - self._extract_member(self._find_link_target(tarinfo), - targetpath) + return except symlink_exception: + keyerror_to_extracterror = True + + try: + unfiltered = self._find_link_target(tarinfo) + except KeyError: + if keyerror_to_extracterror: + raise ExtractError( + "unable to resolve link inside archive") from None + else: + raise + + if filter_function is None: + filtered = unfiltered + else: + if extraction_root is None: + raise ExtractError( + "makelink_with_filter: if filter_function is not None, " + + "extraction_root must also not be None") try: - self._extract_member(self._find_link_target(tarinfo), - targetpath) - except KeyError: - raise ExtractError("unable to resolve link inside archive") from None + filtered = filter_function(unfiltered, extraction_root) + except _FILTER_ERRORS as cause: + raise LinkFallbackError(tarinfo, unfiltered.name) from cause + if filtered is not None: + self._extract_member(filtered, targetpath, + filter_function=filter_function, + extraction_root=extraction_root) def chown(self, tarinfo, targetpath, numeric_owner): """Set owner of targetpath according to tarinfo. If numeric_owner diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index c3b0bdaebc2329..b891d0734ca0a6 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -6,6 +6,7 @@ import sys import unittest import warnings +from ntpath import ALLOW_MISSING from test.support import TestFailed, cpython_only, os_helper from test.support.os_helper import FakePath from test import test_genericpath @@ -76,6 +77,27 @@ def tester(fn, wantResult): %(str(fn), str(wantResult), repr(gotResult))) +def _parameterize(*parameters): + """Simplistic decorator to parametrize a test + + Runs the decorated test multiple times in subTest, with a value from + 'parameters' passed as an extra positional argument. + Calls doCleanups() after each run. + + Not for general use. Intended to avoid indenting for easier backports. + + See https://discuss.python.org/t/91827 for discussing generalizations. + """ + def _parametrize_decorator(func): + def _parameterized(self, *args, **kwargs): + for parameter in parameters: + with self.subTest(parameter): + func(self, *args, parameter, **kwargs) + self.doCleanups() + return _parameterized + return _parametrize_decorator + + class NtpathTestCase(unittest.TestCase): def assertPathEqual(self, path1, path2): if path1 == path2 or _norm(path1) == _norm(path2): @@ -474,6 +496,27 @@ def test_realpath_curdir(self): tester("ntpath.realpath('.\\.')", expected) tester("ntpath.realpath('\\'.join(['.'] * 100))", expected) + def test_realpath_curdir_strict(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('.', strict=True)", expected) + tester("ntpath.realpath('./.', strict=True)", expected) + tester("ntpath.realpath('/'.join(['.'] * 100), strict=True)", expected) + tester("ntpath.realpath('.\\.', strict=True)", expected) + tester("ntpath.realpath('\\'.join(['.'] * 100), strict=True)", expected) + + def test_realpath_curdir_missing_ok(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('.', strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('./.', strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('/'.join(['.'] * 100), strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('.\\.', strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('\\'.join(['.'] * 100), strict=ALLOW_MISSING)", + expected) + def test_realpath_pardir(self): expected = ntpath.normpath(os.getcwd()) tester("ntpath.realpath('..')", ntpath.dirname(expected)) @@ -486,24 +529,59 @@ def test_realpath_pardir(self): tester("ntpath.realpath('\\'.join(['..'] * 50))", ntpath.splitdrive(expected)[0] + '\\') + def test_realpath_pardir_strict(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('..', strict=True)", ntpath.dirname(expected)) + tester("ntpath.realpath('../..', strict=True)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('/'.join(['..'] * 50), strict=True)", + ntpath.splitdrive(expected)[0] + '\\') + tester("ntpath.realpath('..\\..', strict=True)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('\\'.join(['..'] * 50), strict=True)", + ntpath.splitdrive(expected)[0] + '\\') + + def test_realpath_pardir_missing_ok(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('..', strict=ALLOW_MISSING)", + ntpath.dirname(expected)) + tester("ntpath.realpath('../..', strict=ALLOW_MISSING)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('/'.join(['..'] * 50), strict=ALLOW_MISSING)", + ntpath.splitdrive(expected)[0] + '\\') + tester("ntpath.realpath('..\\..', strict=ALLOW_MISSING)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('\\'.join(['..'] * 50), strict=ALLOW_MISSING)", + ntpath.splitdrive(expected)[0] + '\\') + @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') - def test_realpath_basic(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_basic(self, kwargs): ABSTFN = ntpath.abspath(os_helper.TESTFN) open(ABSTFN, "wb").close() self.addCleanup(os_helper.unlink, ABSTFN) self.addCleanup(os_helper.unlink, ABSTFN + "1") os.symlink(ABSTFN, ABSTFN + "1") - self.assertPathEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN) - self.assertPathEqual(ntpath.realpath(os.fsencode(ABSTFN + "1")), + self.assertPathEqual(ntpath.realpath(ABSTFN + "1", **kwargs), ABSTFN) + self.assertPathEqual(ntpath.realpath(os.fsencode(ABSTFN + "1"), **kwargs), os.fsencode(ABSTFN)) # gh-88013: call ntpath.realpath with binary drive name may raise a # TypeError. The drive should not exist to reproduce the bug. drives = {f"{c}:\\" for c in string.ascii_uppercase} - set(os.listdrives()) d = drives.pop().encode() - self.assertEqual(ntpath.realpath(d), d) + self.assertEqual(ntpath.realpath(d, strict=False), d) + + # gh-106242: Embedded nulls and non-strict fallback to abspath + if kwargs: + with self.assertRaises(OSError): + ntpath.realpath(os_helper.TESTFN + "\0spam", + **kwargs) + else: + self.assertEqual(ABSTFN + "\0spam", + ntpath.realpath(os_helper.TESTFN + "\0spam", **kwargs)) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') @@ -526,51 +604,66 @@ def test_realpath_invalid_paths(self): self.assertEqual(realpath(path, strict=False), path) # gh-106242: Embedded nulls should raise OSError (not ValueError) self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) path = ABSTFNb + b'\x00' self.assertEqual(realpath(path, strict=False), path) self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) path = ABSTFN + '\\nonexistent\\x\x00' self.assertEqual(realpath(path, strict=False), path) self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) path = ABSTFNb + b'\\nonexistent\\x\x00' self.assertEqual(realpath(path, strict=False), path) self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) path = ABSTFN + '\x00\\..' self.assertEqual(realpath(path, strict=False), os.getcwd()) self.assertEqual(realpath(path, strict=True), os.getcwd()) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), os.getcwd()) path = ABSTFNb + b'\x00\\..' self.assertEqual(realpath(path, strict=False), os.getcwdb()) self.assertEqual(realpath(path, strict=True), os.getcwdb()) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), os.getcwdb()) path = ABSTFN + '\\nonexistent\\x\x00\\..' self.assertEqual(realpath(path, strict=False), ABSTFN + '\\nonexistent') self.assertRaises(OSError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), ABSTFN + '\\nonexistent') path = ABSTFNb + b'\\nonexistent\\x\x00\\..' self.assertEqual(realpath(path, strict=False), ABSTFNb + b'\\nonexistent') self.assertRaises(OSError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), ABSTFNb + b'\\nonexistent') + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_invalid_unicode_paths(self, kwargs): + realpath = ntpath.realpath + ABSTFN = ntpath.abspath(os_helper.TESTFN) + ABSTFNb = os.fsencode(ABSTFN) path = ABSTFNb + b'\xff' - self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) - self.assertRaises(UnicodeDecodeError, realpath, path, strict=True) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) path = ABSTFNb + b'\\nonexistent\\\xff' - self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) - self.assertRaises(UnicodeDecodeError, realpath, path, strict=True) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) path = ABSTFNb + b'\xff\\..' - self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) - self.assertRaises(UnicodeDecodeError, realpath, path, strict=True) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) path = ABSTFNb + b'\\nonexistent\\\xff\\..' - self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) - self.assertRaises(UnicodeDecodeError, realpath, path, strict=True) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') - def test_realpath_relative(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_relative(self, kwargs): ABSTFN = ntpath.abspath(os_helper.TESTFN) open(ABSTFN, "wb").close() self.addCleanup(os_helper.unlink, ABSTFN) self.addCleanup(os_helper.unlink, ABSTFN + "1") os.symlink(ABSTFN, ntpath.relpath(ABSTFN + "1")) - self.assertPathEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN) + self.assertPathEqual(ntpath.realpath(ABSTFN + "1", **kwargs), ABSTFN) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') @@ -722,7 +815,62 @@ def test_realpath_symlink_loops_strict(self): @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') - def test_realpath_symlink_prefix(self): + def test_realpath_symlink_loops_raise(self): + # Symlink loops raise OSError in ALLOW_MISSING mode + ABSTFN = ntpath.abspath(os_helper.TESTFN) + self.addCleanup(os_helper.unlink, ABSTFN) + self.addCleanup(os_helper.unlink, ABSTFN + "1") + self.addCleanup(os_helper.unlink, ABSTFN + "2") + self.addCleanup(os_helper.unlink, ABSTFN + "y") + self.addCleanup(os_helper.unlink, ABSTFN + "c") + self.addCleanup(os_helper.unlink, ABSTFN + "a") + self.addCleanup(os_helper.unlink, ABSTFN + "x") + + os.symlink(ABSTFN, ABSTFN) + self.assertRaises(OSError, ntpath.realpath, ABSTFN, strict=ALLOW_MISSING) + + os.symlink(ABSTFN + "1", ABSTFN + "2") + os.symlink(ABSTFN + "2", ABSTFN + "1") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1", + strict=ALLOW_MISSING) + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "2", + strict=ALLOW_MISSING) + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1\\x", + strict=ALLOW_MISSING) + + # Windows eliminates '..' components before resolving links; + # realpath is not expected to raise if this removes the loop. + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\.."), + ntpath.dirname(ABSTFN)) + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\x"), + ntpath.dirname(ABSTFN) + "\\x") + + os.symlink(ABSTFN + "x", ABSTFN + "y") + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\" + + ntpath.basename(ABSTFN) + "y"), + ABSTFN + "x") + self.assertRaises( + OSError, ntpath.realpath, + ABSTFN + "1\\..\\" + ntpath.basename(ABSTFN) + "1", + strict=ALLOW_MISSING) + + os.symlink(ntpath.basename(ABSTFN) + "a\\b", ABSTFN + "a") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "a", + strict=ALLOW_MISSING) + + os.symlink("..\\" + ntpath.basename(ntpath.dirname(ABSTFN)) + + "\\" + ntpath.basename(ABSTFN) + "c", ABSTFN + "c") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "c", + strict=ALLOW_MISSING) + + # Test using relative path as well. + self.assertRaises(OSError, ntpath.realpath, ntpath.basename(ABSTFN), + strict=ALLOW_MISSING) + + @os_helper.skip_unless_symlink + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_symlink_prefix(self, kwargs): ABSTFN = ntpath.abspath(os_helper.TESTFN) self.addCleanup(os_helper.unlink, ABSTFN + "3") self.addCleanup(os_helper.unlink, "\\\\?\\" + ABSTFN + "3.") @@ -737,9 +885,9 @@ def test_realpath_symlink_prefix(self): f.write(b'1') os.symlink("\\\\?\\" + ABSTFN + "3.", ABSTFN + "3.link") - self.assertPathEqual(ntpath.realpath(ABSTFN + "3link"), + self.assertPathEqual(ntpath.realpath(ABSTFN + "3link", **kwargs), ABSTFN + "3") - self.assertPathEqual(ntpath.realpath(ABSTFN + "3.link"), + self.assertPathEqual(ntpath.realpath(ABSTFN + "3.link", **kwargs), "\\\\?\\" + ABSTFN + "3.") # Resolved paths should be usable to open target files @@ -749,14 +897,17 @@ def test_realpath_symlink_prefix(self): self.assertEqual(f.read(), b'1') # When the prefix is included, it is not stripped - self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3link"), + self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3link", **kwargs), "\\\\?\\" + ABSTFN + "3") - self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link"), + self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link", **kwargs), "\\\\?\\" + ABSTFN + "3.") @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') def test_realpath_nul(self): tester("ntpath.realpath('NUL')", r'\\.\NUL') + tester("ntpath.realpath('NUL', strict=False)", r'\\.\NUL') + tester("ntpath.realpath('NUL', strict=True)", r'\\.\NUL') + tester("ntpath.realpath('NUL', strict=ALLOW_MISSING)", r'\\.\NUL') @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') @unittest.skipUnless(HAVE_GETSHORTPATHNAME, 'need _getshortpathname') @@ -780,12 +931,20 @@ def test_realpath_cwd(self): self.assertPathEqual(test_file_long, ntpath.realpath(test_file_short)) - with os_helper.change_cwd(test_dir_long): - self.assertPathEqual(test_file_long, ntpath.realpath("file.txt")) - with os_helper.change_cwd(test_dir_long.lower()): - self.assertPathEqual(test_file_long, ntpath.realpath("file.txt")) - with os_helper.change_cwd(test_dir_short): - self.assertPathEqual(test_file_long, ntpath.realpath("file.txt")) + for kwargs in {}, {'strict': True}, {'strict': ALLOW_MISSING}: + with self.subTest(**kwargs): + with os_helper.change_cwd(test_dir_long): + self.assertPathEqual( + test_file_long, + ntpath.realpath("file.txt", **kwargs)) + with os_helper.change_cwd(test_dir_long.lower()): + self.assertPathEqual( + test_file_long, + ntpath.realpath("file.txt", **kwargs)) + with os_helper.change_cwd(test_dir_short): + self.assertPathEqual( + test_file_long, + ntpath.realpath("file.txt", **kwargs)) @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') def test_realpath_permission(self): @@ -806,12 +965,15 @@ def test_realpath_permission(self): # Automatic generation of short names may be disabled on # NTFS volumes for the sake of performance. # They're not supported at all on ReFS and exFAT. - subprocess.run( + p = subprocess.run( # Try to set the short name manually. ['fsutil.exe', 'file', 'setShortName', test_file, 'LONGFI~1.TXT'], creationflags=subprocess.DETACHED_PROCESS ) + if p.returncode: + raise unittest.SkipTest('failed to set short name') + try: self.assertPathEqual(test_file, ntpath.realpath(test_file_short)) except AssertionError: diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index f3f9895f529470..c45ce6d3ef7820 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -4,7 +4,8 @@ import random import sys import unittest -from posixpath import realpath, abspath, dirname, basename +from functools import partial +from posixpath import realpath, abspath, dirname, basename, ALLOW_MISSING from test import support from test import test_genericpath from test.support import import_helper @@ -33,6 +34,27 @@ def skip_if_ABSTFN_contains_backslash(test): msg = "ABSTFN is not a posix path - tests fail" return [test, unittest.skip(msg)(test)][found_backslash] + +def _parameterize(*parameters): + """Simplistic decorator to parametrize a test + + Runs the decorated test multiple times in subTest, with a value from + 'parameters' passed as an extra positional argument. + Does *not* call doCleanups() after each run. + + Not for general use. Intended to avoid indenting for easier backports. + + See https://discuss.python.org/t/91827 for discussing generalizations. + """ + def _parametrize_decorator(func): + def _parameterized(self, *args, **kwargs): + for parameter in parameters: + with self.subTest(parameter): + func(self, *args, parameter, **kwargs) + return _parameterized + return _parametrize_decorator + + class PosixPathTest(unittest.TestCase): def setUp(self): @@ -442,32 +464,35 @@ def test_normpath(self): self.assertEqual(result, expected) @skip_if_ABSTFN_contains_backslash - def test_realpath_curdir(self): - self.assertEqual(realpath('.'), os.getcwd()) - self.assertEqual(realpath('./.'), os.getcwd()) - self.assertEqual(realpath('/'.join(['.'] * 100)), os.getcwd()) + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_curdir(self, kwargs): + self.assertEqual(realpath('.', **kwargs), os.getcwd()) + self.assertEqual(realpath('./.', **kwargs), os.getcwd()) + self.assertEqual(realpath('/'.join(['.'] * 100), **kwargs), os.getcwd()) - self.assertEqual(realpath(b'.'), os.getcwdb()) - self.assertEqual(realpath(b'./.'), os.getcwdb()) - self.assertEqual(realpath(b'/'.join([b'.'] * 100)), os.getcwdb()) + self.assertEqual(realpath(b'.', **kwargs), os.getcwdb()) + self.assertEqual(realpath(b'./.', **kwargs), os.getcwdb()) + self.assertEqual(realpath(b'/'.join([b'.'] * 100), **kwargs), os.getcwdb()) @skip_if_ABSTFN_contains_backslash - def test_realpath_pardir(self): - self.assertEqual(realpath('..'), dirname(os.getcwd())) - self.assertEqual(realpath('../..'), dirname(dirname(os.getcwd()))) - self.assertEqual(realpath('/'.join(['..'] * 100)), '/') + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_pardir(self, kwargs): + self.assertEqual(realpath('..', **kwargs), dirname(os.getcwd())) + self.assertEqual(realpath('../..', **kwargs), dirname(dirname(os.getcwd()))) + self.assertEqual(realpath('/'.join(['..'] * 100), **kwargs), '/') - self.assertEqual(realpath(b'..'), dirname(os.getcwdb())) - self.assertEqual(realpath(b'../..'), dirname(dirname(os.getcwdb()))) - self.assertEqual(realpath(b'/'.join([b'..'] * 100)), b'/') + self.assertEqual(realpath(b'..', **kwargs), dirname(os.getcwdb())) + self.assertEqual(realpath(b'../..', **kwargs), dirname(dirname(os.getcwdb()))) + self.assertEqual(realpath(b'/'.join([b'..'] * 100), **kwargs), b'/') @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_basic(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_basic(self, kwargs): # Basic operation. try: os.symlink(ABSTFN+"1", ABSTFN) - self.assertEqual(realpath(ABSTFN), ABSTFN+"1") + self.assertEqual(realpath(ABSTFN, **kwargs), ABSTFN+"1") finally: os_helper.unlink(ABSTFN) @@ -487,90 +512,115 @@ def test_realpath_invalid_paths(self): path = '/\x00' self.assertRaises(ValueError, realpath, path, strict=False) self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) path = b'/\x00' self.assertRaises(ValueError, realpath, path, strict=False) self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) path = '/nonexistent/x\x00' self.assertRaises(ValueError, realpath, path, strict=False) self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) path = b'/nonexistent/x\x00' self.assertRaises(ValueError, realpath, path, strict=False) self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) path = '/\x00/..' self.assertRaises(ValueError, realpath, path, strict=False) self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) path = b'/\x00/..' self.assertRaises(ValueError, realpath, path, strict=False) self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = '/nonexistent/x\x00/..' self.assertRaises(ValueError, realpath, path, strict=False) self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) path = b'/nonexistent/x\x00/..' self.assertRaises(ValueError, realpath, path, strict=False) self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) path = '/\udfff' if sys.platform == 'win32': self.assertEqual(realpath(path, strict=False), path) self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) else: self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) self.assertRaises(UnicodeEncodeError, realpath, path, strict=True) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) path = '/nonexistent/\udfff' if sys.platform == 'win32': self.assertEqual(realpath(path, strict=False), path) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) else: self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) self.assertRaises(FileNotFoundError, realpath, path, strict=True) path = '/\udfff/..' if sys.platform == 'win32': self.assertEqual(realpath(path, strict=False), '/') self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), '/') else: self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) self.assertRaises(UnicodeEncodeError, realpath, path, strict=True) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) path = '/nonexistent/\udfff/..' if sys.platform == 'win32': self.assertEqual(realpath(path, strict=False), '/nonexistent') + self.assertEqual(realpath(path, strict=ALLOW_MISSING), '/nonexistent') else: self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) self.assertRaises(FileNotFoundError, realpath, path, strict=True) path = b'/\xff' if sys.platform == 'win32': self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) self.assertRaises(UnicodeDecodeError, realpath, path, strict=True) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=ALLOW_MISSING) else: self.assertEqual(realpath(path, strict=False), path) if support.is_wasi: self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) else: self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) path = b'/nonexistent/\xff' if sys.platform == 'win32': self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=ALLOW_MISSING) else: self.assertEqual(realpath(path, strict=False), path) if support.is_wasi: self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) else: self.assertRaises(FileNotFoundError, realpath, path, strict=True) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_relative(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_relative(self, kwargs): try: os.symlink(posixpath.relpath(ABSTFN+"1"), ABSTFN) - self.assertEqual(realpath(ABSTFN), ABSTFN+"1") + self.assertEqual(realpath(ABSTFN, **kwargs), ABSTFN+"1") finally: os_helper.unlink(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_missing_pardir(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_missing_pardir(self, kwargs): try: os.symlink(TESTFN + "1", TESTFN) - self.assertEqual(realpath("nonexistent/../" + TESTFN), ABSTFN + "1") + self.assertEqual( + realpath("nonexistent/../" + TESTFN, **kwargs), ABSTFN + "1") finally: os_helper.unlink(TESTFN) @@ -617,37 +667,38 @@ def test_realpath_symlink_loops(self): @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_symlink_loops_strict(self): + @_parameterize({'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_symlink_loops_strict(self, kwargs): # Bug #43757, raise OSError if we get into an infinite symlink loop in - # strict mode. + # the strict modes. try: os.symlink(ABSTFN, ABSTFN) - self.assertRaises(OSError, realpath, ABSTFN, strict=True) + self.assertRaises(OSError, realpath, ABSTFN, **kwargs) os.symlink(ABSTFN+"1", ABSTFN+"2") os.symlink(ABSTFN+"2", ABSTFN+"1") - self.assertRaises(OSError, realpath, ABSTFN+"1", strict=True) - self.assertRaises(OSError, realpath, ABSTFN+"2", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"1", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"2", **kwargs) - self.assertRaises(OSError, realpath, ABSTFN+"1/x", strict=True) - self.assertRaises(OSError, realpath, ABSTFN+"1/..", strict=True) - self.assertRaises(OSError, realpath, ABSTFN+"1/../x", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"1/x", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"1/..", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"1/../x", **kwargs) os.symlink(ABSTFN+"x", ABSTFN+"y") self.assertRaises(OSError, realpath, - ABSTFN+"1/../" + basename(ABSTFN) + "y", strict=True) + ABSTFN+"1/../" + basename(ABSTFN) + "y", **kwargs) self.assertRaises(OSError, realpath, - ABSTFN+"1/../" + basename(ABSTFN) + "1", strict=True) + ABSTFN+"1/../" + basename(ABSTFN) + "1", **kwargs) os.symlink(basename(ABSTFN) + "a/b", ABSTFN+"a") - self.assertRaises(OSError, realpath, ABSTFN+"a", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"a", **kwargs) os.symlink("../" + basename(dirname(ABSTFN)) + "/" + basename(ABSTFN) + "c", ABSTFN+"c") - self.assertRaises(OSError, realpath, ABSTFN+"c", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"c", **kwargs) # Test using relative path as well. with os_helper.change_cwd(dirname(ABSTFN)): - self.assertRaises(OSError, realpath, basename(ABSTFN), strict=True) + self.assertRaises(OSError, realpath, basename(ABSTFN), **kwargs) finally: os_helper.unlink(ABSTFN) os_helper.unlink(ABSTFN+"1") @@ -658,13 +709,14 @@ def test_realpath_symlink_loops_strict(self): @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_repeated_indirect_symlinks(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_repeated_indirect_symlinks(self, kwargs): # Issue #6975. try: os.mkdir(ABSTFN) os.symlink('../' + basename(ABSTFN), ABSTFN + '/self') os.symlink('self/self/self', ABSTFN + '/link') - self.assertEqual(realpath(ABSTFN + '/link'), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/link', **kwargs), ABSTFN) finally: os_helper.unlink(ABSTFN + '/self') os_helper.unlink(ABSTFN + '/link') @@ -672,14 +724,15 @@ def test_realpath_repeated_indirect_symlinks(self): @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_deep_recursion(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_deep_recursion(self, kwargs): depth = 10 try: os.mkdir(ABSTFN) for i in range(depth): os.symlink('/'.join(['%d' % i] * 10), ABSTFN + '/%d' % (i + 1)) os.symlink('.', ABSTFN + '/0') - self.assertEqual(realpath(ABSTFN + '/%d' % depth), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/%d' % depth, **kwargs), ABSTFN) # Test using relative path as well. with os_helper.change_cwd(ABSTFN): @@ -691,7 +744,8 @@ def test_realpath_deep_recursion(self): @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_parents(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_parents(self, kwargs): # We also need to resolve any symlinks in the parents of a relative # path passed to realpath. E.g.: current working directory is # /usr/doc with 'doc' being a symlink to /usr/share/doc. We call @@ -702,7 +756,8 @@ def test_realpath_resolve_parents(self): os.symlink(ABSTFN + "/y", ABSTFN + "/k") with os_helper.change_cwd(ABSTFN + "/k"): - self.assertEqual(realpath("a"), ABSTFN + "/y/a") + self.assertEqual(realpath("a", **kwargs), + ABSTFN + "/y/a") finally: os_helper.unlink(ABSTFN + "/k") os_helper.rmdir(ABSTFN + "/y") @@ -710,7 +765,8 @@ def test_realpath_resolve_parents(self): @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_before_normalizing(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_before_normalizing(self, kwargs): # Bug #990669: Symbolic links should be resolved before we # normalize the path. E.g.: if we have directories 'a', 'k' and 'y' # in the following hierarchy: @@ -725,10 +781,10 @@ def test_realpath_resolve_before_normalizing(self): os.symlink(ABSTFN + "/k/y", ABSTFN + "/link-y") # Absolute path. - self.assertEqual(realpath(ABSTFN + "/link-y/.."), ABSTFN + "/k") + self.assertEqual(realpath(ABSTFN + "/link-y/..", **kwargs), ABSTFN + "/k") # Relative path. with os_helper.change_cwd(dirname(ABSTFN)): - self.assertEqual(realpath(basename(ABSTFN) + "/link-y/.."), + self.assertEqual(realpath(basename(ABSTFN) + "/link-y/..", **kwargs), ABSTFN + "/k") finally: os_helper.unlink(ABSTFN + "/link-y") @@ -738,7 +794,8 @@ def test_realpath_resolve_before_normalizing(self): @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_first(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_first(self, kwargs): # Bug #1213894: The first component of the path, if not absolute, # must be resolved too. @@ -748,8 +805,8 @@ def test_realpath_resolve_first(self): os.symlink(ABSTFN, ABSTFN + "link") with os_helper.change_cwd(dirname(ABSTFN)): base = basename(ABSTFN) - self.assertEqual(realpath(base + "link"), ABSTFN) - self.assertEqual(realpath(base + "link/k"), ABSTFN + "/k") + self.assertEqual(realpath(base + "link", **kwargs), ABSTFN) + self.assertEqual(realpath(base + "link/k", **kwargs), ABSTFN + "/k") finally: os_helper.unlink(ABSTFN + "link") os_helper.rmdir(ABSTFN + "/k") @@ -767,12 +824,67 @@ def test_realpath_unreadable_symlink(self): self.assertEqual(realpath(ABSTFN + '/foo'), ABSTFN + '/foo') self.assertEqual(realpath(ABSTFN + '/../foo'), dirname(ABSTFN) + '/foo') self.assertEqual(realpath(ABSTFN + '/foo/..'), ABSTFN) - with self.assertRaises(PermissionError): - realpath(ABSTFN, strict=True) finally: os.chmod(ABSTFN, 0o755, follow_symlinks=False) os_helper.unlink(ABSTFN) + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + @unittest.skipIf(os.chmod not in os.supports_follow_symlinks, "Can't set symlink permissions") + @unittest.skipIf(sys.platform != "darwin", "only macOS requires read permission to readlink()") + @_parameterize({'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_unreadable_symlink_strict(self, kwargs): + try: + os.symlink(ABSTFN+"1", ABSTFN) + os.chmod(ABSTFN, 0o000, follow_symlinks=False) + with self.assertRaises(PermissionError): + realpath(ABSTFN, **kwargs) + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/foo', **kwargs), + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/../foo', **kwargs) + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/foo/..', **kwargs) + finally: + os.chmod(ABSTFN, 0o755, follow_symlinks=False) + os.unlink(ABSTFN) + + @skip_if_ABSTFN_contains_backslash + @os_helper.skip_unless_symlink + def test_realpath_unreadable_directory(self): + try: + os.mkdir(ABSTFN) + os.mkdir(ABSTFN + '/k') + os.chmod(ABSTFN, 0o000) + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN) + + try: + os.stat(ABSTFN) + except PermissionError: + pass + else: + self.skipTest('Cannot block permissions') + + self.assertEqual(realpath(ABSTFN + '/k', strict=False), + ABSTFN + '/k') + self.assertRaises(PermissionError, realpath, ABSTFN + '/k', + strict=True) + self.assertRaises(PermissionError, realpath, ABSTFN + '/k', + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + '/missing', strict=False), + ABSTFN + '/missing') + self.assertRaises(PermissionError, realpath, ABSTFN + '/missing', + strict=True) + self.assertRaises(PermissionError, realpath, ABSTFN + '/missing', + strict=ALLOW_MISSING) + finally: + os.chmod(ABSTFN, 0o755) + os_helper.rmdir(ABSTFN + '/k') + os_helper.rmdir(ABSTFN) + @skip_if_ABSTFN_contains_backslash def test_realpath_nonterminal_file(self): try: @@ -780,14 +892,27 @@ def test_realpath_nonterminal_file(self): f.write('test_posixpath wuz ere') self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN) self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN) + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "/subdir") self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", + strict=ALLOW_MISSING) finally: os_helper.unlink(ABSTFN) @@ -800,14 +925,27 @@ def test_realpath_nonterminal_symlink_to_file(self): os.symlink(ABSTFN + "1", ABSTFN) self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN + "1") self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "1") + self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN + "1") + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN + "1") self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN + "1") self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "1/subdir") self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", + strict=ALLOW_MISSING) finally: os_helper.unlink(ABSTFN) os_helper.unlink(ABSTFN + "1") @@ -822,14 +960,27 @@ def test_realpath_nonterminal_symlink_to_symlinks_to_file(self): os.symlink(ABSTFN + "1", ABSTFN) self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN + "2") self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "2") + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "2") + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN + "2") self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN + "2") self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "2/subdir") self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", + strict=ALLOW_MISSING) finally: os_helper.unlink(ABSTFN) os_helper.unlink(ABSTFN + "1") @@ -1017,9 +1168,12 @@ def test_path_normpath(self): def test_path_abspath(self): self.assertPathEqual(self.path.abspath) - def test_path_realpath(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_path_realpath(self, kwargs): self.assertPathEqual(self.path.realpath) + self.assertPathEqual(partial(self.path.realpath, **kwargs)) + def test_path_relpath(self): self.assertPathEqual(self.path.relpath) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index cf218a2bf14369..7055e1ed147a9e 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -2715,6 +2715,31 @@ def test_useful_error_message_when_modules_missing(self): str(excinfo.exception), ) + @unittest.skipUnless(os_helper.can_symlink(), 'requires symlink support') + @unittest.skipUnless(hasattr(os, 'chmod'), "missing os.chmod") + @unittest.mock.patch('os.chmod') + def test_deferred_directory_attributes_update(self, mock_chmod): + # Regression test for gh-127987: setting attributes on arbitrary files + tempdir = os.path.join(TEMPDIR, 'test127987') + def mock_chmod_side_effect(path, mode, **kwargs): + target_path = os.path.realpath(path) + if os.path.commonpath([target_path, tempdir]) != tempdir: + raise Exception("should not try to chmod anything outside the destination", target_path) + mock_chmod.side_effect = mock_chmod_side_effect + + outside_tree_dir = os.path.join(TEMPDIR, 'outside_tree_dir') + with ArchiveMaker() as arc: + arc.add('x', symlink_to='.') + arc.add('x', type=tarfile.DIRTYPE, mode='?rwsrwsrwt') + arc.add('x', symlink_to=outside_tree_dir) + + os.makedirs(outside_tree_dir) + try: + arc.open().extractall(path=tempdir, filter='tar') + finally: + os_helper.rmtree(outside_tree_dir) + os_helper.rmtree(tempdir) + class CommandLineTest(unittest.TestCase): @@ -3275,6 +3300,10 @@ def check_files_present(self, directory): got_paths = set( p.relative_to(directory) for p in pathlib.Path(directory).glob('**/*')) + if self.extraction_filter in (None, 'data'): + # The 'data' filter is expected to reject special files + for path in 'ustar/fifotype', 'ustar/blktype', 'ustar/chrtype': + got_paths.discard(pathlib.Path(path)) self.assertEqual(self.control_paths, got_paths) @contextmanager @@ -3504,12 +3533,28 @@ def __exit__(self, *exc): self.bio = None def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, - mode=None, size=None, **kwargs): - """Add a member to the test archive. Call within `with`.""" + mode=None, size=None, content=None, **kwargs): + """Add a member to the test archive. Call within `with`. + + Provides many shortcuts: + - default `type` is based on symlink_to, hardlink_to, and trailing `/` + in name (which is stripped) + - size & content defaults are based on each other + - content can be str or bytes + - mode should be textual ('-rwxrwxrwx') + + (add more! this is unstable internal test-only API) + """ name = str(name) tarinfo = tarfile.TarInfo(name).replace(**kwargs) + if content is not None: + if isinstance(content, str): + content = content.encode() + size = len(content) if size is not None: tarinfo.size = size + if content is None: + content = bytes(tarinfo.size) if mode: tarinfo.mode = _filemode_to_int(mode) if symlink_to is not None: @@ -3523,7 +3568,7 @@ def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, if type is not None: tarinfo.type = type if tarinfo.isreg(): - fileobj = io.BytesIO(bytes(tarinfo.size)) + fileobj = io.BytesIO(content) else: fileobj = None self.tar_w.addfile(tarinfo, fileobj) @@ -3557,7 +3602,7 @@ class TestExtractionFilters(unittest.TestCase): destdir = outerdir / 'dest' @contextmanager - def check_context(self, tar, filter): + def check_context(self, tar, filter, *, check_flag=True): """Extracts `tar` to `self.destdir` and allows checking the result If an error occurs, it must be checked using `expect_exception` @@ -3566,27 +3611,40 @@ def check_context(self, tar, filter): except the destination directory itself and parent directories of other files. When checking directories, do so before their contents. + + A file called 'flag' is made in outerdir (i.e. outside destdir) + before extraction; it should not be altered nor should its contents + be read/copied. """ with os_helper.temp_dir(self.outerdir): + flag_path = self.outerdir / 'flag' + flag_path.write_text('capture me') try: tar.extractall(self.destdir, filter=filter) except Exception as exc: self.raised_exception = exc + self.reraise_exception = True self.expected_paths = set() else: self.raised_exception = None + self.reraise_exception = False self.expected_paths = set(self.outerdir.glob('**/*')) self.expected_paths.discard(self.destdir) + self.expected_paths.discard(flag_path) try: - yield + yield self finally: tar.close() - if self.raised_exception: + if self.reraise_exception: raise self.raised_exception self.assertEqual(self.expected_paths, set()) + if check_flag: + self.assertEqual(flag_path.read_text(), 'capture me') + else: + assert filter == 'fully_trusted' def expect_file(self, name, type=None, symlink_to=None, mode=None, - size=None): + size=None, content=None): """Check a single file. See check_context.""" if self.raised_exception: raise self.raised_exception @@ -3605,26 +3663,45 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None, # The symlink might be the same (textually) as what we expect, # but some systems change the link to an equivalent path, so # we fall back to samefile(). - if expected != got: - self.assertTrue(got.samefile(expected)) + try: + if expected != got: + self.assertTrue(got.samefile(expected)) + except Exception as e: + # attach a note, so it's shown even if `samefile` fails + e.add_note(f'{expected=}, {got=}') + raise elif type == tarfile.REGTYPE or type is None: self.assertTrue(path.is_file()) elif type == tarfile.DIRTYPE: self.assertTrue(path.is_dir()) elif type == tarfile.FIFOTYPE: self.assertTrue(path.is_fifo()) + elif type == tarfile.SYMTYPE: + self.assertTrue(path.is_symlink()) else: raise NotImplementedError(type) if size is not None: self.assertEqual(path.stat().st_size, size) + if content is not None: + self.assertEqual(path.read_text(), content) for parent in path.parents: self.expected_paths.discard(parent) + def expect_any_tree(self, name): + """Check a directory; forget about its contents.""" + tree_path = (self.destdir / name).resolve() + self.expect_file(tree_path, type=tarfile.DIRTYPE) + self.expected_paths = { + p for p in self.expected_paths + if tree_path not in p.parents + } + def expect_exception(self, exc_type, message_re='.'): with self.assertRaisesRegex(exc_type, message_re): if self.raised_exception is not None: raise self.raised_exception - self.raised_exception = None + self.reraise_exception = False + return self.raised_exception def test_benign_file(self): with ArchiveMaker() as arc: @@ -3709,6 +3786,80 @@ def test_parent_symlink(self): with self.check_context(arc.open(), 'data'): self.expect_file('parent/evil') + @symlink_test + @os_helper.skip_unless_symlink + def test_realpath_limit_attack(self): + # (CVE-2025-4517) + + with ArchiveMaker() as arc: + # populate the symlinks and dirs that expand in os.path.realpath() + # The component length is chosen so that in common cases, the unexpanded + # path fits in PATH_MAX, but it overflows when the final symlink + # is expanded + steps = "abcdefghijklmnop" + if sys.platform == 'win32': + component = 'd' * 25 + elif 'PC_PATH_MAX' in os.pathconf_names: + max_path_len = os.pathconf(self.outerdir.parent, "PC_PATH_MAX") + path_sep_len = 1 + dest_len = len(str(self.destdir)) + path_sep_len + component_len = (max_path_len - dest_len) // (len(steps) + path_sep_len) + component = 'd' * component_len + else: + raise NotImplementedError("Need to guess component length for {sys.platform}") + path = "" + step_path = "" + for i in steps: + arc.add(os.path.join(path, component), type=tarfile.DIRTYPE, + mode='drwxrwxrwx') + arc.add(os.path.join(path, i), symlink_to=component) + path = os.path.join(path, component) + step_path = os.path.join(step_path, i) + # create the final symlink that exceeds PATH_MAX and simply points + # to the top dir. + # this link will never be expanded by + # os.path.realpath(strict=False), nor anything after it. + linkpath = os.path.join(*steps, "l"*254) + parent_segments = [".."] * len(steps) + arc.add(linkpath, symlink_to=os.path.join(*parent_segments)) + # make a symlink outside to keep the tar command happy + arc.add("escape", symlink_to=os.path.join(linkpath, "..")) + # use the symlinks above, that are not checked, to create a hardlink + # to a file outside of the destination path + arc.add("flaglink", hardlink_to=os.path.join("escape", "flag")) + # now that we have the hardlink we can overwrite the file + arc.add("flaglink", content='overwrite') + # we can also create new files as well! + arc.add("escape/newfile", content='new') + + with (self.subTest('fully_trusted'), + self.check_context(arc.open(), filter='fully_trusted', + check_flag=False)): + if sys.platform == 'win32': + self.expect_exception((FileNotFoundError, FileExistsError)) + elif self.raised_exception: + # Cannot symlink/hardlink: tarfile falls back to getmember() + self.expect_exception(KeyError) + # Otherwise, this block should never enter. + else: + self.expect_any_tree(component) + self.expect_file('flaglink', content='overwrite') + self.expect_file('../newfile', content='new') + self.expect_file('escape', type=tarfile.SYMTYPE) + self.expect_file('a', symlink_to=component) + + for filter in 'tar', 'data': + with self.subTest(filter), self.check_context(arc.open(), filter=filter): + exc = self.expect_exception((OSError, KeyError)) + if isinstance(exc, OSError): + if sys.platform == 'win32': + # 3: ERROR_PATH_NOT_FOUND + # 5: ERROR_ACCESS_DENIED + # 206: ERROR_FILENAME_EXCED_RANGE + self.assertIn(exc.winerror, (3, 5, 206)) + else: + self.assertEqual(exc.errno, errno.ENAMETOOLONG) + @symlink_test def test_parent_symlink2(self): # Test interplaying symlinks @@ -3931,8 +4082,8 @@ def test_chains(self): arc.add('symlink2', symlink_to=os.path.join( 'linkdir', 'hardlink2')) arc.add('targetdir/target', size=3) - arc.add('linkdir/hardlink', hardlink_to='targetdir/target') - arc.add('linkdir/hardlink2', hardlink_to='linkdir/symlink') + arc.add('linkdir/hardlink', hardlink_to=os.path.join('targetdir', 'target')) + arc.add('linkdir/hardlink2', hardlink_to=os.path.join('linkdir', 'symlink')) for filter in 'tar', 'data', 'fully_trusted': with self.check_context(arc.open(), filter): @@ -3948,6 +4099,129 @@ def test_chains(self): self.expect_file('linkdir/symlink', size=3) self.expect_file('symlink2', size=3) + @symlink_test + def test_sneaky_hardlink_fallback(self): + # (CVE-2025-4330) + # Test that when hardlink extraction falls back to extracting members + # from the archive, the extracted member is (re-)filtered. + with ArchiveMaker() as arc: + # Create a directory structure so the c/escape symlink stays + # inside the path + arc.add("a/t/dummy") + # Create b/ directory + arc.add("b/") + # Point "c" to the bottom of the tree in "a" + arc.add("c", symlink_to=os.path.join("a", "t")) + # link to non-existant location under "a" + arc.add("c/escape", symlink_to=os.path.join("..", "..", + "link_here")) + # Move "c" to point to "b" ("c/escape" no longer exists) + arc.add("c", symlink_to="b") + # Attempt to create a hard link to "c/escape". Since it doesn't + # exist it will attempt to extract "cescape" but at "boom". + arc.add("boom", hardlink_to=os.path.join("c", "escape")) + + with self.check_context(arc.open(), 'data'): + if not os_helper.can_symlink(): + # When 'c/escape' is extracted, 'c' is a regular + # directory, and 'c/escape' *would* point outside + # the destination if symlinks were allowed. + self.expect_exception( + tarfile.LinkOutsideDestinationError) + elif sys.platform == "win32": + # On Windows, 'c/escape' points outside the destination + self.expect_exception(tarfile.LinkOutsideDestinationError) + else: + e = self.expect_exception( + tarfile.LinkFallbackError, + "link 'boom' would be extracted as a copy of " + + "'c/escape', which was rejected") + self.assertIsInstance(e.__cause__, + tarfile.LinkOutsideDestinationError) + for filter in 'tar', 'fully_trusted': + with self.subTest(filter), self.check_context(arc.open(), filter): + if not os_helper.can_symlink(): + self.expect_file("a/t/dummy") + self.expect_file("b/") + self.expect_file("c/") + else: + self.expect_file("a/t/dummy") + self.expect_file("b/") + self.expect_file("a/t/escape", symlink_to='../../link_here') + self.expect_file("boom", symlink_to='../../link_here') + self.expect_file("c", symlink_to='b') + + @symlink_test + def test_exfiltration_via_symlink(self): + # (CVE-2025-4138) + # Test changing symlinks that result in a symlink pointing outside + # the extraction directory, unless prevented by 'data' filter's + # normalization. + with ArchiveMaker() as arc: + arc.add("escape", symlink_to=os.path.join('link', 'link', '..', '..', 'link-here')) + arc.add("link", symlink_to='./') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + if os_helper.can_symlink(): + self.expect_file("link", symlink_to='./') + if filter == 'data': + self.expect_file("escape", symlink_to='link-here') + else: + self.expect_file("escape", + symlink_to='link/link/../../link-here') + else: + # Nothing is extracted. + pass + + @symlink_test + def test_chmod_outside_dir(self): + # (CVE-2024-12718) + # Test that members used for delayed updates of directory metadata + # are (re-)filtered. + with ArchiveMaker() as arc: + # "pwn" is a veeeery innocent symlink: + arc.add("a/pwn", symlink_to='.') + # But now "pwn" is also a directory, so it's scheduled to have its + # metadata updated later: + arc.add("a/pwn/", mode='drwxrwxrwx') + # Oops, "pwn" is not so innocent any more: + arc.add("a/pwn", symlink_to='x/../') + # Newly created symlink points to the dest dir, + # so it's OK for the "data" filter. + arc.add('a/x', symlink_to=('../')) + # But now "pwn" points outside the dest dir + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter) as cc: + if not os_helper.can_symlink(): + self.expect_file("a/pwn/") + elif filter == 'data': + self.expect_file("a/x", symlink_to='../') + self.expect_file("a/pwn", symlink_to='.') + else: + self.expect_file("a/x", symlink_to='../') + self.expect_file("a/pwn", symlink_to='x/../') + if sys.platform != "win32": + st_mode = cc.outerdir.stat().st_mode + self.assertNotEqual(st_mode & 0o777, 0o777) + + def test_link_fallback_normalizes(self): + # Make sure hardlink fallbacks work for non-normalized paths for all + # filters + with ArchiveMaker() as arc: + arc.add("dir/") + arc.add("dir/../afile") + arc.add("link1", hardlink_to='dir/../afile') + arc.add("link2", hardlink_to='dir/../dir/../afile') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter) as cc: + self.expect_file("dir/") + self.expect_file("afile") + self.expect_file("link1") + self.expect_file("link2") + def test_modes(self): # Test how file modes are extracted # (Note that the modes are ignored on platforms without working chmod) @@ -4072,7 +4346,7 @@ def test_tar_filter(self): # The 'tar' filter returns TarInfo objects with the same name/type. # (It can also fail for particularly "evil" input, but we don't have # that in the test archive.) - with tarfile.TarFile.open(tarname) as tar: + with tarfile.TarFile.open(tarname, encoding="iso8859-1") as tar: for tarinfo in tar.getmembers(): try: filtered = tarfile.tar_filter(tarinfo, '') @@ -4084,7 +4358,7 @@ def test_tar_filter(self): def test_data_filter(self): # The 'data' filter either raises, or returns TarInfo with the same # name/type. - with tarfile.TarFile.open(tarname) as tar: + with tarfile.TarFile.open(tarname, encoding="iso8859-1") as tar: for tarinfo in tar.getmembers(): try: filtered = tarfile.data_filter(tarinfo, '') @@ -4242,13 +4516,13 @@ def valueerror_filter(tarinfo, path): # If errorlevel is 0, errors affected by errorlevel are ignored with self.check_context(arc.open(errorlevel=0), extracterror_filter): - self.expect_file('file') + pass with self.check_context(arc.open(errorlevel=0), filtererror_filter): - self.expect_file('file') + pass with self.check_context(arc.open(errorlevel=0), oserror_filter): - self.expect_file('file') + pass with self.check_context(arc.open(errorlevel=0), tarerror_filter): self.expect_exception(tarfile.TarError) @@ -4259,7 +4533,7 @@ def valueerror_filter(tarinfo, path): # If 1, all fatal errors are raised with self.check_context(arc.open(errorlevel=1), extracterror_filter): - self.expect_file('file') + pass with self.check_context(arc.open(errorlevel=1), filtererror_filter): self.expect_exception(tarfile.FilterError) diff --git a/Misc/NEWS.d/next/Security/2025-06-02-11-32-23.gh-issue-135034.RLGjbp.rst b/Misc/NEWS.d/next/Security/2025-06-02-11-32-23.gh-issue-135034.RLGjbp.rst new file mode 100644 index 00000000000000..08a0087e203671 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-06-02-11-32-23.gh-issue-135034.RLGjbp.rst @@ -0,0 +1,6 @@ +Fixes multiple issues that allowed ``tarfile`` extraction filters +(``filter="data"`` and ``filter="tar"``) to be bypassed using crafted +symlinks and hard links. + +Addresses :cve:`2024-12718`, :cve:`2025-4138`, :cve:`2025-4330`, and :cve:`2025-4517`. + From 54ca55978e305ebb099d1b49633211597625bd52 Mon Sep 17 00:00:00 2001 From: tpburns Date: Tue, 3 Jun 2025 12:00:25 -0400 Subject: [PATCH 071/116] gh-134248 test_getallocatedblocks pre-check to ignore immortalized strings (#134871) When sanity checking against gettotalrefcount(), we exclude the blocks for immortalized strings since their references are not tracked/reported. This now matches refleak.py's book-keeping using the same functions. --- Lib/test/test_sys.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 83745f3d0ba46e..bf415894903e9b 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1135,23 +1135,12 @@ def test_getallocatedblocks(self): b = sys.getallocatedblocks() self.assertLessEqual(b, a) try: - # While we could imagine a Python session where the number of - # multiple buffer objects would exceed the sharing of references, - # it is unlikely to happen in a normal test run. - # - # In free-threaded builds each code object owns an array of - # pointers to copies of the bytecode. When the number of - # code objects is a large fraction of the total number of - # references, this can cause the total number of allocated - # blocks to exceed the total number of references. - # - # For some reason, iOS seems to trigger the "unlikely to happen" - # case reliably under CI conditions. It's not clear why; but as - # this test is checking the behavior of getallocatedblock() - # under garbage collection, we can skip this pre-condition check - # for now. See GH-130384. - if not support.Py_GIL_DISABLED and not support.is_apple_mobile: - self.assertLess(a, sys.gettotalrefcount()) + # The reported blocks will include immortalized strings, but the + # total ref count will not. This will sanity check that among all + # other objects (those eligible for garbage collection) there + # are more references being tracked than allocated blocks. + interned_immortal = sys.getunicodeinternedsize(_only_immortal=True) + self.assertLess(a - interned_immortal, sys.gettotalrefcount()) except AttributeError: # gettotalrefcount() not available pass From 1ffe913c2017b44804aca18befd45689df06c069 Mon Sep 17 00:00:00 2001 From: Duane Griffin Date: Wed, 4 Jun 2025 05:28:58 +1200 Subject: [PATCH 072/116] gh-127081: use `getlogin_r` if available (gh-132751) The `getlogin` function is not thread-safe: replace with `getlogin_r` where available. --- ...-04-21-01-03-15.gh-issue-127081.WXRliX.rst | 2 ++ Modules/posixmodule.c | 18 ++++++++++ configure | 33 +++++++++++++++++++ configure.ac | 14 +++++++- pyconfig.h.in | 13 ++++++++ 5 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-04-21-01-03-15.gh-issue-127081.WXRliX.rst diff --git a/Misc/NEWS.d/next/Library/2025-04-21-01-03-15.gh-issue-127081.WXRliX.rst b/Misc/NEWS.d/next/Library/2025-04-21-01-03-15.gh-issue-127081.WXRliX.rst new file mode 100644 index 00000000000000..63fed60ced03c5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-21-01-03-15.gh-issue-127081.WXRliX.rst @@ -0,0 +1,2 @@ +Fix libc thread safety issues with :mod:`os` by replacing ``getlogin`` with +``getlogin_r`` re-entrant version. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 588894adeac811..07ab5499cd4fa1 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -9562,6 +9562,24 @@ os_getlogin_impl(PyObject *module) } else result = PyErr_SetFromWindowsErr(GetLastError()); +#elif defined (HAVE_GETLOGIN_R) +# if defined (HAVE_MAXLOGNAME) + char name[MAXLOGNAME + 1]; +# elif defined (HAVE_UT_NAMESIZE) + char name[UT_NAMESIZE + 1]; +# else + char name[256]; +# endif + int err = getlogin_r(name, sizeof(name)); + if (err) { + int old_errno = errno; + errno = -err; + posix_error(); + errno = old_errno; + } + else { + result = PyUnicode_DecodeFSDefault(name); + } #else char *name; int old_errno = errno; diff --git a/configure b/configure index b0dc18d5cea749..029bf527da4e3d 100755 --- a/configure +++ b/configure @@ -19458,6 +19458,12 @@ if test "x$ac_cv_func_getlogin" = xyes then : printf "%s\n" "#define HAVE_GETLOGIN 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "getlogin_r" "ac_cv_func_getlogin_r" +if test "x$ac_cv_func_getlogin_r" = xyes +then : + printf "%s\n" "#define HAVE_GETLOGIN_R 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "getpeername" "ac_cv_func_getpeername" if test "x$ac_cv_func_getpeername" = xyes @@ -23843,6 +23849,33 @@ fi +ac_fn_check_decl "$LINENO" "MAXLOGNAME" "ac_cv_have_decl_MAXLOGNAME" "#include +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_MAXLOGNAME" = xyes +then : + +printf "%s\n" "#define HAVE_MAXLOGNAME 1" >>confdefs.h + +fi + +ac_fn_check_decl "$LINENO" "UT_NAMESIZE" "ac_cv_have_decl_UT_NAMESIZE" "#include +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_UT_NAMESIZE" = xyes +then : + ac_have_decl=1 +else case e in #( + e) ac_have_decl=0 ;; +esac +fi +printf "%s\n" "#define HAVE_DECL_UT_NAMESIZE $ac_have_decl" >>confdefs.h +if test $ac_have_decl = 1 +then : + +printf "%s\n" "#define HAVE_UT_NAMESIZE 1" >>confdefs.h + +fi + + # check for openpty, login_tty, and forkpty diff --git a/configure.ac b/configure.ac index 70ad6da86719c6..371b2e8ed73525 100644 --- a/configure.ac +++ b/configure.ac @@ -5219,7 +5219,7 @@ AC_CHECK_FUNCS([ \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ gai_strerror getegid geteuid getgid getgrent getgrgid getgrgid_r \ - getgrnam_r getgrouplist gethostname getitimer getloadavg getlogin \ + getgrnam_r getgrouplist gethostname getitimer getloadavg getlogin getlogin_r \ getpeername getpgid getpid getppid getpriority _getpty \ getpwent getpwnam_r getpwuid getpwuid_r getresgid getresuid getrusage getsid getspent \ getspnam getuid getwd grantpt if_nameindex initgroups kill killpg lchown linkat \ @@ -5538,6 +5538,18 @@ PY_CHECK_FUNC([setgroups], [ #endif ]) +AC_CHECK_DECL([MAXLOGNAME], + [AC_DEFINE([HAVE_MAXLOGNAME], [1], + [Define if you have the 'MAXLOGNAME' constant.])], + [], + [@%:@include ]) + +AC_CHECK_DECLS([UT_NAMESIZE], + [AC_DEFINE([HAVE_UT_NAMESIZE], [1], + [Define if you have the 'HAVE_UT_NAMESIZE' constant.])], + [], + [@%:@include ]) + # check for openpty, login_tty, and forkpty AC_CHECK_FUNCS([openpty], [], diff --git a/pyconfig.h.in b/pyconfig.h.in index 3dbbda157df70e..65a2c55217c258 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -267,6 +267,10 @@ */ #undef HAVE_DECL_TZNAME +/* Define to 1 if you have the declaration of 'UT_NAMESIZE', and to 0 if you + don't. */ +#undef HAVE_DECL_UT_NAMESIZE + /* Define to 1 if you have the device macros. */ #undef HAVE_DEVICE_MACROS @@ -539,6 +543,9 @@ /* Define to 1 if you have the 'getlogin' function. */ #undef HAVE_GETLOGIN +/* Define to 1 if you have the 'getlogin_r' function. */ +#undef HAVE_GETLOGIN_R + /* Define to 1 if you have the 'getnameinfo' function. */ #undef HAVE_GETNAMEINFO @@ -807,6 +814,9 @@ /* Define this if you have the makedev macro. */ #undef HAVE_MAKEDEV +/* Define if you have the 'MAXLOGNAME' constant. */ +#undef HAVE_MAXLOGNAME + /* Define to 1 if you have the 'mbrtowc' function. */ #undef HAVE_MBRTOWC @@ -1575,6 +1585,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_UTMP_H +/* Define if you have the 'HAVE_UT_NAMESIZE' constant. */ +#undef HAVE_UT_NAMESIZE + /* Define to 1 if you have the 'uuid_create' function. */ #undef HAVE_UUID_CREATE From dba9de731b231ca0c079205f496d1e3d178b4fd3 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 4 Jun 2025 01:31:43 +0100 Subject: [PATCH 073/116] gh-135101: When choosing the default simulator device, don't use `simctl --set testing` (#135102) On a fresh Xcode install (including some CI provider configurations), there is no pre-existing testing set that can be used to identify simulator models. Use the default device set to detect available models instead. Live testing simulators are still created in the testing set. --- iOS/testbed/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iOS/testbed/__main__.py b/iOS/testbed/__main__.py index c05497ede3aa61..1146bf3b988cda 100644 --- a/iOS/testbed/__main__.py +++ b/iOS/testbed/__main__.py @@ -127,7 +127,7 @@ async def async_check_output(*args, **kwargs): async def select_simulator_device(): # List the testing simulators, in JSON format raw_json = await async_check_output( - "xcrun", "simctl", "--set", "testing", "list", "-j" + "xcrun", "simctl", "list", "-j" ) json_data = json.loads(raw_json) From 8f778f7bb9a8ad80fc06570566ad4de541826178 Mon Sep 17 00:00:00 2001 From: Christian Veenhuis <124370897+ChVeen@users.noreply.github.com> Date: Wed, 4 Jun 2025 06:57:31 +0200 Subject: [PATCH 074/116] gh-135103: Remove an unused local variable in Lib/code.py (GH-135104) remove unused local variable --- Lib/code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/code.py b/Lib/code.py index b134886dc267fb..f7e275d8801b7c 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -224,7 +224,7 @@ def interact(self, banner=None, exitmsg=None): sys.ps1 = ">>> " delete_ps1_after = True try: - _ps2 = sys.ps2 + sys.ps2 delete_ps2_after = False except AttributeError: sys.ps2 = "... " From cc581f32bf5f15e9f2f89b830ec64ea25684d0cd Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 4 Jun 2025 03:35:56 -0400 Subject: [PATCH 075/116] gh-135099: Only wait on `_PyOS_SigintEvent()` in main thread (GH-135100) On Windows, the `_PyOS_SigintEvent()` event handle is used to interrupt the main thread when Ctrl-C is pressed. Previously, we also waited on the event from other threads, but ignored the result. However, this can race with interpreter shutdown because the main thread closes the handle in `_PySignal_Fini` and threads may still be running and using mutexes during interpreter shtudown. Only use `_PyOS_SigintEvent()` in the main thread in parking_lot.c, like we do in other places in the CPython codebase. --- ...-06-03-18-26-54.gh-issue-135099.Q9usKm.rst | 2 ++ Python/parking_lot.c | 22 ++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2025-06-03-18-26-54.gh-issue-135099.Q9usKm.rst diff --git a/Misc/NEWS.d/next/Windows/2025-06-03-18-26-54.gh-issue-135099.Q9usKm.rst b/Misc/NEWS.d/next/Windows/2025-06-03-18-26-54.gh-issue-135099.Q9usKm.rst new file mode 100644 index 00000000000000..36e70b1c0d8cb6 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2025-06-03-18-26-54.gh-issue-135099.Q9usKm.rst @@ -0,0 +1,2 @@ +Fix a crash that could occur on Windows when a background thread waits on a +:c:type:`PyMutex` while the main thread is shutting down the interpreter. diff --git a/Python/parking_lot.c b/Python/parking_lot.c index 8edf43235942ab..e896dea02712f2 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -112,17 +112,27 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) } } - // NOTE: we wait on the sigint event even in non-main threads to match the - // behavior of the other platforms. Non-main threads will ignore the - // Py_PARK_INTR result. - HANDLE sigint_event = _PyOS_SigintEvent(); - HANDLE handles[2] = { sema->platform_sem, sigint_event }; - DWORD count = sigint_event != NULL ? 2 : 1; + HANDLE handles[2] = { sema->platform_sem, NULL }; + HANDLE sigint_event = NULL; + DWORD count = 1; + if (_Py_IsMainThread()) { + // gh-135099: Wait on the SIGINT event only in the main thread. Other + // threads would ignore the result anyways, and accessing + // `_PyOS_SigintEvent()` from non-main threads may race with + // interpreter shutdown, which closes the event handle. Note that + // non-main interpreters will ignore the result. + sigint_event = _PyOS_SigintEvent(); + if (sigint_event != NULL) { + handles[1] = sigint_event; + count = 2; + } + } wait = WaitForMultipleObjects(count, handles, FALSE, millis); if (wait == WAIT_OBJECT_0) { res = Py_PARK_OK; } else if (wait == WAIT_OBJECT_0 + 1) { + assert(sigint_event != NULL); ResetEvent(sigint_event); res = Py_PARK_INTR; } From 1f515104441898111c20aca5a7bbda1d11b15d36 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 4 Jun 2025 10:23:08 +0100 Subject: [PATCH 076/116] Use a more clear example for the PEP 758 what's new section (#135118) --- Doc/whatsnew/3.14.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 27dfc75c90fbe9..45e68aea5fb9a2 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -342,15 +342,16 @@ For example the following expressions are now valid: .. code-block:: python try: - release_new_sleep_token_album() - except AlbumNotFound, SongsTooGoodToBeReleased: - print("Sorry, no new album this year.") + connect_to_server() + except TimeoutError, ConnectionRefusedError: + print("Network issue encountered.") # The same applies to except* (for exception groups): + try: - release_new_sleep_token_album() - except* AlbumNotFound, SongsTooGoodToBeReleased: - print("Sorry, no new album this year.") + connect_to_server() + except* TimeoutError, ConnectionRefusedError: + print("Network issue encountered.") Check :pep:`758` for more details. From 5b3865418ceb1448bfbf15cddf52c900cd5882a3 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Wed, 4 Jun 2025 15:51:18 +0300 Subject: [PATCH 077/116] gh-135108: Fix utmp.h inclusion in posixmodule.c on NetBSD (GH-135109) --- Modules/posixmodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 07ab5499cd4fa1..7dc5ef39a566e4 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8820,14 +8820,14 @@ os_ptsname_impl(PyObject *module, int fd) #if defined(HAVE_OPENPTY) || defined(HAVE_FORKPTY) || defined(HAVE_LOGIN_TTY) || defined(HAVE_DEV_PTMX) #ifdef HAVE_PTY_H #include -#ifdef HAVE_UTMP_H -#include -#endif /* HAVE_UTMP_H */ #elif defined(HAVE_LIBUTIL_H) #include #elif defined(HAVE_UTIL_H) #include #endif /* HAVE_PTY_H */ +#ifdef HAVE_UTMP_H +#include +#endif /* HAVE_UTMP_H */ #ifdef HAVE_STROPTS_H #include #endif From bc00ce941e03347dade3faa8822f19836b5bbfe4 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Wed, 4 Jun 2025 13:58:34 +0100 Subject: [PATCH 078/116] gh-135074: Fix exception messages in test.support module (GH-135076) --- Lib/test/support/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 351d832a26d1df..f9b66b88d3d05c 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1084,7 +1084,7 @@ def set_memlimit(limit: str) -> None: global real_max_memuse memlimit = _parse_memlimit(limit) if memlimit < _2G - 1: - raise ValueError('Memory limit {limit!r} too low to be useful') + raise ValueError(f'Memory limit {limit!r} too low to be useful') real_max_memuse = memlimit memlimit = min(memlimit, MAX_Py_ssize_t) @@ -2358,7 +2358,7 @@ def infinite_recursion(max_depth=None): # very deep recursion. max_depth = 20_000 elif max_depth < 3: - raise ValueError("max_depth must be at least 3, got {max_depth}") + raise ValueError(f"max_depth must be at least 3, got {max_depth}") depth = get_recursion_depth() depth = max(depth - 1, 1) # Ignore infinite_recursion() frame. limit = depth + max_depth From 40c8be0008ecadb5d0dc9a017434b1133a3a6e06 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 4 Jun 2025 14:00:25 +0100 Subject: [PATCH 079/116] gh-126483: disable warnings filters mutation in concurrent test (GH-132694) The `test_ssl_in_multiple_threads` test failed because `test_check_hostname_idn()` modified the global warnings filters via `warnings_helper.check_no_resource_warning()`. Only check for warnings when the context aware warnings feature is enabled, which makes the warnings filter context-local and thread-safe. --- Lib/test/test_ssl.py | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 2767a53d53c1fc..f123f6ece40669 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -31,6 +31,7 @@ import platform import sysconfig import functools +from contextlib import nullcontext try: import ctypes except ImportError: @@ -2843,6 +2844,7 @@ def test_ssl_in_multiple_threads(self): # See GH-124984: OpenSSL is not thread safe. threads = [] + warnings_filters = sys.flags.context_aware_warnings global USE_SAME_TEST_CONTEXT USE_SAME_TEST_CONTEXT = True try: @@ -2851,7 +2853,10 @@ def test_ssl_in_multiple_threads(self): self.test_alpn_protocols, self.test_getpeercert, self.test_crl_check, - self.test_check_hostname_idn, + functools.partial( + self.test_check_hostname_idn, + warnings_filters=warnings_filters, + ), self.test_wrong_cert_tls12, self.test_wrong_cert_tls13, ): @@ -3097,7 +3102,7 @@ def test_dual_rsa_ecc(self): cipher = s.cipher()[0].split('-') self.assertTrue(cipher[:2], ('ECDHE', 'ECDSA')) - def test_check_hostname_idn(self): + def test_check_hostname_idn(self, warnings_filters=True): if support.verbose: sys.stdout.write("\n") @@ -3152,16 +3157,30 @@ def test_check_hostname_idn(self): server_hostname="python.example.org") as s: with self.assertRaises(ssl.CertificateError): s.connect((HOST, server.port)) - with ThreadedEchoServer(context=server_context, chatty=True) as server: - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(UnicodeError): - context.wrap_socket(socket.socket(), - server_hostname='.pythontest.net') - with ThreadedEchoServer(context=server_context, chatty=True) as server: - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(UnicodeDecodeError): - context.wrap_socket(socket.socket(), - server_hostname=b'k\xf6nig.idn.pythontest.net') + with ( + ThreadedEchoServer(context=server_context, chatty=True) as server, + ( + warnings_helper.check_no_resource_warning(self) + if warnings_filters + else nullcontext() + ), + self.assertRaises(UnicodeError), + ): + context.wrap_socket(socket.socket(), server_hostname='.pythontest.net') + + with ( + ThreadedEchoServer(context=server_context, chatty=True) as server, + ( + warnings_helper.check_no_resource_warning(self) + if warnings_filters + else nullcontext() + ), + self.assertRaises(UnicodeDecodeError), + ): + context.wrap_socket( + socket.socket(), + server_hostname=b'k\xf6nig.idn.pythontest.net', + ) def test_wrong_cert_tls12(self): """Connecting when the server rejects the client's certificate From c21113072cd1f0da83729f99d3576647db85d816 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 4 Jun 2025 15:07:52 +0200 Subject: [PATCH 080/116] gh-134989: Implement PyObject_DelAttr() as a macro in the limited C API (GH-135021) --- Include/abstract.h | 14 ++++++++++++-- .../2025-06-02-13-19-22.gh-issue-134989.sDDyBN.rst | 2 ++ 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-06-02-13-19-22.gh-issue-134989.sDDyBN.rst diff --git a/Include/abstract.h b/Include/abstract.h index b9199fc03a399a..80f3298701d249 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -138,7 +138,12 @@ extern "C" { Delete attribute named attr_name, for object o. Returns -1 on failure. - This is the equivalent of the Python statement: del o.attr_name. */ + This is the equivalent of the Python statement: del o.attr_name. + + Implemented as a macro in the limited C API 3.12 and older. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030d0000 +# define PyObject_DelAttrString(O, A) PyObject_SetAttrString((O), (A), NULL) +#endif /* Implemented elsewhere: @@ -147,7 +152,12 @@ extern "C" { Delete attribute named attr_name, for object o. Returns -1 on failure. This is the equivalent of the Python - statement: del o.attr_name. */ + statement: del o.attr_name. + + Implemented as a macro in the limited C API 3.12 and older. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030d0000 +# define PyObject_DelAttr(O, A) PyObject_SetAttr((O), (A), NULL) +#endif /* Implemented elsewhere: diff --git a/Misc/NEWS.d/next/C_API/2025-06-02-13-19-22.gh-issue-134989.sDDyBN.rst b/Misc/NEWS.d/next/C_API/2025-06-02-13-19-22.gh-issue-134989.sDDyBN.rst new file mode 100644 index 00000000000000..e49f765106582e --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-06-02-13-19-22.gh-issue-134989.sDDyBN.rst @@ -0,0 +1,2 @@ +Implement :c:func:`PyObject_DelAttr` and :c:func:`PyObject_DelAttrString` as +macros in the limited C API 3.12 and older. Patch by Victor Stinner. From 0df15d0d4d55dcf7b7a7f4b2985bf84fd092d257 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Wed, 4 Jun 2025 11:26:19 -0400 Subject: [PATCH 081/116] gh-129876: Update IDLE News3.txt to May 2025 (#135139) --- Lib/idlelib/News3.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/idlelib/News3.txt b/Lib/idlelib/News3.txt index 74d84b3893125a..30784578cc637f 100644 --- a/Lib/idlelib/News3.txt +++ b/Lib/idlelib/News3.txt @@ -4,6 +4,13 @@ Released on 2025-10-07 ========================= +gh-112936: IDLE - Include Shell menu in single-process mode, +though with Restart Shell and View Last Restart disabled. +Patch by Zhikang Yan. + +gh-112938: IDLE - Fix uninteruptable hang when Shell gets +rapid continuous output. + gh-127060: Set TERM environment variable to 'dumb' to not add ANSI escape sequences for text color in tracebacks. IDLE does not understand them. Patch by Victor Stinner. From a10b321a5807ba924c7a7833692fe5d0dc40e875 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 4 Jun 2025 10:06:10 -0700 Subject: [PATCH 082/116] Update the devcontainer image to the latest version (GH-135143) Fixes an issue where the WASI tools were being incorrectly installed under the ARM image. --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index fd1d7151518f40..8e09808f08bba2 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "image": "ghcr.io/python/devcontainer:2025.05.25.15232270922", + "image": "ghcr.io/python/devcontainer:2025.05.29.15334414373", "onCreateCommand": [ // Install common tooling. "dnf", From e598eecf4c97509acef517e94053e45db51636fb Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Wed, 4 Jun 2025 21:39:00 +0200 Subject: [PATCH 083/116] gh-135144: Add `_remote_debugging` to the MSI (legacy) Windows installers. (#135145) Add _remote_debugging to the MSI (legacy) installers. --- Tools/msi/freethreaded/freethreaded_files.wxs | 2 +- Tools/msi/lib/lib_files.wxs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/msi/freethreaded/freethreaded_files.wxs b/Tools/msi/freethreaded/freethreaded_files.wxs index 86d9a8b83f6535..0707e77b5e9ab2 100644 --- a/Tools/msi/freethreaded/freethreaded_files.wxs +++ b/Tools/msi/freethreaded/freethreaded_files.wxs @@ -103,7 +103,7 @@ - + diff --git a/Tools/msi/lib/lib_files.wxs b/Tools/msi/lib/lib_files.wxs index 8439518bcbd804..4d44299f783909 100644 --- a/Tools/msi/lib/lib_files.wxs +++ b/Tools/msi/lib/lib_files.wxs @@ -1,6 +1,6 @@  - + From 6b77af257c25d31f1f137e477cb23e63692ddf29 Mon Sep 17 00:00:00 2001 From: mpage Date: Wed, 4 Jun 2025 16:07:58 -0700 Subject: [PATCH 084/116] gh-134889: Fix handling of a few opcodes when optimizing `LOAD_FAST` (#134958) We were incorrectly handling a few opcodes that leave their operands on the stack. Treat all of these conservatively; assume that they always leave operands on the stack. --- Include/internal/pycore_magic_number.h | 3 +- Lib/test/test_dis.py | 2 +- Lib/test/test_peepholer.py | 92 +++++++++++++++++++ ...-05-30-18-09-54.gh-issue-134889.Ic9UM-.rst | 2 + Python/flowgraph.c | 20 ++++ 5 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-30-18-09-54.gh-issue-134889.Ic9UM-.rst diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index cd1fc873623ed1..347d9762f26bff 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -280,6 +280,7 @@ Known values: Python 3.15a0 3650 (Initial version) Python 3.15a1 3651 (Simplify LOAD_CONST) Python 3.15a1 3652 (Virtual iterators) + Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) Python 3.16 will start with 3700 @@ -293,7 +294,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3652 +#define PYC_MAGIC_NUMBER 3653 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index ec930a728aa5b3..355990ed58ee09 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -606,7 +606,7 @@ async def _asyncwith(c): POP_TOP L1: RESUME 0 -%4d LOAD_FAST_BORROW 0 (c) +%4d LOAD_FAST 0 (c) COPY 1 LOAD_SPECIAL 3 (__aexit__) SWAP 2 diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index f33de3d420ca34..ef596630b930f7 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -2614,6 +2614,90 @@ def test_send(self): ] self.cfg_optimization_test(insts, expected, consts=[None]) + def test_format_simple(self): + # FORMAT_SIMPLE will leave its operand on the stack if it's a unicode + # object. We treat it conservatively and assume that it always leaves + # its operand on the stack. + insts = [ + ("LOAD_FAST", 0, 1), + ("FORMAT_SIMPLE", None, 2), + ("STORE_FAST", 1, 3), + ] + self.check(insts, insts) + + insts = [ + ("LOAD_FAST", 0, 1), + ("FORMAT_SIMPLE", None, 2), + ("POP_TOP", None, 3), + ] + expected = [ + ("LOAD_FAST_BORROW", 0, 1), + ("FORMAT_SIMPLE", None, 2), + ("POP_TOP", None, 3), + ] + self.check(insts, expected) + + def test_set_function_attribute(self): + # SET_FUNCTION_ATTRIBUTE leaves the function on the stack + insts = [ + ("LOAD_CONST", 0, 1), + ("LOAD_FAST", 0, 2), + ("SET_FUNCTION_ATTRIBUTE", 2, 3), + ("STORE_FAST", 1, 4), + ("LOAD_CONST", 0, 5), + ("RETURN_VALUE", None, 6) + ] + self.cfg_optimization_test(insts, insts, consts=[None]) + + insts = [ + ("LOAD_CONST", 0, 1), + ("LOAD_FAST", 0, 2), + ("SET_FUNCTION_ATTRIBUTE", 2, 3), + ("RETURN_VALUE", None, 4) + ] + expected = [ + ("LOAD_CONST", 0, 1), + ("LOAD_FAST_BORROW", 0, 2), + ("SET_FUNCTION_ATTRIBUTE", 2, 3), + ("RETURN_VALUE", None, 4) + ] + self.cfg_optimization_test(insts, expected, consts=[None]) + + def test_get_yield_from_iter(self): + # GET_YIELD_FROM_ITER may leave its operand on the stack + insts = [ + ("LOAD_FAST", 0, 1), + ("GET_YIELD_FROM_ITER", None, 2), + ("LOAD_CONST", 0, 3), + send := self.Label(), + ("SEND", end := self.Label(), 5), + ("YIELD_VALUE", 1, 6), + ("RESUME", 2, 7), + ("JUMP", send, 8), + end, + ("END_SEND", None, 9), + ("LOAD_CONST", 0, 10), + ("RETURN_VALUE", None, 11), + ] + self.cfg_optimization_test(insts, insts, consts=[None]) + + def test_push_exc_info(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("PUSH_EXC_INFO", None, 2), + ] + self.check(insts, insts) + + def test_load_special(self): + # LOAD_SPECIAL may leave self on the stack + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_SPECIAL", 0, 2), + ("STORE_FAST", 1, 3), + ] + self.check(insts, insts) + + def test_del_in_finally(self): # This loads `obj` onto the stack, executes `del obj`, then returns the # `obj` from the stack. See gh-133371 for more details. @@ -2630,6 +2714,14 @@ def create_obj(): gc.collect() self.assertEqual(obj, [42]) + def test_format_simple_unicode(self): + # Repro from gh-134889 + def f(): + var = f"{1}" + var = f"{var}" + return var + self.assertEqual(f(), "1") + if __name__ == "__main__": diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-30-18-09-54.gh-issue-134889.Ic9UM-.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-30-18-09-54.gh-issue-134889.Ic9UM-.rst new file mode 100644 index 00000000000000..3b86134bf16800 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-30-18-09-54.gh-issue-134889.Ic9UM-.rst @@ -0,0 +1,2 @@ +Fix handling of a few opcodes that leave operands on the stack when +optimizing ``LOAD_FAST``. diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 67ccf350b72ed6..2adc8c84d83974 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -2870,9 +2870,11 @@ optimize_load_fast(cfg_builder *g) // how many inputs should be left on the stack. // Opcodes that consume no inputs + case FORMAT_SIMPLE: case GET_ANEXT: case GET_ITER: case GET_LEN: + case GET_YIELD_FROM_ITER: case IMPORT_FROM: case MATCH_KEYS: case MATCH_MAPPING: @@ -2907,6 +2909,16 @@ optimize_load_fast(cfg_builder *g) break; } + case END_SEND: + case SET_FUNCTION_ATTRIBUTE: { + assert(_PyOpcode_num_popped(opcode, oparg) == 2); + assert(_PyOpcode_num_pushed(opcode, oparg) == 1); + ref tos = ref_stack_pop(&refs); + ref_stack_pop(&refs); + PUSH_REF(tos.instr, tos.local); + break; + } + // Opcodes that consume some inputs and push new values case CHECK_EXC_MATCH: { ref_stack_pop(&refs); @@ -2936,6 +2948,14 @@ optimize_load_fast(cfg_builder *g) break; } + case LOAD_SPECIAL: + case PUSH_EXC_INFO: { + ref tos = ref_stack_pop(&refs); + PUSH_REF(i, NOT_LOCAL); + PUSH_REF(tos.instr, tos.local); + break; + } + case SEND: { load_fast_push_block(&sp, instr->i_target, refs.size); ref_stack_pop(&refs); From 2e1544fd2b0cd46ba93fc51e3cdd47f4781d7499 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 5 Jun 2025 06:46:16 +0100 Subject: [PATCH 085/116] gh-131531: android.py enhancements to support cibuildwheel (#132870) Modifies the environment handling and execution arguments of the Android management script to support the compilation of third-party binaries, and the use of the testbed to invoke third-party test code. Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Russell Keith-Magee --- Android/README.md | 4 + Android/android-env.sh | 6 +- Android/android.py | 231 ++++++++++++------ Android/testbed/app/build.gradle.kts | 20 +- .../java/org/python/testbed/PythonSuite.kt | 8 +- .../java/org/python/testbed/MainActivity.kt | 27 +- .../{main.py => android_testbed_main.py} | 20 +- Android/testbed/build.gradle.kts | 2 +- .../gradle/wrapper/gradle-wrapper.properties | 2 +- Doc/using/android.rst | 9 + 10 files changed, 239 insertions(+), 90 deletions(-) rename Android/testbed/app/src/main/python/{main.py => android_testbed_main.py} (68%) diff --git a/Android/README.md b/Android/README.md index 6cabd6ba5d6844..c42eb627006e6a 100644 --- a/Android/README.md +++ b/Android/README.md @@ -156,6 +156,10 @@ repository's `Lib` directory will be picked up immediately. Changes in C files, and architecture-specific files such as sysconfigdata, will not take effect until you re-run `android.py make-host` or `build`. +The testbed app can also be used to test third-party packages. For more details, +run `android.py test --help`, paying attention to the options `--site-packages`, +`--cwd`, `-c` and `-m`. + ## Using in your own app diff --git a/Android/android-env.sh b/Android/android-env.sh index bab4130c9e92d0..7b381a013cf0ba 100644 --- a/Android/android-env.sh +++ b/Android/android-env.sh @@ -3,7 +3,7 @@ : "${HOST:?}" # GNU target triplet # You may also override the following: -: "${api_level:=24}" # Minimum Android API level the build will run on +: "${ANDROID_API_LEVEL:=24}" # Minimum Android API level the build will run on : "${PREFIX:-}" # Path in which to find required libraries @@ -24,7 +24,7 @@ fail() { # * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md # where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.: # https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md -ndk_version=27.1.12297006 +ndk_version=27.2.12479018 ndk=$ANDROID_HOME/ndk/$ndk_version if ! [ -e "$ndk" ]; then @@ -43,7 +43,7 @@ fi toolchain=$(echo "$ndk"/toolchains/llvm/prebuilt/*) export AR="$toolchain/bin/llvm-ar" export AS="$toolchain/bin/llvm-as" -export CC="$toolchain/bin/${clang_triplet}${api_level}-clang" +export CC="$toolchain/bin/${clang_triplet}${ANDROID_API_LEVEL}-clang" export CXX="${CC}++" export LD="$toolchain/bin/ld" export NM="$toolchain/bin/llvm-nm" diff --git a/Android/android.py b/Android/android.py index 3f48b42aa17571..551168fc4b2f5a 100755 --- a/Android/android.py +++ b/Android/android.py @@ -14,7 +14,7 @@ from contextlib import asynccontextmanager from datetime import datetime, timezone from glob import glob -from os.path import basename, relpath +from os.path import abspath, basename, relpath from pathlib import Path from subprocess import CalledProcessError from tempfile import TemporaryDirectory @@ -22,9 +22,13 @@ SCRIPT_NAME = Path(__file__).name ANDROID_DIR = Path(__file__).resolve().parent -CHECKOUT = ANDROID_DIR.parent +PYTHON_DIR = ANDROID_DIR.parent +in_source_tree = ( + ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists() +) + TESTBED_DIR = ANDROID_DIR / "testbed" -CROSS_BUILD_DIR = CHECKOUT / "cross-build" +CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" HOSTS = ["aarch64-linux-android", "x86_64-linux-android"] APP_ID = "org.python.testbed" @@ -76,39 +80,68 @@ def run(command, *, host=None, env=None, log=True, **kwargs): kwargs.setdefault("check", True) if env is None: env = os.environ.copy() - original_env = env.copy() if host: - env_script = ANDROID_DIR / "android-env.sh" - env_output = subprocess.run( - f"set -eu; " - f"HOST={host}; " - f"PREFIX={subdir(host)}/prefix; " - f". {env_script}; " - f"export", - check=True, shell=True, text=True, stdout=subprocess.PIPE - ).stdout - - for line in env_output.splitlines(): - # We don't require every line to match, as there may be some other - # output from installing the NDK. - if match := re.search( - "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line - ): - key, value = match[2], match[3] - if env.get(key) != value: - print(line) - env[key] = value - - if env == original_env: - raise ValueError(f"Found no variables in {env_script.name} output:\n" - + env_output) + host_env = android_env(host) + print_env(host_env) + env.update(host_env) if log: - print(">", " ".join(map(str, command))) + print(">", join_command(command)) return subprocess.run(command, env=env, **kwargs) +# Format a command so it can be copied into a shell. Like shlex.join, but also +# accepts arguments which are Paths, or a single string/Path outside of a list. +def join_command(args): + if isinstance(args, (str, Path)): + return str(args) + else: + return shlex.join(map(str, args)) + + +# Format the environment so it can be pasted into a shell. +def print_env(env): + for key, value in sorted(env.items()): + print(f"export {key}={shlex.quote(value)}") + + +def android_env(host): + if host: + prefix = subdir(host) / "prefix" + else: + prefix = ANDROID_DIR / "prefix" + sysconfig_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py") + sysconfig_filename = next(sysconfig_files).name + host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1] + + env_script = ANDROID_DIR / "android-env.sh" + env_output = subprocess.run( + f"set -eu; " + f"export HOST={host}; " + f"PREFIX={prefix}; " + f". {env_script}; " + f"export", + check=True, shell=True, capture_output=True, encoding='utf-8', + ).stdout + + env = {} + for line in env_output.splitlines(): + # We don't require every line to match, as there may be some other + # output from installing the NDK. + if match := re.search( + "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line + ): + key, value = match[2], match[3] + if os.environ.get(key) != value: + env[key] = value + + if not env: + raise ValueError(f"Found no variables in {env_script.name} output:\n" + + env_output) + return env + + def build_python_path(): """The path to the build Python binary.""" build_dir = subdir("build") @@ -127,7 +160,7 @@ def configure_build_python(context): clean("build") os.chdir(subdir("build", create=True)) - command = [relpath(CHECKOUT / "configure")] + command = [relpath(PYTHON_DIR / "configure")] if context.args: command.extend(context.args) run(command) @@ -139,12 +172,13 @@ def make_build_python(context): def unpack_deps(host, prefix_dir): + os.chdir(prefix_dir) deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" - for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4", + for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.15-4", "sqlite-3.49.1-0", "xz-5.4.6-1"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") - shutil.unpack_archive(filename, prefix_dir) + shutil.unpack_archive(filename) os.remove(filename) @@ -167,7 +201,7 @@ def configure_host_python(context): os.chdir(host_dir) command = [ # Basic cross-compiling configuration - relpath(CHECKOUT / "configure"), + relpath(PYTHON_DIR / "configure"), f"--host={context.host}", f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", f"--with-build-python={build_python_path()}", @@ -196,9 +230,12 @@ def make_host_python(context): for pattern in ("include/python*", "lib/libpython*", "lib/python*"): delete_glob(f"{prefix_dir}/{pattern}") + # The Android environment variables were already captured in the Makefile by + # `configure`, and passing them again when running `make` may cause some + # flags to be duplicated. So we don't use the `host` argument here. os.chdir(host_dir) - run(["make", "-j", str(os.cpu_count())], host=context.host) - run(["make", "install", f"prefix={prefix_dir}"], host=context.host) + run(["make", "-j", str(os.cpu_count())]) + run(["make", "install", f"prefix={prefix_dir}"]) def build_all(context): @@ -228,7 +265,12 @@ def setup_sdk(): if not all((android_home / "licenses" / path).exists() for path in [ "android-sdk-arm-dbt-license", "android-sdk-license" ]): - run([sdkmanager, "--licenses"], text=True, input="y\n" * 100) + run( + [sdkmanager, "--licenses"], + text=True, + capture_output=True, + input="y\n" * 100, + ) # Gradle may install this automatically, but we can't rely on that because # we need to run adb within the logcat task. @@ -474,24 +516,49 @@ async def gradle_task(context): task_prefix = "connected" env["ANDROID_SERIAL"] = context.connected + hidden_output = [] + + def log(line): + # Gradle may take several minutes to install SDK packages, so it's worth + # showing those messages even in non-verbose mode. + if context.verbose or line.startswith('Preparing "Install'): + sys.stdout.write(line) + else: + hidden_output.append(line) + + if context.command: + mode = "-c" + module = context.command + else: + mode = "-m" + module = context.module or "test" + args = [ gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", - "-Pandroid.testInstrumentationRunnerArguments.pythonArgs=" - + shlex.join(context.args), + ] + [ + # Build-time properties + f"-Ppython.{name}={value}" + for name, value in [ + ("sitePackages", context.site_packages), ("cwd", context.cwd) + ] if value + ] + [ + # Runtime properties + f"-Pandroid.testInstrumentationRunnerArguments.python{name}={value}" + for name, value in [ + ("Mode", mode), ("Module", module), ("Args", join_command(context.args)) + ] if value ] - hidden_output = [] + if context.verbose >= 2: + args.append("--info") + log("> " + join_command(args)) + try: async with async_process( *args, cwd=TESTBED_DIR, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) as process: while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - # Gradle may take several minutes to install SDK packages, so - # it's worth showing those messages even in non-verbose mode. - if context.verbose or line.startswith('Preparing "Install'): - sys.stdout.write(line) - else: - hidden_output.append(line) + log(line) status = await wait_for(process.wait(), timeout=1) if status == 0: @@ -604,6 +671,10 @@ def package(context): print(f"Wrote {package_path}") +def env(context): + print_env(android_env(getattr(context, "host", None))) + + # Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated # by the buildbot worker, we'll make an attempt to clean up our subprocesses. def install_signal_handler(): @@ -615,36 +686,41 @@ def signal_handler(*args): def parse_args(): parser = argparse.ArgumentParser() - subcommands = parser.add_subparsers(dest="subcommand") + subcommands = parser.add_subparsers(dest="subcommand", required=True) # Subcommands - build = subcommands.add_parser("build", help="Build everything") - configure_build = subcommands.add_parser("configure-build", - help="Run `configure` for the " - "build Python") - make_build = subcommands.add_parser("make-build", - help="Run `make` for the build Python") - configure_host = subcommands.add_parser("configure-host", - help="Run `configure` for Android") - make_host = subcommands.add_parser("make-host", - help="Run `make` for Android") + build = subcommands.add_parser( + "build", help="Run configure-build, make-build, configure-host and " + "make-host") + configure_build = subcommands.add_parser( + "configure-build", help="Run `configure` for the build Python") subcommands.add_parser( - "clean", help="Delete all build and prefix directories") - subcommands.add_parser( - "build-testbed", help="Build the testbed app") - test = subcommands.add_parser( - "test", help="Run the test suite") + "make-build", help="Run `make` for the build Python") + configure_host = subcommands.add_parser( + "configure-host", help="Run `configure` for Android") + make_host = subcommands.add_parser( + "make-host", help="Run `make` for Android") + + subcommands.add_parser("clean", help="Delete all build directories") + subcommands.add_parser("build-testbed", help="Build the testbed app") + test = subcommands.add_parser("test", help="Run the testbed app") package = subcommands.add_parser("package", help="Make a release package") + env = subcommands.add_parser("env", help="Print environment variables") # Common arguments for subcommand in build, configure_build, configure_host: subcommand.add_argument( "--clean", action="store_true", default=False, dest="clean", - help="Delete the relevant build and prefix directories first") - for subcommand in [build, configure_host, make_host, package]: + help="Delete the relevant build directories first") + + host_commands = [build, configure_host, make_host, package] + if in_source_tree: + host_commands.append(env) + for subcommand in host_commands: subcommand.add_argument( "host", metavar="HOST", choices=HOSTS, help="Host triplet: choices=[%(choices)s]") + for subcommand in build, configure_build, configure_host: subcommand.add_argument("args", nargs="*", help="Extra arguments to pass to `configure`") @@ -654,6 +730,7 @@ def parse_args(): "-v", "--verbose", action="count", default=0, help="Show Gradle output, and non-Python logcat messages. " "Use twice to include high-volume messages which are rarely useful.") + device_group = test.add_mutually_exclusive_group(required=True) device_group.add_argument( "--connected", metavar="SERIAL", help="Run on a connected device. " @@ -661,8 +738,24 @@ def parse_args(): device_group.add_argument( "--managed", metavar="NAME", help="Run on a Gradle-managed device. " "These are defined in `managedDevices` in testbed/app/build.gradle.kts.") + + test.add_argument( + "--site-packages", metavar="DIR", type=abspath, + help="Directory to copy as the app's site-packages.") test.add_argument( - "args", nargs="*", help=f"Arguments for `python -m test`. " + "--cwd", metavar="DIR", type=abspath, + help="Directory to copy as the app's working directory.") + + mode_group = test.add_mutually_exclusive_group() + mode_group.add_argument( + "-c", dest="command", help="Execute the given Python code.") + mode_group.add_argument( + "-m", dest="module", help="Execute the module with the given name.") + test.epilog = ( + "If neither -c nor -m are passed, the default is '-m test', which will " + "run Python's own test suite.") + test.add_argument( + "args", nargs="*", help=f"Arguments to add to sys.argv. " f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.") return parser.parse_args() @@ -688,6 +781,7 @@ def main(): "build-testbed": build_testbed, "test": run_testbed, "package": package, + "env": env, } try: @@ -708,14 +802,9 @@ def print_called_process_error(e): if not content.endswith("\n"): stream.write("\n") - # Format the command so it can be copied into a shell. shlex uses single - # quotes, so we surround the whole command with double quotes. - args_joined = ( - e.cmd if isinstance(e.cmd, str) - else " ".join(shlex.quote(str(arg)) for arg in e.cmd) - ) + # shlex uses single quotes, so we surround the command with double quotes. print( - f'Command "{args_joined}" returned exit status {e.returncode}' + f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}' ) diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts index c627cb1b0e0b22..92cffd61f86876 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Android/testbed/app/build.gradle.kts @@ -85,7 +85,7 @@ android { minSdk = androidEnvFile.useLines { for (line in it) { - """api_level:=(\d+)""".toRegex().find(line)?.let { + """ANDROID_API_LEVEL:=(\d+)""".toRegex().find(line)?.let { return@useLines it.groupValues[1].toInt() } } @@ -205,11 +205,29 @@ androidComponents.onVariants { variant -> into("site-packages") { from("$projectDir/src/main/python") + + val sitePackages = findProperty("python.sitePackages") as String? + if (!sitePackages.isNullOrEmpty()) { + if (!file(sitePackages).exists()) { + throw GradleException("$sitePackages does not exist") + } + from(sitePackages) + } } duplicatesStrategy = DuplicatesStrategy.EXCLUDE exclude("**/__pycache__") } + + into("cwd") { + val cwd = findProperty("python.cwd") as String? + if (!cwd.isNullOrEmpty()) { + if (!file(cwd).exists()) { + throw GradleException("$cwd does not exist") + } + from(cwd) + } + } } } diff --git a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt index 0e888ab71d87da..94be52dd2dc870 100644 --- a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt +++ b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt @@ -17,11 +17,11 @@ class PythonSuite { fun testPython() { val start = System.currentTimeMillis() try { - val context = + val status = PythonTestRunner( InstrumentationRegistry.getInstrumentation().targetContext - val args = - InstrumentationRegistry.getArguments().getString("pythonArgs", "") - val status = PythonTestRunner(context).run(args) + ).run( + InstrumentationRegistry.getArguments() + ) assertEquals(0, status) } finally { // Make sure the process lives long enough for the test script to diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt index c4bf6cbe83d8cd..ef28948486fb52 100644 --- a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -15,17 +15,29 @@ class MainActivity : AppCompatActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) setContentView(R.layout.activity_main) - val status = PythonTestRunner(this).run("-W -uall") + val status = PythonTestRunner(this).run("-m", "test", "-W -uall") findViewById(R.id.tvHello).text = "Exit status $status" } } class PythonTestRunner(val context: Context) { - /** @param args Extra arguments for `python -m test`. - * @return The Python exit status: zero if the tests passed, nonzero if - * they failed. */ - fun run(args: String = "") : Int { + fun run(instrumentationArgs: Bundle) = run( + instrumentationArgs.getString("pythonMode")!!, + instrumentationArgs.getString("pythonModule")!!, + instrumentationArgs.getString("pythonArgs") ?: "", + ) + + /** Run Python. + * + * @param mode Either "-c" or "-m". + * @param module Python statements for "-c" mode, or a module name for + * "-m" mode. + * @param args Arguments to add to sys.argv. Will be parsed by `shlex.split`. + * @return The Python exit status: zero on success, nonzero on failure. */ + fun run(mode: String, module: String, args: String) : Int { + Os.setenv("PYTHON_MODE", mode, true) + Os.setenv("PYTHON_MODULE", module, true) Os.setenv("PYTHON_ARGS", args, true) // Python needs this variable to help it find the temporary directory, @@ -36,8 +48,9 @@ class PythonTestRunner(val context: Context) { System.loadLibrary("main_activity") redirectStdioToLogcat() - // The main module is in src/main/python/main.py. - return runPython(pythonHome.toString(), "main") + // The main module is in src/main/python. We don't simply call it + // "main", as that could clash with third-party test code. + return runPython(pythonHome.toString(), "android_testbed_main") } private fun extractAssets() : File { diff --git a/Android/testbed/app/src/main/python/main.py b/Android/testbed/app/src/main/python/android_testbed_main.py similarity index 68% rename from Android/testbed/app/src/main/python/main.py rename to Android/testbed/app/src/main/python/android_testbed_main.py index d6941b14412fcc..31b8e5343a8449 100644 --- a/Android/testbed/app/src/main/python/main.py +++ b/Android/testbed/app/src/main/python/android_testbed_main.py @@ -26,7 +26,23 @@ # test_signals in test_threadsignals.py. signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) +mode = os.environ["PYTHON_MODE"] +module = os.environ["PYTHON_MODULE"] sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"]) -# The test module will call sys.exit to indicate whether the tests passed. -runpy.run_module("test") +cwd = f"{sys.prefix}/cwd" +if not os.path.exists(cwd): + # Empty directories are lost in the asset packing/unpacking process. + os.mkdir(cwd) +os.chdir(cwd) + +if mode == "-c": + # In -c mode, sys.path starts with an empty string, which means whatever the current + # working directory is at the moment of each import. + sys.path.insert(0, "") + exec(module, {}) +elif mode == "-m": + sys.path.insert(0, os.getcwd()) + runpy.run_module(module, run_name="__main__", alter_sys=True) +else: + raise ValueError(f"unknown mode: {mode}") diff --git a/Android/testbed/build.gradle.kts b/Android/testbed/build.gradle.kts index 4d1d6f87594da3..451517b3f1aeab 100644 --- a/Android/testbed/build.gradle.kts +++ b/Android/testbed/build.gradle.kts @@ -1,5 +1,5 @@ // Top-level build file where you can add configuration options common to all sub-projects/modules. plugins { - id("com.android.application") version "8.6.1" apply false + id("com.android.application") version "8.10.0" apply false id("org.jetbrains.kotlin.android") version "1.9.22" apply false } diff --git a/Android/testbed/gradle/wrapper/gradle-wrapper.properties b/Android/testbed/gradle/wrapper/gradle-wrapper.properties index 36529c896426b0..5d42fbae084da1 100644 --- a/Android/testbed/gradle/wrapper/gradle-wrapper.properties +++ b/Android/testbed/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ #Mon Feb 19 20:29:06 GMT 2024 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/Doc/using/android.rst b/Doc/using/android.rst index 65bf23dc994856..cb762310328f1c 100644 --- a/Doc/using/android.rst +++ b/Doc/using/android.rst @@ -63,3 +63,12 @@ link to the relevant file. * Add code to your app to :source:`start Python in embedded mode `. This will need to be C code called via JNI. + +Building a Python package for Android +------------------------------------- + +Python packages can be built for Android as wheels and released on PyPI. The +recommended tool for doing this is `cibuildwheel +`__, which automates +all the details of setting up a cross-compilation environment, building the +wheel, and testing it on an emulator. From 3d396ab7591d544ac8bc1fb49615b4e867ca1c83 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Jun 2025 11:17:03 +0200 Subject: [PATCH 086/116] gh-135124: Change stdout errors in regrtest worker process (#135138) Set sys.stdout encoder error handler to backslashreplace in regrtest workers to avoid UnicodeEncodeError when printing a traceback or any other non-encodable character. Move the code from the Regrtest class to setup_process(). Call setup_process() earlier, before displaying regrtest headers. --- Lib/test/libregrtest/main.py | 7 +----- Lib/test/libregrtest/setup.py | 9 +++++++ Lib/test/test_regrtest.py | 47 ++++++++++++++++++++++++++++++----- 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 713cbedb299706..0d9c059a93872d 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -543,8 +543,6 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: self.first_runtests = runtests self.logger.set_tests(runtests) - setup_process() - if (runtests.hunt_refleak is not None) and (not self.num_workers): # gh-109739: WindowsLoadTracker thread interferes with refleak check use_load_tracker = False @@ -721,10 +719,7 @@ def _add_python_opts(self) -> None: self._execute_python(cmd, environ) def _init(self): - # Set sys.stdout encoder error handler to backslashreplace, - # similar to sys.stderr error handler, to avoid UnicodeEncodeError - # when printing a traceback or any other non-encodable character. - sys.stdout.reconfigure(errors="backslashreplace") + setup_process() if self.junit_filename and not os.path.isabs(self.junit_filename): self.junit_filename = os.path.abspath(self.junit_filename) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index c3d1f60a400665..9bfc414cd615c8 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -1,5 +1,6 @@ import faulthandler import gc +import io import os import random import signal @@ -52,6 +53,14 @@ def setup_process() -> None: support.record_original_stdout(sys.stdout) + # Set sys.stdout encoder error handler to backslashreplace, + # similar to sys.stderr error handler, to avoid UnicodeEncodeError + # when printing a traceback or any other non-encodable character. + # + # Use an assertion to fix mypy error. + assert isinstance(sys.stdout, io.TextIOWrapper) + sys.stdout.reconfigure(errors="backslashreplace") + # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some # imports might fail. This affects only the modules imported before os.chdir(). diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index f3ac301686b9fc..a43d2678ebd3be 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -768,13 +768,16 @@ def run_command(self, args, input=None, exitcode=0, **kw): self.fail(msg) return proc - def run_python(self, args, **kw): + def run_python(self, args, isolated=True, **kw): extraargs = [] if 'uops' in sys._xoptions: # Pass -X uops along extraargs.extend(['-X', 'uops']) - args = [sys.executable, *extraargs, '-X', 'faulthandler', '-I', *args] - proc = self.run_command(args, **kw) + cmd = [sys.executable, *extraargs, '-X', 'faulthandler'] + if isolated: + cmd.append('-I') + cmd.extend(args) + proc = self.run_command(cmd, **kw) return proc.stdout @@ -831,8 +834,8 @@ def check_output(self, output): self.check_executed_tests(output, self.tests, randomize=True, stats=len(self.tests)) - def run_tests(self, args, env=None): - output = self.run_python(args, env=env) + def run_tests(self, args, env=None, isolated=True): + output = self.run_python(args, env=env, isolated=isolated) self.check_output(output) def test_script_regrtest(self): @@ -2276,7 +2279,6 @@ def test_pass(self): def test_xml(self): code = textwrap.dedent(r""" import unittest - from test import support class VerboseTests(unittest.TestCase): def test_failed(self): @@ -2311,6 +2313,39 @@ def test_failed(self): for out in testcase.iter('system-out'): self.assertEqual(out.text, r"abc \x1b def") + def test_nonascii(self): + code = textwrap.dedent(r""" + import unittest + + class NonASCIITests(unittest.TestCase): + def test_docstring(self): + '''docstring:\u20ac''' + + def test_subtest(self): + with self.subTest(param='subtest:\u20ac'): + pass + + def test_skip(self): + self.skipTest('skipped:\u20ac') + """) + testname = self.create_test(code=code) + + env = dict(os.environ) + env['PYTHONIOENCODING'] = 'ascii' + + def check(output): + self.check_executed_tests(output, testname, stats=TestStats(3, 0, 1)) + self.assertIn(r'docstring:\u20ac', output) + self.assertIn(r'skipped:\u20ac', output) + + # Run sequentially + output = self.run_tests('-v', testname, env=env, isolated=False) + check(output) + + # Run in parallel + output = self.run_tests('-j1', '-v', testname, env=env, isolated=False) + check(output) + class TestUtils(unittest.TestCase): def test_format_duration(self): From 4b44b3409ac026e7f13054a3daa18ab7ee14d85c Mon Sep 17 00:00:00 2001 From: Emma Smith Date: Thu, 5 Jun 2025 04:31:49 -0700 Subject: [PATCH 087/116] gh-134938: Add set_pledged_input_size() to ZstdCompressor (GH-135010) --- Doc/library/compression.zstd.rst | 35 ++++++++- Lib/test/test_zstd.py | 109 ++++++++++++++++++++++++++++ Modules/_zstd/_zstdmodule.c | 3 + Modules/_zstd/_zstdmodule.h | 1 + Modules/_zstd/clinic/compressor.c.h | 41 ++++++++++- Modules/_zstd/compressor.c | 98 +++++++++++++++++++++++++ 6 files changed, 285 insertions(+), 2 deletions(-) diff --git a/Doc/library/compression.zstd.rst b/Doc/library/compression.zstd.rst index 35bcbc2bfd8eac..57ad8e3377fc67 100644 --- a/Doc/library/compression.zstd.rst +++ b/Doc/library/compression.zstd.rst @@ -247,6 +247,27 @@ Compressing and decompressing data in memory The *mode* argument is a :class:`ZstdCompressor` attribute, either :attr:`~.FLUSH_BLOCK`, or :attr:`~.FLUSH_FRAME`. + .. method:: set_pledged_input_size(size) + + Specify the amount of uncompressed data *size* that will be provided for + the next frame. *size* will be written into the frame header of the next + frame unless :attr:`CompressionParameter.content_size_flag` is ``False`` + or ``0``. A size of ``0`` means that the frame is empty. If *size* is + ``None``, the frame header will omit the frame size. Frames that include + the uncompressed data size require less memory to decompress, especially + at higher compression levels. + + If :attr:`last_mode` is not :attr:`FLUSH_FRAME`, a + :exc:`ValueError` is raised as the compressor is not at the start of + a frame. If the pledged size does not match the actual size of data + provided to :meth:`.compress`, future calls to :meth:`!compress` or + :meth:`flush` may raise :exc:`ZstdError` and the last chunk of data may + be lost. + + After :meth:`flush` or :meth:`.compress` are called with mode + :attr:`FLUSH_FRAME`, the next frame will not include the frame size into + the header unless :meth:`!set_pledged_input_size` is called again. + .. attribute:: CONTINUE Collect more data for compression, which may or may not generate output @@ -266,6 +287,13 @@ Compressing and decompressing data in memory :meth:`~.compress` will be written into a new frame and *cannot* reference past data. + .. attribute:: last_mode + + The last mode passed to either :meth:`~.compress` or :meth:`~.flush`. + The value can be one of :attr:`~.CONTINUE`, :attr:`~.FLUSH_BLOCK`, or + :attr:`~.FLUSH_FRAME`. The initial value is :attr:`~.FLUSH_FRAME`, + signifying that the compressor is at the start of a new frame. + .. class:: ZstdDecompressor(zstd_dict=None, options=None) @@ -620,12 +648,17 @@ Advanced parameter control Write the size of the data to be compressed into the Zstandard frame header when known prior to compressing. - This flag only takes effect under the following two scenarios: + This flag only takes effect under the following scenarios: * Calling :func:`compress` for one-shot compression * Providing all of the data to be compressed in the frame in a single :meth:`ZstdCompressor.compress` call, with the :attr:`ZstdCompressor.FLUSH_FRAME` mode. + * Calling :meth:`ZstdCompressor.set_pledged_input_size` with the exact + amount of data that will be provided to the compressor prior to any + calls to :meth:`ZstdCompressor.compress` for the current frame. + :meth:`!ZstdCompressor.set_pledged_input_size` must be called for each + new frame. All other compression calls may not write the size information into the frame header. diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index e475d9346b9594..14a09a886046f7 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -395,6 +395,115 @@ def test_compress_empty(self): c = ZstdCompressor() self.assertNotEqual(c.compress(b'', c.FLUSH_FRAME), b'') + def test_set_pledged_input_size(self): + DAT = DECOMPRESSED_100_PLUS_32KB + CHUNK_SIZE = len(DAT) // 3 + + # wrong value + c = ZstdCompressor() + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(-300) + # overflow + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(2**64) + # ZSTD_CONTENTSIZE_ERROR is invalid + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(2**64-2) + # ZSTD_CONTENTSIZE_UNKNOWN should use None + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(2**64-1) + + # check valid values are settable + c.set_pledged_input_size(2**63) + c.set_pledged_input_size(2**64-3) + + # check that zero means empty frame + c = ZstdCompressor(level=1) + c.set_pledged_input_size(0) + c.compress(b'') + dat = c.flush() + ret = get_frame_info(dat) + self.assertEqual(ret.decompressed_size, 0) + + + # wrong mode + c = ZstdCompressor(level=1) + c.compress(b'123456') + self.assertEqual(c.last_mode, c.CONTINUE) + with self.assertRaisesRegex(ValueError, + r'last_mode == FLUSH_FRAME'): + c.set_pledged_input_size(300) + + # None value + c = ZstdCompressor(level=1) + c.set_pledged_input_size(None) + dat = c.compress(DAT) + c.flush() + + ret = get_frame_info(dat) + self.assertEqual(ret.decompressed_size, None) + + # correct value + c = ZstdCompressor(level=1) + c.set_pledged_input_size(len(DAT)) + + chunks = [] + posi = 0 + while posi < len(DAT): + dat = c.compress(DAT[posi:posi+CHUNK_SIZE]) + posi += CHUNK_SIZE + chunks.append(dat) + + dat = c.flush() + chunks.append(dat) + chunks = b''.join(chunks) + + ret = get_frame_info(chunks) + self.assertEqual(ret.decompressed_size, len(DAT)) + self.assertEqual(decompress(chunks), DAT) + + c.set_pledged_input_size(len(DAT)) # the second frame + dat = c.compress(DAT) + c.flush() + + ret = get_frame_info(dat) + self.assertEqual(ret.decompressed_size, len(DAT)) + self.assertEqual(decompress(dat), DAT) + + # not enough data + c = ZstdCompressor(level=1) + c.set_pledged_input_size(len(DAT)+1) + + for start in range(0, len(DAT), CHUNK_SIZE): + end = min(start+CHUNK_SIZE, len(DAT)) + _dat = c.compress(DAT[start:end]) + + with self.assertRaises(ZstdError): + c.flush() + + # too much data + c = ZstdCompressor(level=1) + c.set_pledged_input_size(len(DAT)) + + for start in range(0, len(DAT), CHUNK_SIZE): + end = min(start+CHUNK_SIZE, len(DAT)) + _dat = c.compress(DAT[start:end]) + + with self.assertRaises(ZstdError): + c.compress(b'extra', ZstdCompressor.FLUSH_FRAME) + + # content size not set if content_size_flag == 0 + c = ZstdCompressor(options={CompressionParameter.content_size_flag: 0}) + c.set_pledged_input_size(10) + dat1 = c.compress(b"hello") + dat2 = c.compress(b"world") + dat3 = c.flush() + frame_data = get_frame_info(dat1 + dat2 + dat3) + self.assertIsNone(frame_data.decompressed_size) + + class DecompressorTestCase(unittest.TestCase): def test_simple_decompress_bad_args(self): diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index b0e50f873f4ca6..d75c0779474a82 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -72,6 +72,9 @@ set_zstd_error(const _zstd_state *state, error_type type, size_t zstd_ret) case ERR_COMPRESS: msg = "Unable to compress Zstandard data: %s"; break; + case ERR_SET_PLEDGED_INPUT_SIZE: + msg = "Unable to set pledged uncompressed content size: %s"; + break; case ERR_LOAD_D_DICT: msg = "Unable to load Zstandard dictionary or prefix for " diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index c73f15b3c5299b..4e8f708f2232c7 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -27,6 +27,7 @@ typedef struct { typedef enum { ERR_DECOMPRESS, ERR_COMPRESS, + ERR_SET_PLEDGED_INPUT_SIZE, ERR_LOAD_D_DICT, ERR_LOAD_C_DICT, diff --git a/Modules/_zstd/clinic/compressor.c.h b/Modules/_zstd/clinic/compressor.c.h index f69161b590e5b7..4f8d93fd9e867c 100644 --- a/Modules/_zstd/clinic/compressor.c.h +++ b/Modules/_zstd/clinic/compressor.c.h @@ -252,4 +252,43 @@ _zstd_ZstdCompressor_flush(PyObject *self, PyObject *const *args, Py_ssize_t nar exit: return return_value; } -/*[clinic end generated code: output=ee2d1dc298de790c input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_zstd_ZstdCompressor_set_pledged_input_size__doc__, +"set_pledged_input_size($self, size, /)\n" +"--\n" +"\n" +"Set the uncompressed content size to be written into the frame header.\n" +"\n" +" size\n" +" The size of the uncompressed data to be provided to the compressor.\n" +"\n" +"This method can be used to ensure the header of the frame about to be written\n" +"includes the size of the data, unless the CompressionParameter.content_size_flag\n" +"is set to False. If last_mode != FLUSH_FRAME, then a RuntimeError is raised.\n" +"\n" +"It is important to ensure that the pledged data size matches the actual data\n" +"size. If they do not match the compressed output data may be corrupted and the\n" +"final chunk written may be lost."); + +#define _ZSTD_ZSTDCOMPRESSOR_SET_PLEDGED_INPUT_SIZE_METHODDEF \ + {"set_pledged_input_size", (PyCFunction)_zstd_ZstdCompressor_set_pledged_input_size, METH_O, _zstd_ZstdCompressor_set_pledged_input_size__doc__}, + +static PyObject * +_zstd_ZstdCompressor_set_pledged_input_size_impl(ZstdCompressor *self, + unsigned long long size); + +static PyObject * +_zstd_ZstdCompressor_set_pledged_input_size(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + unsigned long long size; + + if (!zstd_contentsize_converter(arg, &size)) { + goto exit; + } + return_value = _zstd_ZstdCompressor_set_pledged_input_size_impl((ZstdCompressor *)self, size); + +exit: + return return_value; +} +/*[clinic end generated code: output=c1d5c2cf06a8becd input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index e1217635f60cb0..bc9e6eff89af68 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -45,6 +45,52 @@ typedef struct { #define ZstdCompressor_CAST(op) ((ZstdCompressor *)op) +/*[python input] + +class zstd_contentsize_converter(CConverter): + type = 'unsigned long long' + converter = 'zstd_contentsize_converter' + +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=0932c350d633c7de]*/ + + +static int +zstd_contentsize_converter(PyObject *size, unsigned long long *p) +{ + // None means the user indicates the size is unknown. + if (size == Py_None) { + *p = ZSTD_CONTENTSIZE_UNKNOWN; + } + else { + /* ZSTD_CONTENTSIZE_UNKNOWN is 0ULL - 1 + ZSTD_CONTENTSIZE_ERROR is 0ULL - 2 + Users should only pass values < ZSTD_CONTENTSIZE_ERROR */ + unsigned long long pledged_size = PyLong_AsUnsignedLongLong(size); + /* Here we check for (unsigned long long)-1 as a sign of an error in + PyLong_AsUnsignedLongLong */ + if (pledged_size == (unsigned long long)-1 && PyErr_Occurred()) { + *p = ZSTD_CONTENTSIZE_ERROR; + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_Format(PyExc_ValueError, + "size argument should be a positive int less " + "than %ull", ZSTD_CONTENTSIZE_ERROR); + return 0; + } + return 0; + } + if (pledged_size >= ZSTD_CONTENTSIZE_ERROR) { + *p = ZSTD_CONTENTSIZE_ERROR; + PyErr_Format(PyExc_ValueError, + "size argument should be a positive int less " + "than %ull", ZSTD_CONTENTSIZE_ERROR); + return 0; + } + *p = pledged_size; + } + return 1; +} + #include "clinic/compressor.c.h" static int @@ -643,9 +689,61 @@ _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) return ret; } + +/*[clinic input] +_zstd.ZstdCompressor.set_pledged_input_size + + size: zstd_contentsize + The size of the uncompressed data to be provided to the compressor. + / + +Set the uncompressed content size to be written into the frame header. + +This method can be used to ensure the header of the frame about to be written +includes the size of the data, unless the CompressionParameter.content_size_flag +is set to False. If last_mode != FLUSH_FRAME, then a RuntimeError is raised. + +It is important to ensure that the pledged data size matches the actual data +size. If they do not match the compressed output data may be corrupted and the +final chunk written may be lost. +[clinic start generated code]*/ + +static PyObject * +_zstd_ZstdCompressor_set_pledged_input_size_impl(ZstdCompressor *self, + unsigned long long size) +/*[clinic end generated code: output=3a09e55cc0e3b4f9 input=afd8a7d78cff2eb5]*/ +{ + // Error occured while converting argument, should be unreachable + assert(size != ZSTD_CONTENTSIZE_ERROR); + + /* Thread-safe code */ + PyMutex_Lock(&self->lock); + + /* Check the current mode */ + if (self->last_mode != ZSTD_e_end) { + PyErr_SetString(PyExc_ValueError, + "set_pledged_input_size() method must be called " + "when last_mode == FLUSH_FRAME"); + PyMutex_Unlock(&self->lock); + return NULL; + } + + /* Set pledged content size */ + size_t zstd_ret = ZSTD_CCtx_setPledgedSrcSize(self->cctx, size); + PyMutex_Unlock(&self->lock); + if (ZSTD_isError(zstd_ret)) { + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); + set_zstd_error(mod_state, ERR_SET_PLEDGED_INPUT_SIZE, zstd_ret); + return NULL; + } + + Py_RETURN_NONE; +} + static PyMethodDef ZstdCompressor_methods[] = { _ZSTD_ZSTDCOMPRESSOR_COMPRESS_METHODDEF _ZSTD_ZSTDCOMPRESSOR_FLUSH_METHODDEF + _ZSTD_ZSTDCOMPRESSOR_SET_PLEDGED_INPUT_SIZE_METHODDEF {NULL, NULL} }; From 9258f3da9175134d03f2c8c7c7eed223802ad945 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Jun 2025 14:43:47 +0200 Subject: [PATCH 088/116] gh-134989: Fix Py_RETURN_NONE in the limited C API (GH-135165) Fix Py_RETURN_NONE, Py_RETURN_TRUE and Py_RETURN_FALSE macros in the limited C API 3.11 and older: Don't treat Py_None, Py_True and Py_False as immortal. --- Include/boolobject.h | 13 ++++++++++--- Include/object.h | 9 +++++++-- .../2025-06-05-11-06-07.gh-issue-134989.74p4ud.rst | 3 +++ 3 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-06-05-11-06-07.gh-issue-134989.74p4ud.rst diff --git a/Include/boolobject.h b/Include/boolobject.h index 3037e61bbf6d0c..b56e2baecaa36c 100644 --- a/Include/boolobject.h +++ b/Include/boolobject.h @@ -34,9 +34,16 @@ PyAPI_FUNC(int) Py_IsTrue(PyObject *x); PyAPI_FUNC(int) Py_IsFalse(PyObject *x); #define Py_IsFalse(x) Py_Is((x), Py_False) -/* Macros for returning Py_True or Py_False, respectively */ -#define Py_RETURN_TRUE return Py_True -#define Py_RETURN_FALSE return Py_False +/* Macros for returning Py_True or Py_False, respectively. + * Only treat Py_True and Py_False as immortal in the limited C API 3.12 + * and newer. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030c0000 +# define Py_RETURN_TRUE return Py_NewRef(Py_True) +# define Py_RETURN_FALSE return Py_NewRef(Py_False) +#else +# define Py_RETURN_TRUE return Py_True +# define Py_RETURN_FALSE return Py_False +#endif /* Function to return a bool from a C long */ PyAPI_FUNC(PyObject *) PyBool_FromLong(long); diff --git a/Include/object.h b/Include/object.h index 994cac1ad17501..42aed614d4ad8e 100644 --- a/Include/object.h +++ b/Include/object.h @@ -660,8 +660,13 @@ PyAPI_DATA(PyObject) _Py_NoneStruct; /* Don't use this directly */ PyAPI_FUNC(int) Py_IsNone(PyObject *x); #define Py_IsNone(x) Py_Is((x), Py_None) -/* Macro for returning Py_None from a function */ -#define Py_RETURN_NONE return Py_None +/* Macro for returning Py_None from a function. + * Only treat Py_None as immortal in the limited C API 3.12 and newer. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030c0000 +# define Py_RETURN_NONE return Py_NewRef(Py_None) +#else +# define Py_RETURN_NONE return Py_None +#endif /* Py_NotImplemented is a singleton used to signal that an operation is diff --git a/Misc/NEWS.d/next/C_API/2025-06-05-11-06-07.gh-issue-134989.74p4ud.rst b/Misc/NEWS.d/next/C_API/2025-06-05-11-06-07.gh-issue-134989.74p4ud.rst new file mode 100644 index 00000000000000..844e9a666640c6 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-06-05-11-06-07.gh-issue-134989.74p4ud.rst @@ -0,0 +1,3 @@ +Fix ``Py_RETURN_NONE``, ``Py_RETURN_TRUE`` and ``Py_RETURN_FALSE`` macros in +the limited C API 3.11 and older: don't treat ``Py_None``, ``Py_True`` and +``Py_False`` as immortal. Patch by Victor Stinner. From 8919cb4ad9fb18832a4bc9d5bbea305e9518c7ab Mon Sep 17 00:00:00 2001 From: rialbat <47256826+rialbat@users.noreply.github.com> Date: Thu, 5 Jun 2025 18:08:48 +0300 Subject: [PATCH 089/116] gh-135161: Remove redundant NULL check for 'exc' after dereference in ceval.c (#135162) --- Python/ceval.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ceval.c b/Python/ceval.c index 7aec196cb85704..5ea837e1a6ef31 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3190,7 +3190,7 @@ _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwarg else if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { PyObject *exc = _PyErr_GetRaisedException(tstate); PyObject *args = PyException_GetArgs(exc); - if (exc && PyTuple_Check(args) && PyTuple_GET_SIZE(args) == 1) { + if (PyTuple_Check(args) && PyTuple_GET_SIZE(args) == 1) { _PyErr_Clear(tstate); PyObject *funcstr = _PyObject_FunctionStr(func); if (funcstr != NULL) { From 1b55e12766d007aea9fcd0966e29ce220b67d28e Mon Sep 17 00:00:00 2001 From: Weipeng Hong Date: Fri, 6 Jun 2025 00:00:07 +0800 Subject: [PATCH 090/116] gh-135166: Fix exception type expected by test.test_zstd (GH-135167) --- Lib/test/test_zstd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index 14a09a886046f7..d4c28aed38ef90 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -293,11 +293,11 @@ def test_compress_parameters(self): # zstd lib doesn't support MT compression if not SUPPORT_MULTITHREADING: - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdCompressor(options={CompressionParameter.nb_workers:4}) - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdCompressor(options={CompressionParameter.job_size:4}) - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdCompressor(options={CompressionParameter.overlap_log:4}) # out of bounds error msg From d9cad074d52fe31327429fd81e4d2eeea3dbe35b Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Thu, 5 Jun 2025 10:28:11 -0700 Subject: [PATCH 091/116] gh-134155: fix AttributeError in email._header_value_parser.get_address (#134194) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Append the defect to defects instead of to the parse tree. Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Lib/email/_header_value_parser.py | 4 +- .../test_email/test__header_value_parser.py | 45 +++++++++++++++++++ ...-05-18-23-46-21.gh-issue-134152.30HwbX.rst | 1 + 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-18-23-46-21.gh-issue-134152.30HwbX.rst diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index f11fa83d45ed2d..91243378dc0441 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -1575,7 +1575,7 @@ def get_dtext(value): def _check_for_early_dl_end(value, domain_literal): if value: return False - domain_literal.append(errors.InvalidHeaderDefect( + domain_literal.defects.append(errors.InvalidHeaderDefect( "end of input inside domain-literal")) domain_literal.append(ValueTerminal(']', 'domain-literal-end')) return True @@ -1594,9 +1594,9 @@ def get_domain_literal(value): raise errors.HeaderParseError("expected '[' at start of domain-literal " "but found '{}'".format(value)) value = value[1:] + domain_literal.append(ValueTerminal('[', 'domain-literal-start')) if _check_for_early_dl_end(value, domain_literal): return domain_literal, value - domain_literal.append(ValueTerminal('[', 'domain-literal-start')) if value[0] in WSP: token, value = get_fws(value) domain_literal.append(token) diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index fd4ac2c404ce47..179e236ecdfd7f 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -2491,6 +2491,38 @@ def test_get_address_quoted_strings_in_atom_list(self): self.assertEqual(address.all_mailboxes[0].domain, 'example.com') self.assertEqual(address.all_mailboxes[0].addr_spec, '"example example"@example.com') + def test_get_address_with_invalid_domain(self): + address = self._test_get_x(parser.get_address, + '', + '', + [errors.InvalidHeaderDefect, # missing trailing '>' on angle-addr + errors.InvalidHeaderDefect, # end of input inside domain-literal + ], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 0) + self.assertEqual(len(address.all_mailboxes), 1) + self.assertEqual(address.all_mailboxes[0].domain, '[]') + self.assertEqual(address.all_mailboxes[0].local_part, 'T') + self.assertEqual(address.all_mailboxes[0].token_type, 'invalid-mailbox') + self.assertEqual(address[0].token_type, 'invalid-mailbox') + + address = self._test_get_x(parser.get_address, + '!an??:=m==fr2@[C', + '!an??:=m==fr2@[C];', + '!an??:=m==fr2@[C];', + [errors.InvalidHeaderDefect, # end of header in group + errors.InvalidHeaderDefect, # end of input inside domain-literal + ], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 0) + self.assertEqual(len(address.all_mailboxes), 1) + self.assertEqual(address.all_mailboxes[0].domain, '[C]') + self.assertEqual(address.all_mailboxes[0].local_part, '=m==fr2') + self.assertEqual(address.all_mailboxes[0].token_type, 'invalid-mailbox') + self.assertEqual(address[0].token_type, 'group') # get_address_list @@ -2765,6 +2797,19 @@ def test_parse_valid_message_id(self): ) self.assertEqual(message_id.token_type, 'message-id') + def test_parse_message_id_with_invalid_domain(self): + message_id = self._test_parse_x( + parser.parse_message_id, + "", + "", + [errors.ObsoleteHeaderDefect] + [errors.InvalidHeaderDefect] * 2, + [], + ) + self.assertEqual(message_id.token_type, 'message-id') + self.assertEqual(str(message_id.all_defects[-1]), + "end of input inside domain-literal") + def test_parse_message_id_with_remaining(self): message_id = self._test_parse_x( parser.parse_message_id, diff --git a/Misc/NEWS.d/next/Library/2025-05-18-23-46-21.gh-issue-134152.30HwbX.rst b/Misc/NEWS.d/next/Library/2025-05-18-23-46-21.gh-issue-134152.30HwbX.rst new file mode 100644 index 00000000000000..911a4a59ea6079 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-18-23-46-21.gh-issue-134152.30HwbX.rst @@ -0,0 +1 @@ +:mod:`email`: Fix parsing of email message ID with invalid domain. From b90ecea9e6b33dae360ed7eb2c32598f98444c4d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 5 Jun 2025 18:53:57 +0100 Subject: [PATCH 092/116] GH-132554: Fix tier2 `FOR_ITER` implementation and optimizations (GH-135137) --- Include/internal/pycore_ceval.h | 3 +- Include/internal/pycore_stackref.h | 42 +++++++++++-- Lib/test/test_capi/test_opt.py | 11 ++++ Python/bytecodes.c | 97 +++++++----------------------- Python/ceval.c | 28 ++++++++- Python/executor_cases.c.h | 25 ++------ Python/generated_cases.c.h | 91 +++++++--------------------- Python/optimizer_bytecodes.c | 11 ++++ Python/optimizer_cases.c.h | 12 +++- Python/stackrefs.c | 4 ++ Tools/cases_generator/analyzer.py | 2 + Tools/cases_generator/stack.py | 2 +- 12 files changed, 155 insertions(+), 173 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 092feeb40b04a3..239177deb4a948 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -353,7 +353,8 @@ PyAPI_FUNC(_PyStackRef) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyS extern int _PyRunRemoteDebugger(PyThreadState *tstate); #endif -_PyStackRef _PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index); +PyAPI_FUNC(_PyStackRef) +_PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame* frame, _PyStackRef iter, _PyStackRef *index_ptr); #ifdef __cplusplus } diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index f2ecc30b053568..8791476725289c 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -62,14 +62,15 @@ PyAPI_FUNC(void) _Py_stackref_record_borrow(_PyStackRef ref, const char *filenam extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref); static const _PyStackRef PyStackRef_NULL = { .index = 0 }; +static const _PyStackRef PyStackRef_ERROR = { .index = 2 }; // Use the first 3 even numbers for None, True and False. // Odd numbers are reserved for (tagged) integers -#define PyStackRef_None ((_PyStackRef){ .index = 2 } ) -#define PyStackRef_False ((_PyStackRef){ .index = 4 }) -#define PyStackRef_True ((_PyStackRef){ .index = 6 }) +#define PyStackRef_None ((_PyStackRef){ .index = 4 } ) +#define PyStackRef_False ((_PyStackRef){ .index = 6 }) +#define PyStackRef_True ((_PyStackRef){ .index = 8 }) -#define INITIAL_STACKREF_INDEX 8 +#define INITIAL_STACKREF_INDEX 10 static inline int PyStackRef_IsNull(_PyStackRef ref) @@ -77,6 +78,19 @@ PyStackRef_IsNull(_PyStackRef ref) return ref.index == 0; } +static inline bool +PyStackRef_IsError(_PyStackRef ref) +{ + return ref.index == 2; +} + +static inline bool +PyStackRef_IsValid(_PyStackRef ref) +{ + /* Invalid values are ERROR and NULL */ + return !PyStackRef_IsError(ref) && !PyStackRef_IsNull(ref); +} + static inline int PyStackRef_IsTrue(_PyStackRef ref) { @@ -104,6 +118,7 @@ PyStackRef_IsTaggedInt(_PyStackRef ref) static inline PyObject * _PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber) { + assert(!PyStackRef_IsError(ref)); assert(!PyStackRef_IsTaggedInt(ref)); _Py_stackref_record_borrow(ref, filename, linenumber); return _Py_stackref_get_object(ref); @@ -155,6 +170,7 @@ _PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber) static inline void _PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber) { + assert(!PyStackRef_IsError(ref)); if (PyStackRef_IsNull(ref)) { return; } @@ -165,6 +181,7 @@ _PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber) static inline _PyStackRef _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber) { + assert(!PyStackRef_IsError(ref)); if (PyStackRef_IsTaggedInt(ref)) { return ref; } @@ -241,9 +258,25 @@ PyStackRef_IsNullOrInt(_PyStackRef ref); #else #define Py_INT_TAG 3 +#define Py_TAG_INVALID 2 #define Py_TAG_REFCNT 1 #define Py_TAG_BITS 3 +static const _PyStackRef PyStackRef_ERROR = { .bits = Py_TAG_INVALID }; + +static inline bool +PyStackRef_IsError(_PyStackRef ref) +{ + return ref.bits == Py_TAG_INVALID; +} + +static inline bool +PyStackRef_IsValid(_PyStackRef ref) +{ + /* Invalid values are ERROR and NULL */ + return ref.bits >= Py_INT_TAG; +} + static inline bool PyStackRef_IsTaggedInt(_PyStackRef i) { @@ -284,6 +317,7 @@ PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref) static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED}; + #define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits) #define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED }) #define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED }) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index cb6eae484149ee..a292ebcc7f4aed 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1183,6 +1183,17 @@ def testfunc(n): self.assertIsNotNone(ex) self.assertIn("_RETURN_GENERATOR", get_opnames(ex)) + def test_for_iter(self): + def testfunc(n): + t = 0 + for i in set(range(n)): + t += i + return t + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * (TIER2_THRESHOLD - 1) // 2) + self.assertIsNotNone(ex) + self.assertIn("_FOR_ITER_TIER_TWO", get_opnames(ex)) + @unittest.skip("Tracing into generators currently isn't supported.") def test_for_iter_gen(self): def gen(n): diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f02e32fd1d312a..c4b13da5db41d8 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3125,100 +3125,49 @@ dummy_func( } replaced op(_FOR_ITER, (iter, null_or_index -- iter, null_or_index, next)) { - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (PyStackRef_IsTaggedInt(null_or_index)) { - next = _PyForIter_NextWithIndex(iter_o, null_or_index); - if (PyStackRef_IsNull(next)) { - JUMPBY(oparg + 1); - DISPATCH(); - } - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - } - else { - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - if (!matches) { - ERROR_NO_POP(); - } - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - /* Jump forward oparg, then skip following END_FOR */ - JUMPBY(oparg + 1); - DISPATCH(); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + ERROR_NO_POP(); } - next = PyStackRef_FromPyObjectSteal(next_o); + // Jump forward by oparg and skip the following END_FOR + JUMPBY(oparg + 1); + DISPATCH(); } + next = item; } op(_FOR_ITER_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - EXIT_IF(!PyStackRef_IsNull(null_or_index)); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - if (!matches) { - ERROR_NO_POP(); - } - _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr); - _PyErr_Clear(tstate); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + ERROR_NO_POP(); } /* iterator ended normally */ /* The translator sets the deopt target just past the matching END_FOR */ EXIT_IF(true); } - next = PyStackRef_FromPyObjectSteal(next_o); - // Common case: no jump, leave it to the code generator + next = item; } + macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER; inst(INSTRUMENTED_FOR_ITER, (unused/1, iter, null_or_index -- iter, null_or_index, next)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (PyStackRef_IsTaggedInt(null_or_index)) { - next = _PyForIter_NextWithIndex(iter_o, null_or_index); - if (PyStackRef_IsNull(next)) { - JUMPBY(oparg + 1); - DISPATCH(); - } - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - } - else { - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - if (next_o != NULL) { - next = PyStackRef_FromPyObjectSteal(next_o); - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - } - else { - if (_PyErr_Occurred(tstate)) { - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - if (!matches) { - ERROR_NO_POP(); - } - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - /* Skip END_FOR */ - JUMPBY(oparg + 1); - DISPATCH(); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + ERROR_NO_POP(); } + // Jump forward by oparg and skip the following END_FOR + JUMPBY(oparg + 1); + DISPATCH(); } + next = item; + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } - op(_ITER_CHECK_LIST, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); EXIT_IF(Py_TYPE(iter_o) != &PyList_Type); diff --git a/Python/ceval.c b/Python/ceval.c index 5ea837e1a6ef31..4cfe4bb88f4e48 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3439,8 +3439,8 @@ _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *na return value; } -_PyStackRef -_PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index) +static _PyStackRef +foriter_next(PyObject *seq, _PyStackRef index) { assert(PyStackRef_IsTaggedInt(index)); assert(PyTuple_CheckExact(seq) || PyList_CheckExact(seq)); @@ -3459,6 +3459,30 @@ _PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index) return PyStackRef_FromPyObjectSteal(item); } +_PyStackRef _PyForIter_VirtualIteratorNext(PyThreadState* tstate, _PyInterpreterFrame* frame, _PyStackRef iter, _PyStackRef* index_ptr) +{ + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyStackRef index = *index_ptr; + if (PyStackRef_IsTaggedInt(index)) { + *index_ptr = PyStackRef_IncrementTaggedIntNoOverflow(index); + return foriter_next(iter_o, index); + } + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + if (next_o == NULL) { + if (_PyErr_Occurred(tstate)) { + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr); + _PyErr_Clear(tstate); + } + else { + return PyStackRef_ERROR; + } + } + return PyStackRef_NULL; + } + return PyStackRef_FromPyObjectSteal(next_o); +} + /* Check if a 'cls' provides the given special method. */ static inline int type_has_special_method(PyTypeObject *cls, PyObject *name) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 35b29940cb4a15..d19605169d5e55 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4268,33 +4268,20 @@ _PyStackRef next; null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (!PyStackRef_IsNull(null_or_index)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - JUMP_TO_ERROR(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + JUMP_TO_ERROR(); } if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } } - next = PyStackRef_FromPyObjectSteal(next_o); + next = item; + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e3cd3b71a1de08..c8825df3ade5a5 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5753,41 +5753,18 @@ } // _FOR_ITER { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (PyStackRef_IsTaggedInt(null_or_index)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - next = _PyForIter_NextWithIndex(iter_o, null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(next)) { - JUMPBY(oparg + 1); - DISPATCH(); - } - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - JUMPBY(oparg + 1); - DISPATCH(); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + JUMP_TO_LABEL(error); } - next = PyStackRef_FromPyObjectSteal(next_o); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); } + next = item; } stack_pointer[-1] = null_or_index; stack_pointer[0] = next; @@ -7059,45 +7036,19 @@ /* Skip 1 cache entry */ null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (PyStackRef_IsTaggedInt(null_or_index)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - next = _PyForIter_NextWithIndex(iter_o, null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(next)) { - JUMPBY(oparg + 1); - DISPATCH(); - } - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o != NULL) { - next = PyStackRef_FromPyObjectSteal(next_o); - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - } - else { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - JUMPBY(oparg + 1); - DISPATCH(); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + JUMP_TO_LABEL(error); } + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); } + next = item; + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e1209209660f92..b4220e2c627ecb 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -840,6 +840,17 @@ dummy_func(void) { value = sym_new_unknown(ctx); } + op(_GET_ITER, (iterable -- iter, index_or_null)) { + if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) { + iter = iterable; + index_or_null = sym_new_not_null(ctx); + } + else { + iter = sym_new_not_null(ctx); + index_or_null = sym_new_unknown(ctx); + } + } + op(_FOR_ITER_GEN_FRAME, (unused, unused -- unused, unused, gen_frame: _Py_UOpsAbstractFrame*)) { gen_frame = NULL; /* We are about to hit the end of the trace */ diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index db86edcc7859b5..960c683800455e 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1562,10 +1562,18 @@ } case _GET_ITER: { + JitOptSymbol *iterable; JitOptSymbol *iter; JitOptSymbol *index_or_null; - iter = sym_new_not_null(ctx); - index_or_null = sym_new_not_null(ctx); + iterable = stack_pointer[-1]; + if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) { + iter = iterable; + index_or_null = sym_new_not_null(ctx); + } + else { + iter = sym_new_not_null(ctx); + index_or_null = sym_new_unknown(ctx); + } stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; stack_pointer += 1; diff --git a/Python/stackrefs.c b/Python/stackrefs.c index b2a1369031ad2b..ecc0012ef17b39 100644 --- a/Python/stackrefs.c +++ b/Python/stackrefs.c @@ -40,6 +40,7 @@ make_table_entry(PyObject *obj, const char *filename, int linenumber) PyObject * _Py_stackref_get_object(_PyStackRef ref) { + assert(!PyStackRef_IsError(ref)); if (ref.index == 0) { return NULL; } @@ -64,6 +65,7 @@ PyStackRef_Is(_PyStackRef a, _PyStackRef b) PyObject * _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber) { + assert(!PyStackRef_IsError(ref)); PyInterpreterState *interp = PyInterpreterState_Get(); if (ref.index >= interp->next_stackref) { _Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", (void *)ref.index, filename, linenumber); @@ -128,6 +130,7 @@ _Py_stackref_create(PyObject *obj, const char *filename, int linenumber) void _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber) { + assert(!PyStackRef_IsError(ref)); if (ref.index < INITIAL_STACKREF_INDEX) { return; } @@ -152,6 +155,7 @@ _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref) { + assert(!PyStackRef_IsError(ref)); assert(ref.index < INITIAL_STACKREF_INDEX); TableEntry *entry = make_table_entry(obj, "builtin-object", 0); if (entry == NULL) { diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 3070559db8ae57..1447f365336d82 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -681,6 +681,8 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_UntagInt", "PyStackRef_IncrementTaggedIntNoOverflow", "PyStackRef_IsNullOrInt", + "PyStackRef_IsError", + "PyStackRef_IsValid", ) diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 6b681775f48c81..df168afa88888c 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -496,7 +496,7 @@ def _push_defined_outputs(self) -> None: f"Expected '{undefined}' to be defined before '{out.name}'" else: undefined = out.name - while len(self.outputs) > self.peeks and not self.needs_defining(self.outputs[0]): + while len(self.outputs) > self.peeks and not self.needs_defining(self.outputs[self.peeks]): out = self.outputs.pop(self.peeks) self.stack.push(out) From a7d41e8aab5211f4ed7f636c41d63adcab0affba Mon Sep 17 00:00:00 2001 From: Weipeng Hong Date: Fri, 6 Jun 2025 08:44:25 +0800 Subject: [PATCH 093/116] gh-135155: Added dependencies required for compiling the _zstd module (GH-135156) --- .github/workflows/posix-deps-apt.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index d5538cd9367ec6..7773222af5d26f 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -17,6 +17,7 @@ apt-get -yq install \ libreadline6-dev \ libsqlite3-dev \ libssl-dev \ + libzstd-dev \ lzma \ lzma-dev \ strace \ From 0d9ccc87a2198a0c1881ab4b17a24fc7fec62418 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 6 Jun 2025 03:51:06 +0300 Subject: [PATCH 094/116] gh-134036: Improve error messages for invalid `raise` statements (#134077) --- Grammar/python.gram | 6 + Lib/test/test_syntax.py | 22 + ...-05-16-09-06-38.gh-issue-134036.st2e-B.rst | 2 + Parser/parser.c | 901 ++++++++++-------- 4 files changed, 529 insertions(+), 402 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-16-09-06-38.gh-issue-134036.st2e-B.rst diff --git a/Grammar/python.gram b/Grammar/python.gram index de435537095031..a5ab769910b5ac 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -184,6 +184,7 @@ return_stmt[stmt_ty]: | 'return' a=[star_expressions] { _PyAST_Return(a, EXTRA) } raise_stmt[stmt_ty]: + | invalid_raise_stmt | 'raise' a=expression b=['from' z=expression { z }] { _PyAST_Raise(a, b, EXTRA) } | 'raise' { _PyAST_Raise(NULL, NULL, EXTRA) } @@ -1287,6 +1288,11 @@ invalid_ann_assign_target[expr_ty]: | list | tuple | '(' a=invalid_ann_assign_target ')' { a } +invalid_raise_stmt: + | a='raise' b='from' { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "did you forget an expression between 'raise' and 'from'?") } + | 'raise' expression a='from' { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "did you forget an expression after 'from'?") } invalid_del_stmt: | 'del' a=star_expressions { RAISE_SYNTAX_ERROR_INVALID_TARGET(DEL_TARGETS, a) } diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index c7ac791415830f..965c096475f590 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1695,6 +1695,28 @@ ... SyntaxError: invalid syntax +Better errors for `raise` statement: + + >>> raise ValueError from + Traceback (most recent call last): + SyntaxError: did you forget an expression after 'from'? + + >>> raise mod.ValueError() from + Traceback (most recent call last): + SyntaxError: did you forget an expression after 'from'? + + >>> raise from exc + Traceback (most recent call last): + SyntaxError: did you forget an expression between 'raise' and 'from'? + + >>> raise from None + Traceback (most recent call last): + SyntaxError: did you forget an expression between 'raise' and 'from'? + + >>> raise from + Traceback (most recent call last): + SyntaxError: did you forget an expression between 'raise' and 'from'? + Check that an multiple exception types with missing parentheses raise a custom exception only when using 'as' diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-16-09-06-38.gh-issue-134036.st2e-B.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-16-09-06-38.gh-issue-134036.st2e-B.rst new file mode 100644 index 00000000000000..176aab1c93ab83 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-16-09-06-38.gh-issue-134036.st2e-B.rst @@ -0,0 +1,2 @@ +Improve :exc:`SyntaxError` message when using invalid :keyword:`raise` +statements. diff --git a/Parser/parser.c b/Parser/parser.c index d5aafef826ed3a..82311b4f40eebf 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -21,54 +21,54 @@ static KeywordToken *reserved_keywords[] = { (KeywordToken[]) {{NULL, -1}}, (KeywordToken[]) {{NULL, -1}}, (KeywordToken[]) { - {"if", 682}, - {"as", 680}, - {"in", 695}, + {"if", 686}, + {"as", 684}, + {"in", 699}, {"or", 588}, {"is", 596}, {NULL, -1}, }, (KeywordToken[]) { - {"del", 625}, - {"def", 699}, - {"for", 694}, - {"try", 656}, + {"del", 629}, + {"def", 703}, + {"for", 698}, + {"try", 660}, {"and", 589}, - {"not", 703}, + {"not", 707}, {NULL, -1}, }, (KeywordToken[]) { - {"from", 633}, + {"from", 637}, {"pass", 526}, - {"with", 647}, - {"elif", 687}, - {"else", 686}, + {"with", 651}, + {"elif", 691}, + {"else", 690}, {"None", 623}, {"True", 622}, {NULL, -1}, }, (KeywordToken[]) { - {"raise", 525}, + {"raise", 627}, {"yield", 587}, {"break", 527}, - {"async", 698}, - {"class", 701}, - {"while", 689}, + {"async", 702}, + {"class", 705}, + {"while", 693}, {"False", 624}, {"await", 597}, {NULL, -1}, }, (KeywordToken[]) { {"return", 522}, - {"import", 634}, + {"import", 638}, {"assert", 532}, {"global", 529}, - {"except", 677}, + {"except", 681}, {"lambda", 621}, {NULL, -1}, }, (KeywordToken[]) { - {"finally", 673}, + {"finally", 677}, {NULL, -1}, }, (KeywordToken[]) { @@ -298,235 +298,236 @@ static char *soft_keywords[] = { #define invalid_named_expression_type 1211 #define invalid_assignment_type 1212 #define invalid_ann_assign_target_type 1213 -#define invalid_del_stmt_type 1214 -#define invalid_block_type 1215 -#define invalid_comprehension_type 1216 -#define invalid_dict_comprehension_type 1217 -#define invalid_parameters_type 1218 -#define invalid_default_type 1219 -#define invalid_star_etc_type 1220 -#define invalid_kwds_type 1221 -#define invalid_parameters_helper_type 1222 -#define invalid_lambda_parameters_type 1223 -#define invalid_lambda_parameters_helper_type 1224 -#define invalid_lambda_star_etc_type 1225 -#define invalid_lambda_kwds_type 1226 -#define invalid_double_type_comments_type 1227 -#define invalid_with_item_type 1228 -#define invalid_for_if_clause_type 1229 -#define invalid_for_target_type 1230 -#define invalid_group_type 1231 -#define invalid_import_type 1232 -#define invalid_dotted_as_name_type 1233 -#define invalid_import_from_as_name_type 1234 -#define invalid_import_from_targets_type 1235 -#define invalid_with_stmt_type 1236 -#define invalid_with_stmt_indent_type 1237 -#define invalid_try_stmt_type 1238 -#define invalid_except_stmt_type 1239 -#define invalid_except_star_stmt_type 1240 -#define invalid_finally_stmt_type 1241 -#define invalid_except_stmt_indent_type 1242 -#define invalid_except_star_stmt_indent_type 1243 -#define invalid_match_stmt_type 1244 -#define invalid_case_block_type 1245 -#define invalid_as_pattern_type 1246 -#define invalid_class_pattern_type 1247 -#define invalid_class_argument_pattern_type 1248 -#define invalid_if_stmt_type 1249 -#define invalid_elif_stmt_type 1250 -#define invalid_else_stmt_type 1251 -#define invalid_while_stmt_type 1252 -#define invalid_for_stmt_type 1253 -#define invalid_def_raw_type 1254 -#define invalid_class_def_raw_type 1255 -#define invalid_double_starred_kvpairs_type 1256 -#define invalid_kvpair_type 1257 -#define invalid_starred_expression_unpacking_type 1258 -#define invalid_starred_expression_type 1259 -#define invalid_fstring_replacement_field_type 1260 -#define invalid_fstring_conversion_character_type 1261 -#define invalid_tstring_replacement_field_type 1262 -#define invalid_tstring_conversion_character_type 1263 -#define invalid_arithmetic_type 1264 -#define invalid_factor_type 1265 -#define invalid_type_params_type 1266 -#define _loop0_1_type 1267 -#define _loop1_2_type 1268 -#define _loop0_3_type 1269 -#define _gather_4_type 1270 -#define _tmp_5_type 1271 -#define _tmp_6_type 1272 -#define _tmp_7_type 1273 -#define _tmp_8_type 1274 -#define _tmp_9_type 1275 -#define _tmp_10_type 1276 -#define _tmp_11_type 1277 -#define _loop1_12_type 1278 -#define _tmp_13_type 1279 -#define _loop0_14_type 1280 -#define _gather_15_type 1281 -#define _tmp_16_type 1282 -#define _tmp_17_type 1283 -#define _loop0_18_type 1284 -#define _loop1_19_type 1285 -#define _loop0_20_type 1286 -#define _gather_21_type 1287 -#define _tmp_22_type 1288 -#define _loop0_23_type 1289 -#define _gather_24_type 1290 -#define _loop1_25_type 1291 -#define _tmp_26_type 1292 -#define _tmp_27_type 1293 -#define _loop0_28_type 1294 -#define _loop0_29_type 1295 -#define _loop1_30_type 1296 -#define _loop1_31_type 1297 -#define _loop0_32_type 1298 -#define _loop1_33_type 1299 -#define _loop0_34_type 1300 -#define _gather_35_type 1301 -#define _tmp_36_type 1302 -#define _loop1_37_type 1303 -#define _loop1_38_type 1304 -#define _loop1_39_type 1305 -#define _loop0_40_type 1306 -#define _gather_41_type 1307 -#define _tmp_42_type 1308 -#define _tmp_43_type 1309 -#define _tmp_44_type 1310 -#define _loop0_45_type 1311 -#define _gather_46_type 1312 -#define _loop0_47_type 1313 -#define _gather_48_type 1314 -#define _tmp_49_type 1315 -#define _loop0_50_type 1316 -#define _gather_51_type 1317 -#define _loop0_52_type 1318 -#define _gather_53_type 1319 -#define _loop0_54_type 1320 -#define _gather_55_type 1321 -#define _loop1_56_type 1322 -#define _loop1_57_type 1323 -#define _loop0_58_type 1324 -#define _gather_59_type 1325 -#define _loop1_60_type 1326 -#define _loop1_61_type 1327 -#define _loop1_62_type 1328 -#define _tmp_63_type 1329 -#define _loop0_64_type 1330 -#define _gather_65_type 1331 -#define _tmp_66_type 1332 -#define _tmp_67_type 1333 -#define _tmp_68_type 1334 -#define _tmp_69_type 1335 -#define _tmp_70_type 1336 -#define _loop0_71_type 1337 -#define _loop0_72_type 1338 -#define _loop1_73_type 1339 -#define _loop1_74_type 1340 -#define _loop0_75_type 1341 -#define _loop1_76_type 1342 -#define _loop0_77_type 1343 -#define _loop0_78_type 1344 -#define _loop0_79_type 1345 -#define _loop0_80_type 1346 -#define _loop1_81_type 1347 -#define _tmp_82_type 1348 -#define _loop0_83_type 1349 -#define _gather_84_type 1350 -#define _loop1_85_type 1351 -#define _loop0_86_type 1352 -#define _tmp_87_type 1353 -#define _loop0_88_type 1354 -#define _gather_89_type 1355 -#define _tmp_90_type 1356 -#define _loop0_91_type 1357 -#define _gather_92_type 1358 -#define _loop0_93_type 1359 -#define _gather_94_type 1360 -#define _loop0_95_type 1361 -#define _loop0_96_type 1362 -#define _gather_97_type 1363 -#define _loop1_98_type 1364 -#define _tmp_99_type 1365 -#define _loop0_100_type 1366 -#define _gather_101_type 1367 -#define _loop0_102_type 1368 -#define _gather_103_type 1369 -#define _tmp_104_type 1370 -#define _tmp_105_type 1371 -#define _loop0_106_type 1372 -#define _gather_107_type 1373 -#define _tmp_108_type 1374 -#define _tmp_109_type 1375 -#define _tmp_110_type 1376 -#define _tmp_111_type 1377 -#define _tmp_112_type 1378 -#define _loop1_113_type 1379 -#define _tmp_114_type 1380 -#define _tmp_115_type 1381 -#define _tmp_116_type 1382 -#define _tmp_117_type 1383 -#define _tmp_118_type 1384 -#define _loop0_119_type 1385 -#define _loop0_120_type 1386 -#define _tmp_121_type 1387 -#define _tmp_122_type 1388 -#define _tmp_123_type 1389 -#define _tmp_124_type 1390 -#define _tmp_125_type 1391 -#define _tmp_126_type 1392 -#define _tmp_127_type 1393 -#define _tmp_128_type 1394 -#define _tmp_129_type 1395 -#define _loop0_130_type 1396 -#define _gather_131_type 1397 -#define _tmp_132_type 1398 -#define _tmp_133_type 1399 -#define _tmp_134_type 1400 -#define _tmp_135_type 1401 -#define _loop0_136_type 1402 -#define _gather_137_type 1403 -#define _tmp_138_type 1404 -#define _loop0_139_type 1405 -#define _gather_140_type 1406 -#define _loop0_141_type 1407 -#define _gather_142_type 1408 -#define _tmp_143_type 1409 -#define _loop0_144_type 1410 -#define _tmp_145_type 1411 -#define _tmp_146_type 1412 -#define _tmp_147_type 1413 -#define _tmp_148_type 1414 -#define _tmp_149_type 1415 -#define _tmp_150_type 1416 -#define _tmp_151_type 1417 -#define _tmp_152_type 1418 -#define _tmp_153_type 1419 -#define _tmp_154_type 1420 -#define _tmp_155_type 1421 -#define _tmp_156_type 1422 -#define _tmp_157_type 1423 -#define _tmp_158_type 1424 -#define _tmp_159_type 1425 -#define _tmp_160_type 1426 -#define _tmp_161_type 1427 -#define _tmp_162_type 1428 -#define _tmp_163_type 1429 -#define _tmp_164_type 1430 -#define _tmp_165_type 1431 -#define _tmp_166_type 1432 -#define _tmp_167_type 1433 -#define _tmp_168_type 1434 -#define _tmp_169_type 1435 -#define _tmp_170_type 1436 -#define _loop0_171_type 1437 -#define _tmp_172_type 1438 -#define _tmp_173_type 1439 -#define _tmp_174_type 1440 -#define _tmp_175_type 1441 -#define _tmp_176_type 1442 +#define invalid_raise_stmt_type 1214 +#define invalid_del_stmt_type 1215 +#define invalid_block_type 1216 +#define invalid_comprehension_type 1217 +#define invalid_dict_comprehension_type 1218 +#define invalid_parameters_type 1219 +#define invalid_default_type 1220 +#define invalid_star_etc_type 1221 +#define invalid_kwds_type 1222 +#define invalid_parameters_helper_type 1223 +#define invalid_lambda_parameters_type 1224 +#define invalid_lambda_parameters_helper_type 1225 +#define invalid_lambda_star_etc_type 1226 +#define invalid_lambda_kwds_type 1227 +#define invalid_double_type_comments_type 1228 +#define invalid_with_item_type 1229 +#define invalid_for_if_clause_type 1230 +#define invalid_for_target_type 1231 +#define invalid_group_type 1232 +#define invalid_import_type 1233 +#define invalid_dotted_as_name_type 1234 +#define invalid_import_from_as_name_type 1235 +#define invalid_import_from_targets_type 1236 +#define invalid_with_stmt_type 1237 +#define invalid_with_stmt_indent_type 1238 +#define invalid_try_stmt_type 1239 +#define invalid_except_stmt_type 1240 +#define invalid_except_star_stmt_type 1241 +#define invalid_finally_stmt_type 1242 +#define invalid_except_stmt_indent_type 1243 +#define invalid_except_star_stmt_indent_type 1244 +#define invalid_match_stmt_type 1245 +#define invalid_case_block_type 1246 +#define invalid_as_pattern_type 1247 +#define invalid_class_pattern_type 1248 +#define invalid_class_argument_pattern_type 1249 +#define invalid_if_stmt_type 1250 +#define invalid_elif_stmt_type 1251 +#define invalid_else_stmt_type 1252 +#define invalid_while_stmt_type 1253 +#define invalid_for_stmt_type 1254 +#define invalid_def_raw_type 1255 +#define invalid_class_def_raw_type 1256 +#define invalid_double_starred_kvpairs_type 1257 +#define invalid_kvpair_type 1258 +#define invalid_starred_expression_unpacking_type 1259 +#define invalid_starred_expression_type 1260 +#define invalid_fstring_replacement_field_type 1261 +#define invalid_fstring_conversion_character_type 1262 +#define invalid_tstring_replacement_field_type 1263 +#define invalid_tstring_conversion_character_type 1264 +#define invalid_arithmetic_type 1265 +#define invalid_factor_type 1266 +#define invalid_type_params_type 1267 +#define _loop0_1_type 1268 +#define _loop1_2_type 1269 +#define _loop0_3_type 1270 +#define _gather_4_type 1271 +#define _tmp_5_type 1272 +#define _tmp_6_type 1273 +#define _tmp_7_type 1274 +#define _tmp_8_type 1275 +#define _tmp_9_type 1276 +#define _tmp_10_type 1277 +#define _tmp_11_type 1278 +#define _loop1_12_type 1279 +#define _tmp_13_type 1280 +#define _loop0_14_type 1281 +#define _gather_15_type 1282 +#define _tmp_16_type 1283 +#define _tmp_17_type 1284 +#define _loop0_18_type 1285 +#define _loop1_19_type 1286 +#define _loop0_20_type 1287 +#define _gather_21_type 1288 +#define _tmp_22_type 1289 +#define _loop0_23_type 1290 +#define _gather_24_type 1291 +#define _loop1_25_type 1292 +#define _tmp_26_type 1293 +#define _tmp_27_type 1294 +#define _loop0_28_type 1295 +#define _loop0_29_type 1296 +#define _loop1_30_type 1297 +#define _loop1_31_type 1298 +#define _loop0_32_type 1299 +#define _loop1_33_type 1300 +#define _loop0_34_type 1301 +#define _gather_35_type 1302 +#define _tmp_36_type 1303 +#define _loop1_37_type 1304 +#define _loop1_38_type 1305 +#define _loop1_39_type 1306 +#define _loop0_40_type 1307 +#define _gather_41_type 1308 +#define _tmp_42_type 1309 +#define _tmp_43_type 1310 +#define _tmp_44_type 1311 +#define _loop0_45_type 1312 +#define _gather_46_type 1313 +#define _loop0_47_type 1314 +#define _gather_48_type 1315 +#define _tmp_49_type 1316 +#define _loop0_50_type 1317 +#define _gather_51_type 1318 +#define _loop0_52_type 1319 +#define _gather_53_type 1320 +#define _loop0_54_type 1321 +#define _gather_55_type 1322 +#define _loop1_56_type 1323 +#define _loop1_57_type 1324 +#define _loop0_58_type 1325 +#define _gather_59_type 1326 +#define _loop1_60_type 1327 +#define _loop1_61_type 1328 +#define _loop1_62_type 1329 +#define _tmp_63_type 1330 +#define _loop0_64_type 1331 +#define _gather_65_type 1332 +#define _tmp_66_type 1333 +#define _tmp_67_type 1334 +#define _tmp_68_type 1335 +#define _tmp_69_type 1336 +#define _tmp_70_type 1337 +#define _loop0_71_type 1338 +#define _loop0_72_type 1339 +#define _loop1_73_type 1340 +#define _loop1_74_type 1341 +#define _loop0_75_type 1342 +#define _loop1_76_type 1343 +#define _loop0_77_type 1344 +#define _loop0_78_type 1345 +#define _loop0_79_type 1346 +#define _loop0_80_type 1347 +#define _loop1_81_type 1348 +#define _tmp_82_type 1349 +#define _loop0_83_type 1350 +#define _gather_84_type 1351 +#define _loop1_85_type 1352 +#define _loop0_86_type 1353 +#define _tmp_87_type 1354 +#define _loop0_88_type 1355 +#define _gather_89_type 1356 +#define _tmp_90_type 1357 +#define _loop0_91_type 1358 +#define _gather_92_type 1359 +#define _loop0_93_type 1360 +#define _gather_94_type 1361 +#define _loop0_95_type 1362 +#define _loop0_96_type 1363 +#define _gather_97_type 1364 +#define _loop1_98_type 1365 +#define _tmp_99_type 1366 +#define _loop0_100_type 1367 +#define _gather_101_type 1368 +#define _loop0_102_type 1369 +#define _gather_103_type 1370 +#define _tmp_104_type 1371 +#define _tmp_105_type 1372 +#define _loop0_106_type 1373 +#define _gather_107_type 1374 +#define _tmp_108_type 1375 +#define _tmp_109_type 1376 +#define _tmp_110_type 1377 +#define _tmp_111_type 1378 +#define _tmp_112_type 1379 +#define _loop1_113_type 1380 +#define _tmp_114_type 1381 +#define _tmp_115_type 1382 +#define _tmp_116_type 1383 +#define _tmp_117_type 1384 +#define _tmp_118_type 1385 +#define _loop0_119_type 1386 +#define _loop0_120_type 1387 +#define _tmp_121_type 1388 +#define _tmp_122_type 1389 +#define _tmp_123_type 1390 +#define _tmp_124_type 1391 +#define _tmp_125_type 1392 +#define _tmp_126_type 1393 +#define _tmp_127_type 1394 +#define _tmp_128_type 1395 +#define _tmp_129_type 1396 +#define _loop0_130_type 1397 +#define _gather_131_type 1398 +#define _tmp_132_type 1399 +#define _tmp_133_type 1400 +#define _tmp_134_type 1401 +#define _tmp_135_type 1402 +#define _loop0_136_type 1403 +#define _gather_137_type 1404 +#define _tmp_138_type 1405 +#define _loop0_139_type 1406 +#define _gather_140_type 1407 +#define _loop0_141_type 1408 +#define _gather_142_type 1409 +#define _tmp_143_type 1410 +#define _loop0_144_type 1411 +#define _tmp_145_type 1412 +#define _tmp_146_type 1413 +#define _tmp_147_type 1414 +#define _tmp_148_type 1415 +#define _tmp_149_type 1416 +#define _tmp_150_type 1417 +#define _tmp_151_type 1418 +#define _tmp_152_type 1419 +#define _tmp_153_type 1420 +#define _tmp_154_type 1421 +#define _tmp_155_type 1422 +#define _tmp_156_type 1423 +#define _tmp_157_type 1424 +#define _tmp_158_type 1425 +#define _tmp_159_type 1426 +#define _tmp_160_type 1427 +#define _tmp_161_type 1428 +#define _tmp_162_type 1429 +#define _tmp_163_type 1430 +#define _tmp_164_type 1431 +#define _tmp_165_type 1432 +#define _tmp_166_type 1433 +#define _tmp_167_type 1434 +#define _tmp_168_type 1435 +#define _tmp_169_type 1436 +#define _tmp_170_type 1437 +#define _loop0_171_type 1438 +#define _tmp_172_type 1439 +#define _tmp_173_type 1440 +#define _tmp_174_type 1441 +#define _tmp_175_type 1442 +#define _tmp_176_type 1443 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -742,6 +743,7 @@ static void *invalid_expression_rule(Parser *p); static void *invalid_named_expression_rule(Parser *p); static void *invalid_assignment_rule(Parser *p); static expr_ty invalid_ann_assign_target_rule(Parser *p); +static void *invalid_raise_stmt_rule(Parser *p); static void *invalid_del_stmt_rule(Parser *p); static void *invalid_block_rule(Parser *p); static void *invalid_comprehension_rule(Parser *p); @@ -1698,7 +1700,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'raise' raise_stmt")); stmt_ty raise_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 525) // token='raise' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 627) // token='raise' && (raise_stmt_var = raise_stmt_rule(p)) // raise_stmt ) @@ -1740,7 +1742,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'del' del_stmt")); stmt_ty del_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 625) // token='del' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 629) // token='del' && (del_stmt_var = del_stmt_rule(p)) // del_stmt ) @@ -1936,7 +1938,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'if' if_stmt")); stmt_ty if_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 682) // token='if' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 686) // token='if' && (if_stmt_var = if_stmt_rule(p)) // if_stmt ) @@ -2020,7 +2022,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'try' try_stmt")); stmt_ty try_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 656) // token='try' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 660) // token='try' && (try_stmt_var = try_stmt_rule(p)) // try_stmt ) @@ -2041,7 +2043,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'while' while_stmt")); stmt_ty while_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 689) // token='while' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 693) // token='while' && (while_stmt_var = while_stmt_rule(p)) // while_stmt ) @@ -2767,7 +2769,7 @@ return_stmt_rule(Parser *p) return _res; } -// raise_stmt: 'raise' expression ['from' expression] | 'raise' +// raise_stmt: invalid_raise_stmt | 'raise' expression ['from' expression] | 'raise' static stmt_ty raise_stmt_rule(Parser *p) { @@ -2789,6 +2791,25 @@ raise_stmt_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro + if (p->call_invalid_rules) { // invalid_raise_stmt + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> raise_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_raise_stmt")); + void *invalid_raise_stmt_var; + if ( + (invalid_raise_stmt_var = invalid_raise_stmt_rule(p)) // invalid_raise_stmt + ) + { + D(fprintf(stderr, "%*c+ raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_raise_stmt")); + _res = invalid_raise_stmt_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s raise_stmt[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_raise_stmt")); + } { // 'raise' expression ['from' expression] if (p->error_indicator) { p->level--; @@ -2799,7 +2820,7 @@ raise_stmt_rule(Parser *p) expr_ty a; void *b; if ( - (_keyword = _PyPegen_expect_token(p, 525)) // token='raise' + (_keyword = _PyPegen_expect_token(p, 627)) // token='raise' && (a = expression_rule(p)) // expression && @@ -2836,7 +2857,7 @@ raise_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> raise_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'raise'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 525)) // token='raise' + (_keyword = _PyPegen_expect_token(p, 627)) // token='raise' ) { D(fprintf(stderr, "%*c+ raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise'")); @@ -3209,7 +3230,7 @@ del_stmt_rule(Parser *p) Token * _keyword; asdl_expr_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 625)) // token='del' + (_keyword = _PyPegen_expect_token(p, 629)) // token='del' && (a = del_targets_rule(p)) // del_targets && @@ -3498,7 +3519,7 @@ import_name_rule(Parser *p) Token * _keyword; asdl_alias_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 634)) // token='import' + (_keyword = _PyPegen_expect_token(p, 638)) // token='import' && (a = dotted_as_names_rule(p)) // dotted_as_names ) @@ -3567,13 +3588,13 @@ import_from_rule(Parser *p) expr_ty b; asdl_alias_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 633)) // token='from' + (_keyword = _PyPegen_expect_token(p, 637)) // token='from' && (a = _loop0_18_rule(p)) // (('.' | '...'))* && (b = dotted_name_rule(p)) // dotted_name && - (_keyword_1 = _PyPegen_expect_token(p, 634)) // token='import' + (_keyword_1 = _PyPegen_expect_token(p, 638)) // token='import' && (c = import_from_targets_rule(p)) // import_from_targets ) @@ -3611,11 +3632,11 @@ import_from_rule(Parser *p) asdl_seq * a; asdl_alias_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 633)) // token='from' + (_keyword = _PyPegen_expect_token(p, 637)) // token='from' && (a = _loop1_19_rule(p)) // (('.' | '...'))+ && - (_keyword_1 = _PyPegen_expect_token(p, 634)) // token='import' + (_keyword_1 = _PyPegen_expect_token(p, 638)) // token='import' && (b = import_from_targets_rule(p)) // import_from_targets ) @@ -4402,7 +4423,7 @@ class_def_raw_rule(Parser *p) asdl_stmt_seq* c; void *t; if ( - (_keyword = _PyPegen_expect_token(p, 701)) // token='class' + (_keyword = _PyPegen_expect_token(p, 705)) // token='class' && (a = _PyPegen_name_token(p)) // NAME && @@ -4569,7 +4590,7 @@ function_def_raw_rule(Parser *p) void *t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 699)) // token='def' + (_keyword = _PyPegen_expect_token(p, 703)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && @@ -4630,9 +4651,9 @@ function_def_raw_rule(Parser *p) void *t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='async' + (_keyword = _PyPegen_expect_token(p, 702)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 699)) // token='def' + (_keyword_1 = _PyPegen_expect_token(p, 703)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && @@ -5970,7 +5991,7 @@ if_stmt_rule(Parser *p) asdl_stmt_seq* b; stmt_ty c; if ( - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 686)) // token='if' && (a = named_expression_rule(p)) // named_expression && @@ -6015,7 +6036,7 @@ if_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 686)) // token='if' && (a = named_expression_rule(p)) // named_expression && @@ -6110,7 +6131,7 @@ elif_stmt_rule(Parser *p) asdl_stmt_seq* b; stmt_ty c; if ( - (_keyword = _PyPegen_expect_token(p, 687)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 691)) // token='elif' && (a = named_expression_rule(p)) // named_expression && @@ -6155,7 +6176,7 @@ elif_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 687)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 691)) // token='elif' && (a = named_expression_rule(p)) // named_expression && @@ -6236,7 +6257,7 @@ else_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 686)) // token='else' + (_keyword = _PyPegen_expect_token(p, 690)) // token='else' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -6315,7 +6336,7 @@ while_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 689)) // token='while' + (_keyword = _PyPegen_expect_token(p, 693)) // token='while' && (a = named_expression_rule(p)) // named_expression && @@ -6415,11 +6436,11 @@ for_stmt_rule(Parser *p) expr_ty t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 694)) // token='for' + (_keyword = _PyPegen_expect_token(p, 698)) // token='for' && (t = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 695)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 699)) // token='in' && (_cut_var = 1) && @@ -6477,13 +6498,13 @@ for_stmt_rule(Parser *p) expr_ty t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='async' + (_keyword = _PyPegen_expect_token(p, 702)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 694)) // token='for' + (_keyword_1 = _PyPegen_expect_token(p, 698)) // token='for' && (t = star_targets_rule(p)) // star_targets && - (_keyword_2 = _PyPegen_expect_token(p, 695)) // token='in' + (_keyword_2 = _PyPegen_expect_token(p, 699)) // token='in' && (_cut_var = 1) && @@ -6612,7 +6633,7 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 647)) // token='with' + (_keyword = _PyPegen_expect_token(p, 651)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -6663,7 +6684,7 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 647)) // token='with' + (_keyword = _PyPegen_expect_token(p, 651)) // token='with' && (a = (asdl_withitem_seq*)_gather_35_rule(p)) // ','.with_item+ && @@ -6712,9 +6733,9 @@ with_stmt_rule(Parser *p) asdl_withitem_seq* a; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='async' + (_keyword = _PyPegen_expect_token(p, 702)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 647)) // token='with' + (_keyword_1 = _PyPegen_expect_token(p, 651)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -6764,9 +6785,9 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='async' + (_keyword = _PyPegen_expect_token(p, 702)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 647)) // token='with' + (_keyword_1 = _PyPegen_expect_token(p, 651)) // token='with' && (a = (asdl_withitem_seq*)_gather_35_rule(p)) // ','.with_item+ && @@ -6852,7 +6873,7 @@ with_item_rule(Parser *p) if ( (e = expression_rule(p)) // expression && - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 684)) // token='as' && (t = star_target_rule(p)) // star_target && @@ -6977,7 +6998,7 @@ try_stmt_rule(Parser *p) asdl_stmt_seq* b; asdl_stmt_seq* f; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='try' + (_keyword = _PyPegen_expect_token(p, 660)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7021,7 +7042,7 @@ try_stmt_rule(Parser *p) asdl_excepthandler_seq* ex; void *f; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='try' + (_keyword = _PyPegen_expect_token(p, 660)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7069,7 +7090,7 @@ try_stmt_rule(Parser *p) asdl_excepthandler_seq* ex; void *f; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='try' + (_keyword = _PyPegen_expect_token(p, 660)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7168,7 +7189,7 @@ except_block_rule(Parser *p) asdl_stmt_seq* b; expr_ty e; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 681)) // token='except' && (e = expression_rule(p)) // expression && @@ -7212,11 +7233,11 @@ except_block_rule(Parser *p) expr_ty e; expr_ty t; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 681)) // token='except' && (e = expression_rule(p)) // expression && - (_keyword_1 = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 684)) // token='as' && (t = _PyPegen_name_token(p)) // NAME && @@ -7258,7 +7279,7 @@ except_block_rule(Parser *p) asdl_stmt_seq* b; expr_ty e; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 681)) // token='except' && (e = expressions_rule(p)) // expressions && @@ -7299,7 +7320,7 @@ except_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 681)) // token='except' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -7411,7 +7432,7 @@ except_star_block_rule(Parser *p) asdl_stmt_seq* b; expr_ty e; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 681)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -7458,13 +7479,13 @@ except_star_block_rule(Parser *p) expr_ty e; expr_ty t; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 681)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (e = expression_rule(p)) // expression && - (_keyword_1 = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 684)) // token='as' && (t = _PyPegen_name_token(p)) // NAME && @@ -7507,7 +7528,7 @@ except_star_block_rule(Parser *p) asdl_stmt_seq* b; expr_ty e; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 681)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -7607,7 +7628,7 @@ finally_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 673)) // token='finally' + (_keyword = _PyPegen_expect_token(p, 677)) // token='finally' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7915,7 +7936,7 @@ guard_rule(Parser *p) Token * _keyword; expr_ty guard; if ( - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 686)) // token='if' && (guard = named_expression_rule(p)) // named_expression ) @@ -8110,7 +8131,7 @@ as_pattern_rule(Parser *p) if ( (pattern = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 684)) // token='as' && (target = pattern_capture_target_rule(p)) // pattern_capture_target ) @@ -11407,11 +11428,11 @@ expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 686)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 686)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 690)) // token='else' && (c = expression_rule(p)) // expression ) @@ -11517,7 +11538,7 @@ yield_expr_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 587)) // token='yield' && - (_keyword_1 = _PyPegen_expect_token(p, 633)) // token='from' + (_keyword_1 = _PyPegen_expect_token(p, 637)) // token='from' && (a = expression_rule(p)) // expression ) @@ -12293,7 +12314,7 @@ inversion_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 703)) // token='not' + (_keyword = _PyPegen_expect_token(p, 707)) // token='not' && (a = inversion_rule(p)) // inversion ) @@ -12947,9 +12968,9 @@ notin_bitwise_or_rule(Parser *p) Token * _keyword_1; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 703)) // token='not' + (_keyword = _PyPegen_expect_token(p, 707)) // token='not' && - (_keyword_1 = _PyPegen_expect_token(p, 695)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 699)) // token='in' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -12995,7 +13016,7 @@ in_bitwise_or_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 695)) // token='in' + (_keyword = _PyPegen_expect_token(p, 699)) // token='in' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -13044,7 +13065,7 @@ isnot_bitwise_or_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 596)) // token='is' && - (_keyword_1 = _PyPegen_expect_token(p, 703)) // token='not' + (_keyword_1 = _PyPegen_expect_token(p, 707)) // token='not' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -17651,13 +17672,13 @@ for_if_clause_rule(Parser *p) expr_ty b; asdl_expr_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='async' + (_keyword = _PyPegen_expect_token(p, 702)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 694)) // token='for' + (_keyword_1 = _PyPegen_expect_token(p, 698)) // token='for' && (a = star_targets_rule(p)) // star_targets && - (_keyword_2 = _PyPegen_expect_token(p, 695)) // token='in' + (_keyword_2 = _PyPegen_expect_token(p, 699)) // token='in' && (_cut_var = 1) && @@ -17696,11 +17717,11 @@ for_if_clause_rule(Parser *p) expr_ty b; asdl_expr_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 694)) // token='for' + (_keyword = _PyPegen_expect_token(p, 698)) // token='for' && (a = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 695)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 699)) // token='in' && (_cut_var = 1) && @@ -21001,11 +21022,11 @@ expression_without_invalid_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 686)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 686)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 690)) // token='else' && (c = expression_rule(p)) // expression ) @@ -21305,7 +21326,7 @@ invalid_expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 686)) // token='if' && (b = disjunction_rule(p)) // disjunction && @@ -21338,11 +21359,11 @@ invalid_expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 686)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 686)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 690)) // token='else' && _PyPegen_lookahead(0, (void *(*)(Parser *)) expression_rule, p) ) @@ -21374,11 +21395,11 @@ invalid_expression_rule(Parser *p) if ( (a = (stmt_ty)_tmp_116_rule(p)) // pass_stmt | break_stmt | continue_stmt && - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 686)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 686)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 690)) // token='else' && (c = simple_stmt_rule(p)) // simple_stmt ) @@ -21889,6 +21910,82 @@ invalid_ann_assign_target_rule(Parser *p) return _res; } +// invalid_raise_stmt: 'raise' 'from' | 'raise' expression 'from' +static void * +invalid_raise_stmt_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // 'raise' 'from' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_raise_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'raise' 'from'")); + Token * a; + Token * b; + if ( + (a = _PyPegen_expect_token(p, 627)) // token='raise' + && + (b = _PyPegen_expect_token(p, 637)) // token='from' + ) + { + D(fprintf(stderr, "%*c+ invalid_raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise' 'from'")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "did you forget an expression between 'raise' and 'from'?" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_raise_stmt[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'raise' 'from'")); + } + { // 'raise' expression 'from' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_raise_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'raise' expression 'from'")); + Token * _keyword; + Token * a; + expr_ty expression_var; + if ( + (_keyword = _PyPegen_expect_token(p, 627)) // token='raise' + && + (expression_var = expression_rule(p)) // expression + && + (a = _PyPegen_expect_token(p, 637)) // token='from' + ) + { + D(fprintf(stderr, "%*c+ invalid_raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise' expression 'from'")); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "did you forget an expression after 'from'?" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_raise_stmt[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'raise' expression 'from'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // invalid_del_stmt: 'del' star_expressions static void * invalid_del_stmt_rule(Parser *p) @@ -21911,7 +22008,7 @@ invalid_del_stmt_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 625)) // token='del' + (_keyword = _PyPegen_expect_token(p, 629)) // token='del' && (a = star_expressions_rule(p)) // star_expressions ) @@ -23347,7 +23444,7 @@ invalid_with_item_rule(Parser *p) if ( (expression_var = expression_rule(p)) // expression && - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 684)) // token='as' && (a = expression_rule(p)) // expression && @@ -23397,13 +23494,13 @@ invalid_for_if_clause_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings void *_tmp_135_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 694)) // token='for' + (_keyword = _PyPegen_expect_token(p, 698)) // token='for' && (_tmp_135_var = _tmp_135_rule(p)) // bitwise_or ((',' bitwise_or))* ','? && - _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 695) // token='in' + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 699) // token='in' ) { D(fprintf(stderr, "%*c+ invalid_for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); @@ -23449,9 +23546,9 @@ invalid_for_target_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings expr_ty a; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 694)) // token='for' + (_keyword = _PyPegen_expect_token(p, 698)) // token='for' && (a = star_expressions_rule(p)) // star_expressions ) @@ -23581,11 +23678,11 @@ invalid_import_rule(Parser *p) Token * a; expr_ty dotted_name_var; if ( - (a = _PyPegen_expect_token(p, 634)) // token='import' + (a = _PyPegen_expect_token(p, 638)) // token='import' && (_gather_137_var = _gather_137_rule(p)) // ','.dotted_name+ && - (_keyword = _PyPegen_expect_token(p, 633)) // token='from' + (_keyword = _PyPegen_expect_token(p, 637)) // token='from' && (dotted_name_var = dotted_name_rule(p)) // dotted_name ) @@ -23612,7 +23709,7 @@ invalid_import_rule(Parser *p) Token * _keyword; Token * token; if ( - (_keyword = _PyPegen_expect_token(p, 634)) // token='import' + (_keyword = _PyPegen_expect_token(p, 638)) // token='import' && (token = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -23661,7 +23758,7 @@ invalid_dotted_as_name_rule(Parser *p) if ( (dotted_name_var = dotted_name_rule(p)) // dotted_name && - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 684)) // token='as' && _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_138_rule, p) && @@ -23712,7 +23809,7 @@ invalid_import_from_as_name_rule(Parser *p) if ( (name_var = _PyPegen_name_token(p)) // NAME && - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 684)) // token='as' && _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_138_rule, p) && @@ -23838,9 +23935,9 @@ invalid_with_stmt_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 647)) // token='with' + (_keyword = _PyPegen_expect_token(p, 651)) // token='with' && (_gather_140_var = _gather_140_rule(p)) // ','.(expression ['as' star_target])+ && @@ -23876,9 +23973,9 @@ invalid_with_stmt_rule(Parser *p) UNUSED(_opt_var_1); // Silence compiler warnings Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 647)) // token='with' + (_keyword = _PyPegen_expect_token(p, 651)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -23938,9 +24035,9 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 647)) // token='with' + (a = _PyPegen_expect_token(p, 651)) // token='with' && (_gather_140_var = _gather_140_rule(p)) // ','.(expression ['as' star_target])+ && @@ -23981,9 +24078,9 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 647)) // token='with' + (a = _PyPegen_expect_token(p, 651)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -24046,7 +24143,7 @@ invalid_try_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 656)) // token='try' + (a = _PyPegen_expect_token(p, 660)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24078,7 +24175,7 @@ invalid_try_stmt_rule(Parser *p) Token * _literal; asdl_stmt_seq* block_var; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='try' + (_keyword = _PyPegen_expect_token(p, 660)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24117,7 +24214,7 @@ invalid_try_stmt_rule(Parser *p) Token * b; expr_ty expression_var; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='try' + (_keyword = _PyPegen_expect_token(p, 660)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24125,7 +24222,7 @@ invalid_try_stmt_rule(Parser *p) && (_loop1_37_var = _loop1_37_rule(p)) // except_block+ && - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 681)) // token='except' && (b = _PyPegen_expect_token(p, 16)) // token='*' && @@ -24164,7 +24261,7 @@ invalid_try_stmt_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings Token * a; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='try' + (_keyword = _PyPegen_expect_token(p, 660)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24172,7 +24269,7 @@ invalid_try_stmt_rule(Parser *p) && (_loop1_38_var = _loop1_38_rule(p)) // except_star_block+ && - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 681)) // token='except' && (_opt_var = _tmp_145_rule(p), !p->error_indicator) // [expression ['as' NAME]] && @@ -24229,7 +24326,7 @@ invalid_except_stmt_rule(Parser *p) expr_ty expressions_var; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 681)) // token='except' && (a = expression_rule(p)) // expression && @@ -24237,7 +24334,7 @@ invalid_except_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_keyword_1 = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 684)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -24269,7 +24366,7 @@ invalid_except_stmt_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 681)) // token='except' && (expression_var = expression_rule(p)) // expression && @@ -24300,7 +24397,7 @@ invalid_except_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 681)) // token='except' && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -24331,11 +24428,11 @@ invalid_except_stmt_rule(Parser *p) asdl_stmt_seq* block_var; expr_ty expression_var; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 681)) // token='except' && (expression_var = expression_rule(p)) // expression && - (_keyword_1 = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 684)) // token='as' && (a = expression_rule(p)) // expression && @@ -24395,7 +24492,7 @@ invalid_except_star_stmt_rule(Parser *p) expr_ty expressions_var; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 681)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -24405,7 +24502,7 @@ invalid_except_star_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_keyword_1 = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 684)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -24438,7 +24535,7 @@ invalid_except_star_stmt_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 681)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -24472,7 +24569,7 @@ invalid_except_star_stmt_rule(Parser *p) void *_tmp_146_var; Token * a; if ( - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 681)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -24506,13 +24603,13 @@ invalid_except_star_stmt_rule(Parser *p) asdl_stmt_seq* block_var; expr_ty expression_var; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 681)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (expression_var = expression_rule(p)) // expression && - (_keyword_1 = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 684)) // token='as' && (a = expression_rule(p)) // expression && @@ -24563,7 +24660,7 @@ invalid_finally_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 673)) // token='finally' + (a = _PyPegen_expect_token(p, 677)) // token='finally' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24619,7 +24716,7 @@ invalid_except_stmt_indent_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 681)) // token='except' && (expression_var = expression_rule(p)) // expression && @@ -24655,7 +24752,7 @@ invalid_except_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 681)) // token='except' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24711,7 +24808,7 @@ invalid_except_star_stmt_indent_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 681)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -24950,7 +25047,7 @@ invalid_as_pattern_rule(Parser *p) if ( (or_pattern_var = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 684)) // token='as' && (a = _PyPegen_expect_soft_keyword(p, "_")) // soft_keyword='"_"' ) @@ -24980,7 +25077,7 @@ invalid_as_pattern_rule(Parser *p) if ( (or_pattern_var = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 684)) // token='as' && (a = expression_rule(p)) // expression ) @@ -25132,7 +25229,7 @@ invalid_if_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 686)) // token='if' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -25163,7 +25260,7 @@ invalid_if_stmt_rule(Parser *p) expr_ty a_1; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 682)) // token='if' + (a = _PyPegen_expect_token(p, 686)) // token='if' && (a_1 = named_expression_rule(p)) // named_expression && @@ -25218,7 +25315,7 @@ invalid_elif_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 687)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 691)) // token='elif' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -25249,7 +25346,7 @@ invalid_elif_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 687)) // token='elif' + (a = _PyPegen_expect_token(p, 691)) // token='elif' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -25302,7 +25399,7 @@ invalid_else_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 686)) // token='else' + (a = _PyPegen_expect_token(p, 690)) // token='else' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -25335,13 +25432,13 @@ invalid_else_stmt_rule(Parser *p) Token * _literal; asdl_stmt_seq* block_var; if ( - (_keyword = _PyPegen_expect_token(p, 686)) // token='else' + (_keyword = _PyPegen_expect_token(p, 690)) // token='else' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && (block_var = block_rule(p)) // block && - (_keyword_1 = _PyPegen_expect_token(p, 687)) // token='elif' + (_keyword_1 = _PyPegen_expect_token(p, 691)) // token='elif' ) { D(fprintf(stderr, "%*c+ invalid_else_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else' ':' block 'elif'")); @@ -25388,7 +25485,7 @@ invalid_while_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 689)) // token='while' + (_keyword = _PyPegen_expect_token(p, 693)) // token='while' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -25419,7 +25516,7 @@ invalid_while_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 689)) // token='while' + (a = _PyPegen_expect_token(p, 693)) // token='while' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -25478,13 +25575,13 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 694)) // token='for' + (_keyword = _PyPegen_expect_token(p, 698)) // token='for' && (star_targets_var = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 695)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 699)) // token='in' && (star_expressions_var = star_expressions_rule(p)) // star_expressions && @@ -25519,13 +25616,13 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 694)) // token='for' + (a = _PyPegen_expect_token(p, 698)) // token='for' && (star_targets_var = star_targets_rule(p)) // star_targets && - (_keyword = _PyPegen_expect_token(p, 695)) // token='in' + (_keyword = _PyPegen_expect_token(p, 699)) // token='in' && (star_expressions_var = star_expressions_rule(p)) // star_expressions && @@ -25591,9 +25688,9 @@ invalid_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 699)) // token='def' + (a = _PyPegen_expect_token(p, 703)) // token='def' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -25650,9 +25747,9 @@ invalid_def_raw_rule(Parser *p) asdl_stmt_seq* block_var; expr_ty name_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 699)) // token='def' + (_keyword = _PyPegen_expect_token(p, 703)) // token='def' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -25716,7 +25813,7 @@ invalid_class_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 701)) // token='class' + (_keyword = _PyPegen_expect_token(p, 705)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -25755,7 +25852,7 @@ invalid_class_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 701)) // token='class' + (a = _PyPegen_expect_token(p, 705)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -27015,7 +27112,7 @@ invalid_arithmetic_rule(Parser *p) && (_tmp_153_var = _tmp_153_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' && - (a = _PyPegen_expect_token(p, 703)) // token='not' + (a = _PyPegen_expect_token(p, 707)) // token='not' && (b = inversion_rule(p)) // inversion ) @@ -27064,7 +27161,7 @@ invalid_factor_rule(Parser *p) if ( (_tmp_154_var = _tmp_154_rule(p)) // '+' | '-' | '~' && - (a = _PyPegen_expect_token(p, 703)) // token='not' + (a = _PyPegen_expect_token(p, 707)) // token='not' && (b = factor_rule(p)) // factor ) @@ -27411,7 +27508,7 @@ _tmp_5_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_5[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 634)) // token='import' + (_keyword = _PyPegen_expect_token(p, 638)) // token='import' ) { D(fprintf(stderr, "%*c+ _tmp_5[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'import'")); @@ -27430,7 +27527,7 @@ _tmp_5_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_5[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'from'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 633)) // token='from' + (_keyword = _PyPegen_expect_token(p, 637)) // token='from' ) { D(fprintf(stderr, "%*c+ _tmp_5[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'from'")); @@ -27468,7 +27565,7 @@ _tmp_6_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_6[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 699)) // token='def' + (_keyword = _PyPegen_expect_token(p, 703)) // token='def' ) { D(fprintf(stderr, "%*c+ _tmp_6[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def'")); @@ -27506,7 +27603,7 @@ _tmp_6_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_6[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='async' + (_keyword = _PyPegen_expect_token(p, 702)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_6[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -27544,7 +27641,7 @@ _tmp_7_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'class'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 701)) // token='class' + (_keyword = _PyPegen_expect_token(p, 705)) // token='class' ) { D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class'")); @@ -27601,7 +27698,7 @@ _tmp_8_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_8[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'with'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 647)) // token='with' + (_keyword = _PyPegen_expect_token(p, 651)) // token='with' ) { D(fprintf(stderr, "%*c+ _tmp_8[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'with'")); @@ -27620,7 +27717,7 @@ _tmp_8_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_8[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='async' + (_keyword = _PyPegen_expect_token(p, 702)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_8[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -27658,7 +27755,7 @@ _tmp_9_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'for'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 694)) // token='for' + (_keyword = _PyPegen_expect_token(p, 698)) // token='for' ) { D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'for'")); @@ -27677,7 +27774,7 @@ _tmp_9_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='async' + (_keyword = _PyPegen_expect_token(p, 702)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -27902,7 +27999,7 @@ _tmp_13_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 633)) // token='from' + (_keyword = _PyPegen_expect_token(p, 637)) // token='from' && (z = expression_rule(p)) // expression ) @@ -28424,7 +28521,7 @@ _tmp_22_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 684)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -34188,7 +34285,7 @@ _tmp_115_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 686)) // token='else' + (_keyword = _PyPegen_expect_token(p, 690)) // token='else' ) { D(fprintf(stderr, "%*c+ _tmp_115[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); @@ -35927,7 +36024,7 @@ _tmp_143_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 681)) // token='except' ) { D(fprintf(stderr, "%*c+ _tmp_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); @@ -35946,7 +36043,7 @@ _tmp_143_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 673)) // token='finally' + (_keyword = _PyPegen_expect_token(p, 677)) // token='finally' ) { D(fprintf(stderr, "%*c+ _tmp_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); @@ -37169,7 +37266,7 @@ _tmp_163_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 686)) // token='if' && (z = disjunction_rule(p)) // disjunction ) @@ -37827,7 +37924,7 @@ _tmp_176_rule(Parser *p) Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 684)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) From 6ef06fad84244261c695ec337c7d2734277054db Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 6 Jun 2025 14:52:48 +0300 Subject: [PATCH 095/116] gh-135120: Add test.support.subTests() (GH-135121) --- Lib/test/support/__init__.py | 25 ++ Lib/test/test_http_cookiejar.py | 187 ++++---- Lib/test/test_ntpath.py | 20 +- Lib/test/test_posixpath.py | 18 +- Lib/test/test_urlparse.py | 398 +++++++++--------- ...-06-04-13-07-44.gh-issue-135120.NapnZT.rst | 1 + 6 files changed, 307 insertions(+), 342 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2025-06-04-13-07-44.gh-issue-135120.NapnZT.rst diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index f9b66b88d3d05c..48e74adcce3bb7 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -945,6 +945,31 @@ def check_sizeof(test, o, size): % (type(o), result, size) test.assertEqual(result, size, msg) +def subTests(arg_names, arg_values, /, *, _do_cleanups=False): + """Run multiple subtests with different parameters. + """ + single_param = False + if isinstance(arg_names, str): + arg_names = arg_names.replace(',',' ').split() + if len(arg_names) == 1: + single_param = True + arg_values = tuple(arg_values) + def decorator(func): + if isinstance(func, type): + raise TypeError('subTests() can only decorate methods, not classes') + @functools.wraps(func) + def wrapper(self, /, *args, **kwargs): + for values in arg_values: + if single_param: + values = (values,) + subtest_kwargs = dict(zip(arg_names, values)) + with self.subTest(**subtest_kwargs): + func(self, *args, **kwargs, **subtest_kwargs) + if _do_cleanups: + self.doCleanups() + return wrapper + return decorator + #======================================================================= # Decorator/context manager for running a code in a different locale, # correctly resetting it afterwards. diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index 6bc33b15ec32e9..04cb440cd4ccf6 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -4,6 +4,7 @@ import stat import sys import re +from test import support from test.support import os_helper from test.support import warnings_helper import time @@ -105,8 +106,7 @@ def test_http2time_formats(self): self.assertEqual(http2time(s.lower()), test_t, s.lower()) self.assertEqual(http2time(s.upper()), test_t, s.upper()) - def test_http2time_garbage(self): - for test in [ + @support.subTests('test', [ '', 'Garbage', 'Mandag 16. September 1996', @@ -121,10 +121,9 @@ def test_http2time_garbage(self): '08-01-3697739', '09 Feb 19942632 22:23:32 GMT', 'Wed, 09 Feb 1994834 22:23:32 GMT', - ]: - self.assertIsNone(http2time(test), - "http2time(%s) is not None\n" - "http2time(test) %s" % (test, http2time(test))) + ]) + def test_http2time_garbage(self, test): + self.assertIsNone(http2time(test)) def test_http2time_redos_regression_actually_completes(self): # LOOSE_HTTP_DATE_RE was vulnerable to malicious input which caused catastrophic backtracking (REDoS). @@ -149,9 +148,7 @@ def parse_date(text): self.assertEqual(parse_date("1994-02-03 19:45:29 +0530"), (1994, 2, 3, 14, 15, 29)) - def test_iso2time_formats(self): - # test iso2time for supported dates. - tests = [ + @support.subTests('s', [ '1994-02-03 00:00:00 -0000', # ISO 8601 format '1994-02-03 00:00:00 +0000', # ISO 8601 format '1994-02-03 00:00:00', # zone is optional @@ -164,16 +161,15 @@ def test_iso2time_formats(self): # A few tests with extra space at various places ' 1994-02-03 ', ' 1994-02-03T00:00:00 ', - ] - + ]) + def test_iso2time_formats(self, s): + # test iso2time for supported dates. test_t = 760233600 # assume broken POSIX counting of seconds - for s in tests: - self.assertEqual(iso2time(s), test_t, s) - self.assertEqual(iso2time(s.lower()), test_t, s.lower()) - self.assertEqual(iso2time(s.upper()), test_t, s.upper()) + self.assertEqual(iso2time(s), test_t, s) + self.assertEqual(iso2time(s.lower()), test_t, s.lower()) + self.assertEqual(iso2time(s.upper()), test_t, s.upper()) - def test_iso2time_garbage(self): - for test in [ + @support.subTests('test', [ '', 'Garbage', 'Thursday, 03-Feb-94 00:00:00 GMT', @@ -186,9 +182,9 @@ def test_iso2time_garbage(self): '01-01-1980 00:00:62', '01-01-1980T00:00:62', '19800101T250000Z', - ]: - self.assertIsNone(iso2time(test), - "iso2time(%r)" % test) + ]) + def test_iso2time_garbage(self, test): + self.assertIsNone(iso2time(test)) def test_iso2time_performance_regression(self): # If ISO_DATE_RE regresses to quadratic complexity, this test will take a very long time to succeed. @@ -199,24 +195,23 @@ def test_iso2time_performance_regression(self): class HeaderTests(unittest.TestCase): - def test_parse_ns_headers(self): - # quotes should be stripped - expected = [[('foo', 'bar'), ('expires', 2209069412), ('version', '0')]] - for hdr in [ + @support.subTests('hdr', [ 'foo=bar; expires=01 Jan 2040 22:23:32 GMT', 'foo=bar; expires="01 Jan 2040 22:23:32 GMT"', - ]: - self.assertEqual(parse_ns_headers([hdr]), expected) - - def test_parse_ns_headers_version(self): - + ]) + def test_parse_ns_headers(self, hdr): # quotes should be stripped - expected = [[('foo', 'bar'), ('version', '1')]] - for hdr in [ + expected = [[('foo', 'bar'), ('expires', 2209069412), ('version', '0')]] + self.assertEqual(parse_ns_headers([hdr]), expected) + + @support.subTests('hdr', [ 'foo=bar; version="1"', 'foo=bar; Version="1"', - ]: - self.assertEqual(parse_ns_headers([hdr]), expected) + ]) + def test_parse_ns_headers_version(self, hdr): + # quotes should be stripped + expected = [[('foo', 'bar'), ('version', '1')]] + self.assertEqual(parse_ns_headers([hdr]), expected) def test_parse_ns_headers_special_names(self): # names such as 'expires' are not special in first name=value pair @@ -226,8 +221,7 @@ def test_parse_ns_headers_special_names(self): expected = [[("expires", "01 Jan 2040 22:23:32 GMT"), ("version", "0")]] self.assertEqual(parse_ns_headers([hdr]), expected) - def test_join_header_words(self): - for src, expected in [ + @support.subTests('src,expected', [ ([[("foo", None), ("bar", "baz")]], "foo; bar=baz"), (([]), ""), (([[]]), ""), @@ -237,12 +231,11 @@ def test_join_header_words(self): 'n; foo="foo;_", bar=foo_bar'), ([[("n", "m"), ("foo", None)], [("bar", "foo_bar")]], 'n=m; foo, bar=foo_bar'), - ]: - with self.subTest(src=src): - self.assertEqual(join_header_words(src), expected) + ]) + def test_join_header_words(self, src, expected): + self.assertEqual(join_header_words(src), expected) - def test_split_header_words(self): - tests = [ + @support.subTests('arg,expect', [ ("foo", [[("foo", None)]]), ("foo=bar", [[("foo", "bar")]]), (" foo ", [[("foo", None)]]), @@ -259,24 +252,22 @@ def test_split_header_words(self): (r'foo; bar=baz, spam=, foo="\,\;\"", bar= ', [[("foo", None), ("bar", "baz")], [("spam", "")], [("foo", ',;"')], [("bar", "")]]), - ] - - for arg, expect in tests: - try: - result = split_header_words([arg]) - except: - import traceback, io - f = io.StringIO() - traceback.print_exc(None, f) - result = "(error -- traceback follows)\n\n%s" % f.getvalue() - self.assertEqual(result, expect, """ + ]) + def test_split_header_words(self, arg, expect): + try: + result = split_header_words([arg]) + except: + import traceback, io + f = io.StringIO() + traceback.print_exc(None, f) + result = "(error -- traceback follows)\n\n%s" % f.getvalue() + self.assertEqual(result, expect, """ When parsing: '%s' Expected: '%s' Got: '%s' """ % (arg, expect, result)) - def test_roundtrip(self): - tests = [ + @support.subTests('arg,expect', [ ("foo", "foo"), ("foo=bar", "foo=bar"), (" foo ", "foo"), @@ -309,12 +300,11 @@ def test_roundtrip(self): ('n; foo="foo;_", bar="foo,_"', 'n; foo="foo;_", bar="foo,_"'), - ] - - for arg, expect in tests: - input = split_header_words([arg]) - res = join_header_words(input) - self.assertEqual(res, expect, """ + ]) + def test_roundtrip(self, arg, expect): + input = split_header_words([arg]) + res = join_header_words(input) + self.assertEqual(res, expect, """ When parsing: '%s' Expected: '%s' Got: '%s' @@ -516,14 +506,7 @@ class CookieTests(unittest.TestCase): ## just the 7 special TLD's listed in their spec. And folks rely on ## that... - def test_domain_return_ok(self): - # test optimization: .domain_return_ok() should filter out most - # domains in the CookieJar before we try to access them (because that - # may require disk access -- in particular, with MSIECookieJar) - # This is only a rough check for performance reasons, so it's not too - # critical as long as it's sufficiently liberal. - pol = DefaultCookiePolicy() - for url, domain, ok in [ + @support.subTests('url,domain,ok', [ ("http://foo.bar.com/", "blah.com", False), ("http://foo.bar.com/", "rhubarb.blah.com", False), ("http://foo.bar.com/", "rhubarb.foo.bar.com", False), @@ -543,11 +526,18 @@ def test_domain_return_ok(self): ("http://foo/", ".local", True), ("http://barfoo.com", ".foo.com", False), ("http://barfoo.com", "foo.com", False), - ]: - request = urllib.request.Request(url) - r = pol.domain_return_ok(domain, request) - if ok: self.assertTrue(r) - else: self.assertFalse(r) + ]) + def test_domain_return_ok(self, url, domain, ok): + # test optimization: .domain_return_ok() should filter out most + # domains in the CookieJar before we try to access them (because that + # may require disk access -- in particular, with MSIECookieJar) + # This is only a rough check for performance reasons, so it's not too + # critical as long as it's sufficiently liberal. + pol = DefaultCookiePolicy() + request = urllib.request.Request(url) + r = pol.domain_return_ok(domain, request) + if ok: self.assertTrue(r) + else: self.assertFalse(r) def test_missing_value(self): # missing = sign in Cookie: header is regarded by Mozilla as a missing @@ -581,10 +571,7 @@ def test_missing_value(self): self.assertEqual(interact_netscape(c, "http://www.acme.com/foo/"), '"spam"; eggs') - def test_rfc2109_handling(self): - # RFC 2109 cookies are handled as RFC 2965 or Netscape cookies, - # dependent on policy settings - for rfc2109_as_netscape, rfc2965, version in [ + @support.subTests('rfc2109_as_netscape,rfc2965,version', [ # default according to rfc2965 if not explicitly specified (None, False, 0), (None, True, 1), @@ -593,24 +580,27 @@ def test_rfc2109_handling(self): (False, True, 1), (True, False, 0), (True, True, 0), - ]: - policy = DefaultCookiePolicy( - rfc2109_as_netscape=rfc2109_as_netscape, - rfc2965=rfc2965) - c = CookieJar(policy) - interact_netscape(c, "http://www.example.com/", "ni=ni; Version=1") - try: - cookie = c._cookies["www.example.com"]["/"]["ni"] - except KeyError: - self.assertIsNone(version) # didn't expect a stored cookie - else: - self.assertEqual(cookie.version, version) - # 2965 cookies are unaffected - interact_2965(c, "http://www.example.com/", - "foo=bar; Version=1") - if rfc2965: - cookie2965 = c._cookies["www.example.com"]["/"]["foo"] - self.assertEqual(cookie2965.version, 1) + ]) + def test_rfc2109_handling(self, rfc2109_as_netscape, rfc2965, version): + # RFC 2109 cookies are handled as RFC 2965 or Netscape cookies, + # dependent on policy settings + policy = DefaultCookiePolicy( + rfc2109_as_netscape=rfc2109_as_netscape, + rfc2965=rfc2965) + c = CookieJar(policy) + interact_netscape(c, "http://www.example.com/", "ni=ni; Version=1") + try: + cookie = c._cookies["www.example.com"]["/"]["ni"] + except KeyError: + self.assertIsNone(version) # didn't expect a stored cookie + else: + self.assertEqual(cookie.version, version) + # 2965 cookies are unaffected + interact_2965(c, "http://www.example.com/", + "foo=bar; Version=1") + if rfc2965: + cookie2965 = c._cookies["www.example.com"]["/"]["foo"] + self.assertEqual(cookie2965.version, 1) def test_ns_parser(self): c = CookieJar() @@ -778,8 +768,7 @@ def test_default_path_with_query(self): # Cookie is sent back to the same URI. self.assertEqual(interact_netscape(cj, uri), value) - def test_escape_path(self): - cases = [ + @support.subTests('arg,result', [ # quoted safe ("/foo%2f/bar", "/foo%2F/bar"), ("/foo%2F/bar", "/foo%2F/bar"), @@ -799,9 +788,9 @@ def test_escape_path(self): ("/foo/bar\u00fc", "/foo/bar%C3%BC"), # UTF-8 encoded # unicode ("/foo/bar\uabcd", "/foo/bar%EA%AF%8D"), # UTF-8 encoded - ] - for arg, result in cases: - self.assertEqual(escape_path(arg), result) + ]) + def test_escape_path(self, arg, result): + self.assertEqual(escape_path(arg), result) def test_request_path(self): # with parameters diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index b891d0734ca0a6..22f6403d482bc4 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -7,6 +7,7 @@ import unittest import warnings from ntpath import ALLOW_MISSING +from test import support from test.support import TestFailed, cpython_only, os_helper from test.support.os_helper import FakePath from test import test_genericpath @@ -78,24 +79,7 @@ def tester(fn, wantResult): def _parameterize(*parameters): - """Simplistic decorator to parametrize a test - - Runs the decorated test multiple times in subTest, with a value from - 'parameters' passed as an extra positional argument. - Calls doCleanups() after each run. - - Not for general use. Intended to avoid indenting for easier backports. - - See https://discuss.python.org/t/91827 for discussing generalizations. - """ - def _parametrize_decorator(func): - def _parameterized(self, *args, **kwargs): - for parameter in parameters: - with self.subTest(parameter): - func(self, *args, parameter, **kwargs) - self.doCleanups() - return _parameterized - return _parametrize_decorator + return support.subTests('kwargs', parameters, _do_cleanups=True) class NtpathTestCase(unittest.TestCase): diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index c45ce6d3ef7820..21f06712548d88 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -36,23 +36,7 @@ def skip_if_ABSTFN_contains_backslash(test): def _parameterize(*parameters): - """Simplistic decorator to parametrize a test - - Runs the decorated test multiple times in subTest, with a value from - 'parameters' passed as an extra positional argument. - Does *not* call doCleanups() after each run. - - Not for general use. Intended to avoid indenting for easier backports. - - See https://discuss.python.org/t/91827 for discussing generalizations. - """ - def _parametrize_decorator(func): - def _parameterized(self, *args, **kwargs): - for parameter in parameters: - with self.subTest(parameter): - func(self, *args, parameter, **kwargs) - return _parameterized - return _parametrize_decorator + return support.subTests('kwargs', parameters) class PosixPathTest(unittest.TestCase): diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index aabc360289a0d0..b2bde5a9b1d696 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -2,6 +2,7 @@ import unicodedata import unittest import urllib.parse +from test import support RFC1808_BASE = "http://a/b/c/d;p?q#f" RFC2396_BASE = "http://a/b/c/d;p?q" @@ -156,27 +157,25 @@ def checkRoundtrips(self, url, parsed, split, url2=None): self.assertEqual(result3.hostname, result.hostname) self.assertEqual(result3.port, result.port) - def test_qsl(self): - for orig, expect in parse_qsl_test_cases: - result = urllib.parse.parse_qsl(orig, keep_blank_values=True) - self.assertEqual(result, expect, "Error parsing %r" % orig) - expect_without_blanks = [v for v in expect if len(v[1])] - result = urllib.parse.parse_qsl(orig, keep_blank_values=False) - self.assertEqual(result, expect_without_blanks, - "Error parsing %r" % orig) - - def test_qs(self): - for orig, expect in parse_qs_test_cases: - result = urllib.parse.parse_qs(orig, keep_blank_values=True) - self.assertEqual(result, expect, "Error parsing %r" % orig) - expect_without_blanks = {v: expect[v] - for v in expect if len(expect[v][0])} - result = urllib.parse.parse_qs(orig, keep_blank_values=False) - self.assertEqual(result, expect_without_blanks, - "Error parsing %r" % orig) - - def test_roundtrips(self): - str_cases = [ + @support.subTests('orig,expect', parse_qsl_test_cases) + def test_qsl(self, orig, expect): + result = urllib.parse.parse_qsl(orig, keep_blank_values=True) + self.assertEqual(result, expect) + expect_without_blanks = [v for v in expect if len(v[1])] + result = urllib.parse.parse_qsl(orig, keep_blank_values=False) + self.assertEqual(result, expect_without_blanks) + + @support.subTests('orig,expect', parse_qs_test_cases) + def test_qs(self, orig, expect): + result = urllib.parse.parse_qs(orig, keep_blank_values=True) + self.assertEqual(result, expect) + expect_without_blanks = {v: expect[v] + for v in expect if len(expect[v][0])} + result = urllib.parse.parse_qs(orig, keep_blank_values=False) + self.assertEqual(result, expect_without_blanks) + + @support.subTests('bytes', (False, True)) + @support.subTests('url,parsed,split', [ ('path/to/file', ('', '', 'path/to/file', '', '', ''), ('', '', 'path/to/file', '', '')), @@ -263,23 +262,21 @@ def test_roundtrips(self): ('sch_me:path/to/file', ('', '', 'sch_me:path/to/file', '', '', ''), ('', '', 'sch_me:path/to/file', '', '')), - ] - def _encode(t): - return (t[0].encode('ascii'), - tuple(x.encode('ascii') for x in t[1]), - tuple(x.encode('ascii') for x in t[2])) - bytes_cases = [_encode(x) for x in str_cases] - str_cases += [ ('schème:path/to/file', ('', '', 'schème:path/to/file', '', '', ''), ('', '', 'schème:path/to/file', '', '')), - ] - for url, parsed, split in str_cases + bytes_cases: - with self.subTest(url): - self.checkRoundtrips(url, parsed, split) - - def test_roundtrips_normalization(self): - str_cases = [ + ]) + def test_roundtrips(self, bytes, url, parsed, split): + if bytes: + if not url.isascii(): + self.skipTest('non-ASCII bytes') + url = str_encode(url) + parsed = tuple_encode(parsed) + split = tuple_encode(split) + self.checkRoundtrips(url, parsed, split) + + @support.subTests('bytes', (False, True)) + @support.subTests('url,url2,parsed,split', [ ('///path/to/file', '/path/to/file', ('', '', '/path/to/file', '', '', ''), @@ -300,22 +297,18 @@ def test_roundtrips_normalization(self): 'https:///tmp/junk.txt', ('https', '', '/tmp/junk.txt', '', '', ''), ('https', '', '/tmp/junk.txt', '', '')), - ] - def _encode(t): - return (t[0].encode('ascii'), - t[1].encode('ascii'), - tuple(x.encode('ascii') for x in t[2]), - tuple(x.encode('ascii') for x in t[3])) - bytes_cases = [_encode(x) for x in str_cases] - for url, url2, parsed, split in str_cases + bytes_cases: - with self.subTest(url): - self.checkRoundtrips(url, parsed, split, url2) - - def test_http_roundtrips(self): - # urllib.parse.urlsplit treats 'http:' as an optimized special case, - # so we test both 'http:' and 'https:' in all the following. - # Three cheers for white box knowledge! - str_cases = [ + ]) + def test_roundtrips_normalization(self, bytes, url, url2, parsed, split): + if bytes: + url = str_encode(url) + url2 = str_encode(url2) + parsed = tuple_encode(parsed) + split = tuple_encode(split) + self.checkRoundtrips(url, parsed, split, url2) + + @support.subTests('bytes', (False, True)) + @support.subTests('scheme', ('http', 'https')) + @support.subTests('url,parsed,split', [ ('://www.python.org', ('www.python.org', '', '', '', ''), ('www.python.org', '', '', '')), @@ -331,23 +324,20 @@ def test_http_roundtrips(self): ('://a/b/c/d;p?q#f', ('a', '/b/c/d', 'p', 'q', 'f'), ('a', '/b/c/d;p', 'q', 'f')), - ] - def _encode(t): - return (t[0].encode('ascii'), - tuple(x.encode('ascii') for x in t[1]), - tuple(x.encode('ascii') for x in t[2])) - bytes_cases = [_encode(x) for x in str_cases] - str_schemes = ('http', 'https') - bytes_schemes = (b'http', b'https') - str_tests = str_schemes, str_cases - bytes_tests = bytes_schemes, bytes_cases - for schemes, test_cases in (str_tests, bytes_tests): - for scheme in schemes: - for url, parsed, split in test_cases: - url = scheme + url - parsed = (scheme,) + parsed - split = (scheme,) + split - self.checkRoundtrips(url, parsed, split) + ]) + def test_http_roundtrips(self, bytes, scheme, url, parsed, split): + # urllib.parse.urlsplit treats 'http:' as an optimized special case, + # so we test both 'http:' and 'https:' in all the following. + # Three cheers for white box knowledge! + if bytes: + scheme = str_encode(scheme) + url = str_encode(url) + parsed = tuple_encode(parsed) + split = tuple_encode(split) + url = scheme + url + parsed = (scheme,) + parsed + split = (scheme,) + split + self.checkRoundtrips(url, parsed, split) def checkJoin(self, base, relurl, expected, *, relroundtrip=True): with self.subTest(base=base, relurl=relurl): @@ -363,12 +353,13 @@ def checkJoin(self, base, relurl, expected, *, relroundtrip=True): relurlb = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurlb)) self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) - def test_unparse_parse(self): - str_cases = ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',] - bytes_cases = [x.encode('ascii') for x in str_cases] - for u in str_cases + bytes_cases: - self.assertEqual(urllib.parse.urlunsplit(urllib.parse.urlsplit(u)), u) - self.assertEqual(urllib.parse.urlunparse(urllib.parse.urlparse(u)), u) + @support.subTests('bytes', (False, True)) + @support.subTests('u', ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',]) + def test_unparse_parse(self, bytes, u): + if bytes: + u = str_encode(u) + self.assertEqual(urllib.parse.urlunsplit(urllib.parse.urlsplit(u)), u) + self.assertEqual(urllib.parse.urlunparse(urllib.parse.urlparse(u)), u) def test_RFC1808(self): # "normal" cases from RFC 1808: @@ -695,8 +686,8 @@ def test_urljoins_relative_base(self): self.checkJoin('///b/c', '///w', '///w') self.checkJoin('///b/c', 'w', '///b/w') - def test_RFC2732(self): - str_cases = [ + @support.subTests('bytes', (False, True)) + @support.subTests('url,hostname,port', [ ('http://Test.python.org:5432/foo/', 'test.python.org', 5432), ('http://12.34.56.78:5432/foo/', '12.34.56.78', 5432), ('http://[::1]:5432/foo/', '::1', 5432), @@ -727,26 +718,28 @@ def test_RFC2732(self): ('http://[::12.34.56.78]:/foo/', '::12.34.56.78', None), ('http://[::ffff:12.34.56.78]:/foo/', '::ffff:12.34.56.78', None), - ] - def _encode(t): - return t[0].encode('ascii'), t[1].encode('ascii'), t[2] - bytes_cases = [_encode(x) for x in str_cases] - for url, hostname, port in str_cases + bytes_cases: - urlparsed = urllib.parse.urlparse(url) - self.assertEqual((urlparsed.hostname, urlparsed.port) , (hostname, port)) - - str_cases = [ + ]) + def test_RFC2732(self, bytes, url, hostname, port): + if bytes: + url = str_encode(url) + hostname = str_encode(hostname) + urlparsed = urllib.parse.urlparse(url) + self.assertEqual((urlparsed.hostname, urlparsed.port), (hostname, port)) + + @support.subTests('bytes', (False, True)) + @support.subTests('invalid_url', [ 'http://::12.34.56.78]/', 'http://[::1/foo/', 'ftp://[::1/foo/bad]/bad', 'http://[::1/foo/bad]/bad', - 'http://[::ffff:12.34.56.78'] - bytes_cases = [x.encode('ascii') for x in str_cases] - for invalid_url in str_cases + bytes_cases: - self.assertRaises(ValueError, urllib.parse.urlparse, invalid_url) - - def test_urldefrag(self): - str_cases = [ + 'http://[::ffff:12.34.56.78']) + def test_RFC2732_invalid(self, bytes, invalid_url): + if bytes: + invalid_url = str_encode(invalid_url) + self.assertRaises(ValueError, urllib.parse.urlparse, invalid_url) + + @support.subTests('bytes', (False, True)) + @support.subTests('url,defrag,frag', [ ('http://python.org#frag', 'http://python.org', 'frag'), ('http://python.org', 'http://python.org', ''), ('http://python.org/#frag', 'http://python.org/', 'frag'), @@ -770,18 +763,18 @@ def test_urldefrag(self): ('http:?q#f', 'http:?q', 'f'), ('//a/b/c;p?q#f', '//a/b/c;p?q', 'f'), ('://a/b/c;p?q#f', '://a/b/c;p?q', 'f'), - ] - def _encode(t): - return type(t)(x.encode('ascii') for x in t) - bytes_cases = [_encode(x) for x in str_cases] - for url, defrag, frag in str_cases + bytes_cases: - with self.subTest(url): - result = urllib.parse.urldefrag(url) - hash = '#' if isinstance(url, str) else b'#' - self.assertEqual(result.geturl(), url.rstrip(hash)) - self.assertEqual(result, (defrag, frag)) - self.assertEqual(result.url, defrag) - self.assertEqual(result.fragment, frag) + ]) + def test_urldefrag(self, bytes, url, defrag, frag): + if bytes: + url = str_encode(url) + defrag = str_encode(defrag) + frag = str_encode(frag) + result = urllib.parse.urldefrag(url) + hash = '#' if isinstance(url, str) else b'#' + self.assertEqual(result.geturl(), url.rstrip(hash)) + self.assertEqual(result, (defrag, frag)) + self.assertEqual(result.url, defrag) + self.assertEqual(result.fragment, frag) def test_urlsplit_scoped_IPv6(self): p = urllib.parse.urlsplit('http://[FE80::822a:a8ff:fe49:470c%tESt]:1234') @@ -981,42 +974,35 @@ def test_urlsplit_strip_url(self): self.assertEqual(p.scheme, "https") self.assertEqual(p.geturl(), "https://www.python.org/") - def test_attributes_bad_port(self): + @support.subTests('bytes', (False, True)) + @support.subTests('parse', (urllib.parse.urlsplit, urllib.parse.urlparse)) + @support.subTests('port', ("foo", "1.5", "-1", "0x10", "-0", "1_1", " 1", "1 ", "६")) + def test_attributes_bad_port(self, bytes, parse, port): """Check handling of invalid ports.""" - for bytes in (False, True): - for parse in (urllib.parse.urlsplit, urllib.parse.urlparse): - for port in ("foo", "1.5", "-1", "0x10", "-0", "1_1", " 1", "1 ", "६"): - with self.subTest(bytes=bytes, parse=parse, port=port): - netloc = "www.example.net:" + port - url = "http://" + netloc + "/" - if bytes: - if netloc.isascii() and port.isascii(): - netloc = netloc.encode("ascii") - url = url.encode("ascii") - else: - continue - p = parse(url) - self.assertEqual(p.netloc, netloc) - with self.assertRaises(ValueError): - p.port + netloc = "www.example.net:" + port + url = "http://" + netloc + "/" + if bytes: + if not (netloc.isascii() and port.isascii()): + self.skipTest('non-ASCII bytes') + netloc = str_encode(netloc) + url = str_encode(url) + p = parse(url) + self.assertEqual(p.netloc, netloc) + with self.assertRaises(ValueError): + p.port - def test_attributes_bad_scheme(self): + @support.subTests('bytes', (False, True)) + @support.subTests('parse', (urllib.parse.urlsplit, urllib.parse.urlparse)) + @support.subTests('scheme', (".", "+", "-", "0", "http&", "६http")) + def test_attributes_bad_scheme(self, bytes, parse, scheme): """Check handling of invalid schemes.""" - for bytes in (False, True): - for parse in (urllib.parse.urlsplit, urllib.parse.urlparse): - for scheme in (".", "+", "-", "0", "http&", "६http"): - with self.subTest(bytes=bytes, parse=parse, scheme=scheme): - url = scheme + "://www.example.net" - if bytes: - if url.isascii(): - url = url.encode("ascii") - else: - continue - p = parse(url) - if bytes: - self.assertEqual(p.scheme, b"") - else: - self.assertEqual(p.scheme, "") + url = scheme + "://www.example.net" + if bytes: + if not url.isascii(): + self.skipTest('non-ASCII bytes') + url = url.encode("ascii") + p = parse(url) + self.assertEqual(p.scheme, b"" if bytes else "") def test_attributes_without_netloc(self): # This example is straight from RFC 3261. It looks like it @@ -1128,24 +1114,21 @@ def test_anyscheme(self): self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff?query"), (b'x-newscheme', b'foo.com', b'/stuff', b'', b'query', b'')) - def test_default_scheme(self): + @support.subTests('func', (urllib.parse.urlparse, urllib.parse.urlsplit)) + def test_default_scheme(self, func): # Exercise the scheme parameter of urlparse() and urlsplit() - for func in (urllib.parse.urlparse, urllib.parse.urlsplit): - with self.subTest(function=func): - result = func("http://example.net/", "ftp") - self.assertEqual(result.scheme, "http") - result = func(b"http://example.net/", b"ftp") - self.assertEqual(result.scheme, b"http") - self.assertEqual(func("path", "ftp").scheme, "ftp") - self.assertEqual(func("path", scheme="ftp").scheme, "ftp") - self.assertEqual(func(b"path", scheme=b"ftp").scheme, b"ftp") - self.assertEqual(func("path").scheme, "") - self.assertEqual(func(b"path").scheme, b"") - self.assertEqual(func(b"path", "").scheme, b"") - - def test_parse_fragments(self): - # Exercise the allow_fragments parameter of urlparse() and urlsplit() - tests = ( + result = func("http://example.net/", "ftp") + self.assertEqual(result.scheme, "http") + result = func(b"http://example.net/", b"ftp") + self.assertEqual(result.scheme, b"http") + self.assertEqual(func("path", "ftp").scheme, "ftp") + self.assertEqual(func("path", scheme="ftp").scheme, "ftp") + self.assertEqual(func(b"path", scheme=b"ftp").scheme, b"ftp") + self.assertEqual(func("path").scheme, "") + self.assertEqual(func(b"path").scheme, b"") + self.assertEqual(func(b"path", "").scheme, b"") + + @support.subTests('url,attr,expected_frag', ( ("http:#frag", "path", "frag"), ("//example.net#frag", "path", "frag"), ("index.html#frag", "path", "frag"), @@ -1156,24 +1139,24 @@ def test_parse_fragments(self): ("//abc#@frag", "path", "@frag"), ("//abc:80#@frag", "path", "@frag"), ("//abc#@frag:80", "path", "@frag:80"), - ) - for url, attr, expected_frag in tests: - for func in (urllib.parse.urlparse, urllib.parse.urlsplit): - if attr == "params" and func is urllib.parse.urlsplit: - attr = "path" - with self.subTest(url=url, function=func): - result = func(url, allow_fragments=False) - self.assertEqual(result.fragment, "") - self.assertEndsWith(getattr(result, attr), - "#" + expected_frag) - self.assertEqual(func(url, "", False).fragment, "") - - result = func(url, allow_fragments=True) - self.assertEqual(result.fragment, expected_frag) - self.assertNotEndsWith(getattr(result, attr), expected_frag) - self.assertEqual(func(url, "", True).fragment, - expected_frag) - self.assertEqual(func(url).fragment, expected_frag) + )) + @support.subTests('func', (urllib.parse.urlparse, urllib.parse.urlsplit)) + def test_parse_fragments(self, url, attr, expected_frag, func): + # Exercise the allow_fragments parameter of urlparse() and urlsplit() + if attr == "params" and func is urllib.parse.urlsplit: + attr = "path" + result = func(url, allow_fragments=False) + self.assertEqual(result.fragment, "") + self.assertEndsWith(getattr(result, attr), + "#" + expected_frag) + self.assertEqual(func(url, "", False).fragment, "") + + result = func(url, allow_fragments=True) + self.assertEqual(result.fragment, expected_frag) + self.assertNotEndsWith(getattr(result, attr), expected_frag) + self.assertEqual(func(url, "", True).fragment, + expected_frag) + self.assertEqual(func(url).fragment, expected_frag) def test_mixed_types_rejected(self): # Several functions that process either strings or ASCII encoded bytes @@ -1199,7 +1182,14 @@ def test_mixed_types_rejected(self): with self.assertRaisesRegex(TypeError, "Cannot mix str"): urllib.parse.urljoin(b"http://python.org", "http://python.org") - def _check_result_type(self, str_type): + @support.subTests('result_type', [ + urllib.parse.DefragResult, + urllib.parse.SplitResult, + urllib.parse.ParseResult, + ]) + def test_result_pairs(self, result_type): + # Check encoding and decoding between result pairs + str_type = result_type num_args = len(str_type._fields) bytes_type = str_type._encoded_counterpart self.assertIs(bytes_type._decoded_counterpart, str_type) @@ -1224,16 +1214,6 @@ def _check_result_type(self, str_type): self.assertEqual(str_result.encode(encoding, errors), bytes_args) self.assertEqual(str_result.encode(encoding, errors), bytes_result) - def test_result_pairs(self): - # Check encoding and decoding between result pairs - result_types = [ - urllib.parse.DefragResult, - urllib.parse.SplitResult, - urllib.parse.ParseResult, - ] - for result_type in result_types: - self._check_result_type(result_type) - def test_parse_qs_encoding(self): result = urllib.parse.parse_qs("key=\u0141%E9", encoding="latin-1") self.assertEqual(result, {'key': ['\u0141\xE9']}) @@ -1265,8 +1245,7 @@ def test_parse_qsl_max_num_fields(self): urllib.parse.parse_qsl('&'.join(['a=a']*11), max_num_fields=10) urllib.parse.parse_qsl('&'.join(['a=a']*10), max_num_fields=10) - def test_parse_qs_separator(self): - parse_qs_semicolon_cases = [ + @support.subTests('orig,expect', [ (";", {}), (";;", {}), (";a=b", {'a': ['b']}), @@ -1277,17 +1256,14 @@ def test_parse_qs_separator(self): (b";a=b", {b'a': [b'b']}), (b"a=a+b;b=b+c", {b'a': [b'a b'], b'b': [b'b c']}), (b"a=1;a=2", {b'a': [b'1', b'2']}), - ] - for orig, expect in parse_qs_semicolon_cases: - with self.subTest(f"Original: {orig!r}, Expected: {expect!r}"): - result = urllib.parse.parse_qs(orig, separator=';') - self.assertEqual(result, expect, "Error parsing %r" % orig) - result_bytes = urllib.parse.parse_qs(orig, separator=b';') - self.assertEqual(result_bytes, expect, "Error parsing %r" % orig) - - - def test_parse_qsl_separator(self): - parse_qsl_semicolon_cases = [ + ]) + def test_parse_qs_separator(self, orig, expect): + result = urllib.parse.parse_qs(orig, separator=';') + self.assertEqual(result, expect) + result_bytes = urllib.parse.parse_qs(orig, separator=b';') + self.assertEqual(result_bytes, expect) + + @support.subTests('orig,expect', [ (";", []), (";;", []), (";a=b", [('a', 'b')]), @@ -1298,13 +1274,12 @@ def test_parse_qsl_separator(self): (b";a=b", [(b'a', b'b')]), (b"a=a+b;b=b+c", [(b'a', b'a b'), (b'b', b'b c')]), (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]), - ] - for orig, expect in parse_qsl_semicolon_cases: - with self.subTest(f"Original: {orig!r}, Expected: {expect!r}"): - result = urllib.parse.parse_qsl(orig, separator=';') - self.assertEqual(result, expect, "Error parsing %r" % orig) - result_bytes = urllib.parse.parse_qsl(orig, separator=b';') - self.assertEqual(result_bytes, expect, "Error parsing %r" % orig) + ]) + def test_parse_qsl_separator(self, orig, expect): + result = urllib.parse.parse_qsl(orig, separator=';') + self.assertEqual(result, expect) + result_bytes = urllib.parse.parse_qsl(orig, separator=b';') + self.assertEqual(result_bytes, expect) def test_parse_qsl_bytes(self): self.assertEqual(urllib.parse.parse_qsl(b'a=b'), [(b'a', b'b')]) @@ -1695,11 +1670,12 @@ def test_to_bytes(self): self.assertRaises(UnicodeError, urllib.parse._to_bytes, 'http://www.python.org/medi\u00e6val') - def test_unwrap(self): - for wrapped_url in ('', '', - 'URL:scheme://host/path', 'scheme://host/path'): - url = urllib.parse.unwrap(wrapped_url) - self.assertEqual(url, 'scheme://host/path') + @support.subTests('wrapped_url', + ('', '', + 'URL:scheme://host/path', 'scheme://host/path')) + def test_unwrap(self, wrapped_url): + url = urllib.parse.unwrap(wrapped_url) + self.assertEqual(url, 'scheme://host/path') class DeprecationTest(unittest.TestCase): @@ -1780,5 +1756,11 @@ def test_to_bytes_deprecation(self): 'urllib.parse.to_bytes() is deprecated as of 3.8') +def str_encode(s): + return s.encode('ascii') + +def tuple_encode(t): + return tuple(str_encode(x) for x in t) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Tests/2025-06-04-13-07-44.gh-issue-135120.NapnZT.rst b/Misc/NEWS.d/next/Tests/2025-06-04-13-07-44.gh-issue-135120.NapnZT.rst new file mode 100644 index 00000000000000..772173774b1ac1 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-06-04-13-07-44.gh-issue-135120.NapnZT.rst @@ -0,0 +1 @@ +Add :func:`!test.support.subTests`. From e004cf8fd5c006a7a1c60807a03066f4c43452e5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 6 Jun 2025 14:11:49 +0200 Subject: [PATCH 096/116] gh-134993: Add os.lstat() to os.supports_dir_fd (#135188) --- Lib/os.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/os.py b/Lib/os.py index 266e40b56f6c81..643a7b2f58176a 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -118,6 +118,7 @@ def _add(str, fn): _add("HAVE_FCHMODAT", "chmod") _add("HAVE_FCHOWNAT", "chown") _add("HAVE_FSTATAT", "stat") + _add("HAVE_LSTAT", "lstat") _add("HAVE_FUTIMESAT", "utime") _add("HAVE_LINKAT", "link") _add("HAVE_MKDIRAT", "mkdir") From 82415acf62f039b384e7ffdc5a24ff1f1207cfe5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 6 Jun 2025 14:22:14 +0200 Subject: [PATCH 097/116] gh-134036: Update test_syntax for gh-133999 (#135204) --- Lib/test/test_syntax.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 965c096475f590..13aaba405e3204 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1436,17 +1436,17 @@ >>> try: pass ... except TypeError as name: raise from None Traceback (most recent call last): - SyntaxError: invalid syntax + SyntaxError: did you forget an expression between 'raise' and 'from'? >>> try: pass ... except* TypeError as name: raise from None Traceback (most recent call last): - SyntaxError: invalid syntax + SyntaxError: did you forget an expression between 'raise' and 'from'? >>> match 1: ... case 1 | 2 as abc: raise from None Traceback (most recent call last): - SyntaxError: invalid syntax + SyntaxError: did you forget an expression between 'raise' and 'from'? Ensure that early = are not matched by the parser as invalid comparisons >>> f(2, 4, x=34); 1 $ 2 From 39859fcac5c8335ca9c1bddc5945a7d3e934aebc Mon Sep 17 00:00:00 2001 From: Rafael Fontenelle Date: Fri, 6 Jun 2025 09:23:07 -0300 Subject: [PATCH 098/116] Fix versionadded directive rendering in c-api/arg.rst (#135199) --- Doc/c-api/arg.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 3bbc990b6329c0..49dbc8d71cce62 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -685,6 +685,7 @@ Building values ``p`` (:class:`bool`) [int] Convert a C :c:expr:`int` to a Python :class:`bool` object. + .. versionadded:: 3.14 ``c`` (:class:`bytes` of length 1) [char] From 343182853f19a42c0ba8980d3104076a8c7bcfe7 Mon Sep 17 00:00:00 2001 From: Jean-Louis GUENEGO Date: Fri, 6 Jun 2025 15:16:12 +0200 Subject: [PATCH 099/116] feat(docs): type fix - apply pep8 by using docstring instead of comment in the doc. (#135181) Giving the right example incitates the tutorial readers to do the same in the future. --- Doc/tutorial/modules.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index de7aa0e2342946..47bf7547b4ae1d 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -27,14 +27,16 @@ called :file:`fibo.py` in the current directory with the following contents:: # Fibonacci numbers module - def fib(n): # write Fibonacci series up to n + def fib(n): + """Write Fibonacci series up to n.""" a, b = 0, 1 while a < n: print(a, end=' ') a, b = b, a+b print() - def fib2(n): # return Fibonacci series up to n + def fib2(n): + """Return Fibonacci series up to n.""" result = [] a, b = 0, 1 while a < n: From e413e2671916ed8f4513af92830f4fb2bc59b1d2 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 6 Jun 2025 15:51:00 +0200 Subject: [PATCH 100/116] gh-134891: Add PyUnstable_Unicode_GET_CACHED_HASH (GH-134892) --- Doc/c-api/unicode.rst | 16 ++++++++++++++++ Doc/whatsnew/3.15.rst | 4 ++++ Include/cpython/unicodeobject.h | 11 +++++++++++ Lib/test/test_capi/test_unicode.py | 14 ++++++++++++++ ...025-05-29-16-56-23.gh-issue-134891.7eKO8U.rst | 2 ++ Modules/_testcapi/unicode.c | 7 +++++++ Objects/unicodeobject.c | 6 +----- 7 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-05-29-16-56-23.gh-issue-134891.7eKO8U.rst diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 45f50ba5f97d26..07fdcfd9729ee0 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -191,6 +191,22 @@ access to internal read-only data of Unicode objects: .. versionadded:: 3.2 +.. c:function:: Py_hash_t PyUnstable_Unicode_GET_CACHED_HASH(PyObject *str) + + If the hash of *str*, as returned by :c:func:`PyObject_Hash`, has been + cached and is immediately available, return it. + Otherwise, return ``-1`` *without* setting an exception. + + If *str* is not a string (that is, if ``PyUnicode_Check(obj)`` + is false), the behavior is undefined. + + This function never fails with an exception. + + Note that there are no guarantees on when an object's hash is cached, + and the (non-)existence of a cached hash does not imply that the string has + any other properties. + + Unicode Character Properties """""""""""""""""""""""""""" diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index daf3e8fb6c2c2b..e5bccf17eb63e2 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -304,6 +304,10 @@ New features input string contains non-ASCII characters. (Contributed by Victor Stinner in :gh:`133968`.) +* Add :c:type:`PyUnstable_Unicode_GET_CACHED_HASH` to get the cached hash of + a string. See the documentation for caveats. + (Contributed by Petr Viktorin in :gh:`131510`) + Porting to Python 3.15 ---------------------- diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 3d0414f5291fe4..7c1aac9696dec9 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -300,6 +300,17 @@ static inline Py_ssize_t PyUnicode_GET_LENGTH(PyObject *op) { } #define PyUnicode_GET_LENGTH(op) PyUnicode_GET_LENGTH(_PyObject_CAST(op)) +/* Returns the cached hash, or -1 if not cached yet. */ +static inline Py_hash_t +PyUnstable_Unicode_GET_CACHED_HASH(PyObject *op) { + assert(PyUnicode_Check(op)); +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&_PyASCIIObject_CAST(op)->hash); +#else + return _PyASCIIObject_CAST(op)->hash; +#endif +} + /* Write into the canonical representation, this function does not do any sanity checks and is intended for usage in loops. The caller should cache the kind and data pointers obtained from other function calls. diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index c8be4f3faa9483..6a9c60f3a6d75e 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -1739,6 +1739,20 @@ def test_pep393_utf8_caching_bug(self): # Check that the second call returns the same result self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_GET_CACHED_HASH(self): + from _testcapi import unicode_GET_CACHED_HASH + content_bytes = b'some new string' + # avoid parser interning & constant folding + obj = str(content_bytes, 'ascii') + # impl detail: fresh strings do not have cached hash + self.assertEqual(unicode_GET_CACHED_HASH(obj), -1) + # impl detail: adding string to a dict caches its hash + {obj: obj} + # impl detail: ASCII string hashes are equal to bytes ones + self.assertEqual(unicode_GET_CACHED_HASH(obj), hash(content_bytes)) + class PyUnicodeWriterTest(unittest.TestCase): def create_writer(self, size): diff --git a/Misc/NEWS.d/next/C_API/2025-05-29-16-56-23.gh-issue-134891.7eKO8U.rst b/Misc/NEWS.d/next/C_API/2025-05-29-16-56-23.gh-issue-134891.7eKO8U.rst new file mode 100644 index 00000000000000..db30d5e9a94584 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-05-29-16-56-23.gh-issue-134891.7eKO8U.rst @@ -0,0 +1,2 @@ +Add :c:type:`PyUnstable_Unicode_GET_CACHED_HASH` to get the cached hash of a +string. diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index e70f5c68bc3b69..203282dd53dd0a 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -220,6 +220,12 @@ unicode_copycharacters(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", to_copy, copied); } +static PyObject* +unicode_GET_CACHED_HASH(PyObject *self, PyObject *arg) +{ + return PyLong_FromSsize_t(PyUnstable_Unicode_GET_CACHED_HASH(arg)); +} + // --- PyUnicodeWriter type ------------------------------------------------- @@ -570,6 +576,7 @@ static PyMethodDef TestMethods[] = { {"unicode_asucs4copy", unicode_asucs4copy, METH_VARARGS}, {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, {"unicode_copycharacters", unicode_copycharacters, METH_VARARGS}, + {"unicode_GET_CACHED_HASH", unicode_GET_CACHED_HASH, METH_O}, {NULL}, }; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 5611f839627a2e..5c2308a012142a 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -167,11 +167,7 @@ static inline void PyUnicode_SET_UTF8_LENGTH(PyObject *op, Py_ssize_t length) #define _PyUnicode_HASH(op) \ (_PyASCIIObject_CAST(op)->hash) -static inline Py_hash_t PyUnicode_HASH(PyObject *op) -{ - assert(_PyUnicode_CHECK(op)); - return FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyASCIIObject_CAST(op)->hash); -} +#define PyUnicode_HASH PyUnstable_Unicode_GET_CACHED_HASH static inline void PyUnicode_SET_HASH(PyObject *op, Py_hash_t hash) { From 1adca08d658ee2d520f3193960eaf3ae2ead1cef Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 6 Jun 2025 16:08:58 +0200 Subject: [PATCH 101/116] gh-134160: Use PyModuleDef.m_free in the example module xxlimited (GH-135174) Co-authored-by: neonene <53406459+neonene@users.noreply.github.com> --- Modules/xxlimited.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 26ac35734fb060..0480fb0849876b 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -424,6 +424,13 @@ xx_clear(PyObject *module) return 0; } +static void +xx_free(void *module) +{ + // allow xx_modexec to omit calling xx_clear on error + (void)xx_clear((PyObject *)module); +} + static struct PyModuleDef xxmodule = { PyModuleDef_HEAD_INIT, .m_name = "xxlimited", @@ -433,9 +440,7 @@ static struct PyModuleDef xxmodule = { .m_slots = xx_slots, .m_traverse = xx_traverse, .m_clear = xx_clear, - /* m_free is not necessary here: xx_clear clears all references, - * and the module state is deallocated along with the module. - */ + .m_free = xx_free, }; From b22b964a5c245e13ae0439c5efbb8984acf607ab Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 6 Jun 2025 16:51:40 +0200 Subject: [PATCH 102/116] gh-57089: Note _layout_ in the bitfield docs (GH-134148) Co-authored-by: Meador Inge Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/library/ctypes.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 8e74c6c9dee744..2ee4450698a107 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -714,10 +714,16 @@ item in the :attr:`~Structure._fields_` tuples:: ... ("second_16", c_int, 16)] ... >>> print(Int.first_16) - + >>> print(Int.second_16) - - >>> + + +It is important to note that bit field allocation and layout in memory are not +defined as a C standard; their implementation is compiler-specific. +By default, Python will attempt to match the behavior of a "native" compiler +for the current platform. +See the :attr:`~Structure._layout_` attribute for details on the default +behavior and how to change it. .. _ctypes-arrays: From 62b3d2d443785c4ea5262edb4f9f7040440f9463 Mon Sep 17 00:00:00 2001 From: Tan Long Date: Fri, 6 Jun 2025 22:52:41 +0800 Subject: [PATCH 103/116] gh-133390: Support SQL keyword completion for sqlite3 CLI (#133393) --- Doc/whatsnew/3.15.rst | 7 ++ Lib/sqlite3/__main__.py | 11 +-- Lib/sqlite3/_completer.py | 42 ++++++++ Lib/test/test_sqlite3/test_cli.py | 98 +++++++++++++++++++ Misc/ACKS | 1 + ...-05-05-03-14-08.gh-issue-133390.AuTggn.rst | 1 + Modules/_sqlite/module.c | 39 ++++++++ 7 files changed, 193 insertions(+), 6 deletions(-) create mode 100644 Lib/sqlite3/_completer.py create mode 100644 Misc/NEWS.d/next/Library/2025-05-05-03-14-08.gh-issue-133390.AuTggn.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index e5bccf17eb63e2..2f8335a895c70b 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -134,6 +134,13 @@ shelve (Contributed by Andrea Oliveri in :gh:`134004`.) +sqlite3 +------- + +* Support SQL keyword completion in the :mod:`sqlite3` command-line interface. + (Contributed by Long Tan in :gh:`133393`.) + + ssl --- diff --git a/Lib/sqlite3/__main__.py b/Lib/sqlite3/__main__.py index c2fa23c46cf990..9e74b49ee828bc 100644 --- a/Lib/sqlite3/__main__.py +++ b/Lib/sqlite3/__main__.py @@ -12,6 +12,8 @@ from textwrap import dedent from _colorize import get_theme, theme_no_color +from ._completer import completer + def execute(c, sql, suppress_errors=True, theme=theme_no_color): """Helper that wraps execution of SQL code. @@ -136,12 +138,9 @@ def main(*args): execute(con, args.sql, suppress_errors=False, theme=theme) else: # No SQL provided; start the REPL. - console = SqliteInteractiveConsole(con, use_color=True) - try: - import readline # noqa: F401 - except ImportError: - pass - console.interact(banner, exitmsg="") + with completer(): + console = SqliteInteractiveConsole(con, use_color=True) + console.interact(banner, exitmsg="") finally: con.close() diff --git a/Lib/sqlite3/_completer.py b/Lib/sqlite3/_completer.py new file mode 100644 index 00000000000000..f21ef69cad6439 --- /dev/null +++ b/Lib/sqlite3/_completer.py @@ -0,0 +1,42 @@ +from contextlib import contextmanager + +try: + from _sqlite3 import SQLITE_KEYWORDS +except ImportError: + SQLITE_KEYWORDS = () + +_completion_matches = [] + + +def _complete(text, state): + global _completion_matches + + if state == 0: + text_upper = text.upper() + _completion_matches = [c for c in SQLITE_KEYWORDS if c.startswith(text_upper)] + try: + return _completion_matches[state] + " " + except IndexError: + return None + + +@contextmanager +def completer(): + try: + import readline + except ImportError: + yield + return + + old_completer = readline.get_completer() + try: + readline.set_completer(_complete) + if readline.backend == "editline": + # libedit uses "^I" instead of "tab" + command_string = "bind ^I rl_complete" + else: + command_string = "tab: complete" + readline.parse_and_bind(command_string) + yield + finally: + readline.set_completer(old_completer) diff --git a/Lib/test/test_sqlite3/test_cli.py b/Lib/test/test_sqlite3/test_cli.py index 37e0f74f688659..7f0b0f3650535a 100644 --- a/Lib/test/test_sqlite3/test_cli.py +++ b/Lib/test/test_sqlite3/test_cli.py @@ -1,14 +1,19 @@ """sqlite3 CLI tests.""" import sqlite3 +import sys +import textwrap import unittest from sqlite3.__main__ import main as cli +from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink +from test.support.pty_helper import run_pty from test.support import ( captured_stdout, captured_stderr, captured_stdin, force_not_colorized_test_class, + requires_subprocess, ) @@ -200,5 +205,98 @@ def test_color(self): self.assertIn('\x1b[1;35mOperationalError (SQLITE_ERROR)\x1b[0m: ' '\x1b[35mnear "sel": syntax error\x1b[0m', err) + +@requires_subprocess() +@force_not_colorized_test_class +class Completion(unittest.TestCase): + PS1 = "sqlite> " + + @classmethod + def setUpClass(cls): + _sqlite3 = import_module("_sqlite3") + if not hasattr(_sqlite3, "SQLITE_KEYWORDS"): + raise unittest.SkipTest("unable to determine SQLite keywords") + + readline = import_module("readline") + if readline.backend == "editline": + raise unittest.SkipTest("libedit readline is not supported") + + def write_input(self, input_, env=None): + script = textwrap.dedent(""" + import readline + from sqlite3.__main__ import main + + readline.parse_and_bind("set colored-completion-prefix off") + main() + """) + return run_pty(script, input_, env) + + def test_complete_sql_keywords(self): + # List candidates starting with 'S', there should be multiple matches. + input_ = b"S\t\tEL\t 1;\n.quit\n" + output = self.write_input(input_) + self.assertIn(b"SELECT", output) + self.assertIn(b"SET", output) + self.assertIn(b"SAVEPOINT", output) + self.assertIn(b"(1,)", output) + + # Keywords are completed in upper case for even lower case user input. + input_ = b"sel\t\t 1;\n.quit\n" + output = self.write_input(input_) + self.assertIn(b"SELECT", output) + self.assertIn(b"(1,)", output) + + @unittest.skipIf(sys.platform.startswith("freebsd"), + "Two actual tabs are inserted when there are no matching" + " completions in the pseudo-terminal opened by run_pty()" + " on FreeBSD") + def test_complete_no_match(self): + input_ = b"xyzzy\t\t\b\b\b\b\b\b\b.quit\n" + # Set NO_COLOR to disable coloring for self.PS1. + output = self.write_input(input_, env={"NO_COLOR": "1"}) + lines = output.decode().splitlines() + indices = ( + i for i, line in enumerate(lines, 1) + if line.startswith(f"{self.PS1}xyzzy") + ) + line_num = next(indices, -1) + self.assertNotEqual(line_num, -1) + # Completions occupy lines, assert no extra lines when there is nothing + # to complete. + self.assertEqual(line_num, len(lines)) + + def test_complete_no_input(self): + from _sqlite3 import SQLITE_KEYWORDS + + script = textwrap.dedent(""" + import readline + from sqlite3.__main__ import main + + # Configure readline to ...: + # - hide control sequences surrounding each candidate + # - hide "Display all xxx possibilities? (y or n)" + # - hide "--More--" + # - show candidates one per line + readline.parse_and_bind("set colored-completion-prefix off") + readline.parse_and_bind("set colored-stats off") + readline.parse_and_bind("set completion-query-items 0") + readline.parse_and_bind("set page-completions off") + readline.parse_and_bind("set completion-display-width 0") + + main() + """) + input_ = b"\t\t.quit\n" + output = run_pty(script, input_, env={"NO_COLOR": "1"}) + lines = output.decode().splitlines() + indices = [ + i for i, line in enumerate(lines) + if line.startswith(self.PS1) + ] + self.assertEqual(len(indices), 2) + start, end = indices + candidates = [l.strip() for l in lines[start+1:end]] + self.assertEqual(candidates, sorted(SQLITE_KEYWORDS)) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/ACKS b/Misc/ACKS index 2435943f1bb2bd..739af8d9e11a10 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1868,6 +1868,7 @@ Neil Tallim Geoff Talvola Anish Tambe Musashi Tamura +Long Tan William Tanksley Christian Tanzer Steven Taschuk diff --git a/Misc/NEWS.d/next/Library/2025-05-05-03-14-08.gh-issue-133390.AuTggn.rst b/Misc/NEWS.d/next/Library/2025-05-05-03-14-08.gh-issue-133390.AuTggn.rst new file mode 100644 index 00000000000000..38d5c311b1d437 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-05-03-14-08.gh-issue-133390.AuTggn.rst @@ -0,0 +1 @@ +Support keyword completion in the :mod:`sqlite3` command-line interface. diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 909ddd1f990e19..5464fd1227ad20 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -32,6 +32,7 @@ #include "microprotocols.h" #include "row.h" #include "blob.h" +#include "util.h" #if SQLITE_VERSION_NUMBER < 3015002 #error "SQLite 3.15.2 or higher required" @@ -404,6 +405,40 @@ pysqlite_error_name(int rc) return NULL; } +static int +add_keyword_tuple(PyObject *module) +{ +#if SQLITE_VERSION_NUMBER >= 3024000 + int count = sqlite3_keyword_count(); + PyObject *keywords = PyTuple_New(count); + if (keywords == NULL) { + return -1; + } + for (int i = 0; i < count; i++) { + const char *keyword; + int size; + int result = sqlite3_keyword_name(i, &keyword, &size); + if (result != SQLITE_OK) { + pysqlite_state *state = pysqlite_get_state(module); + set_error_from_code(state, result); + goto error; + } + PyObject *kwd = PyUnicode_FromStringAndSize(keyword, size); + if (!kwd) { + goto error; + } + PyTuple_SET_ITEM(keywords, i, kwd); + } + return PyModule_Add(module, "SQLITE_KEYWORDS", keywords); + +error: + Py_DECREF(keywords); + return -1; +#else + return 0; +#endif +} + static int add_integer_constants(PyObject *module) { #define ADD_INT(ival) \ @@ -702,6 +737,10 @@ module_exec(PyObject *module) goto error; } + if (add_keyword_tuple(module) < 0) { + goto error; + } + if (PyModule_AddStringConstant(module, "sqlite_version", sqlite3_libversion())) { goto error; } From f00512db20561370faad437853f6ecee0eec4856 Mon Sep 17 00:00:00 2001 From: Yuki Kobayashi Date: Sat, 7 Jun 2025 00:51:47 +0900 Subject: [PATCH 104/116] Docs: Update `PyExc_*` tables in the c-api documentation (GH-131640) Add `PyExc_BaseExceptionGroup` and `PyExc_EncodingWarning` --- Doc/c-api/exceptions.rst | 12 ++++++++++++ Doc/conf.py | 2 ++ 2 files changed, 14 insertions(+) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index c8e1b5c2461738..885dbeb75303d1 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -982,6 +982,7 @@ the variables: .. index:: single: PyExc_BaseException (C var) + single: PyExc_BaseExceptionGroup (C var) single: PyExc_Exception (C var) single: PyExc_ArithmeticError (C var) single: PyExc_AssertionError (C var) @@ -1041,6 +1042,8 @@ the variables: +=========================================+=================================+==========+ | :c:data:`PyExc_BaseException` | :exc:`BaseException` | [1]_ | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BaseExceptionGroup` | :exc:`BaseExceptionGroup` | [1]_ | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_Exception` | :exc:`Exception` | [1]_ | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | [1]_ | @@ -1164,6 +1167,9 @@ the variables: .. versionadded:: 3.6 :c:data:`PyExc_ModuleNotFoundError`. +.. versionadded:: 3.11 + :c:data:`PyExc_BaseExceptionGroup`. + These are compatibility aliases to :c:data:`PyExc_OSError`: .. index:: @@ -1207,6 +1213,7 @@ the variables: single: PyExc_Warning (C var) single: PyExc_BytesWarning (C var) single: PyExc_DeprecationWarning (C var) + single: PyExc_EncodingWarning (C var) single: PyExc_FutureWarning (C var) single: PyExc_ImportWarning (C var) single: PyExc_PendingDeprecationWarning (C var) @@ -1225,6 +1232,8 @@ the variables: +------------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | +------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_EncodingWarning` | :exc:`EncodingWarning` | | ++------------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | +------------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | @@ -1245,6 +1254,9 @@ the variables: .. versionadded:: 3.2 :c:data:`PyExc_ResourceWarning`. +.. versionadded:: 3.10 + :c:data:`PyExc_EncodingWarning`. + Notes: .. [3] diff --git a/Doc/conf.py b/Doc/conf.py index 7fadad66cb3238..b08f5452901b15 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -234,6 +234,7 @@ ('c:data', 'PyExc_AssertionError'), ('c:data', 'PyExc_AttributeError'), ('c:data', 'PyExc_BaseException'), + ('c:data', 'PyExc_BaseExceptionGroup'), ('c:data', 'PyExc_BlockingIOError'), ('c:data', 'PyExc_BrokenPipeError'), ('c:data', 'PyExc_BufferError'), @@ -287,6 +288,7 @@ # C API: Standard Python warning classes ('c:data', 'PyExc_BytesWarning'), ('c:data', 'PyExc_DeprecationWarning'), + ('c:data', 'PyExc_EncodingWarning'), ('c:data', 'PyExc_FutureWarning'), ('c:data', 'PyExc_ImportWarning'), ('c:data', 'PyExc_PendingDeprecationWarning'), From 46151648ca8ba1edd2a2783e8e154692a13d8ea8 Mon Sep 17 00:00:00 2001 From: "Tomas R." Date: Sat, 7 Jun 2025 01:44:43 +0200 Subject: [PATCH 105/116] GH-131798: Optimize away type(x) in the JIT when the result is known (GH-135194) --- Lib/test/test_capi/test_opt.py | 25 ++++++++++++++++--- ...-06-05-21-58-30.gh-issue-131798.nt5Ab7.rst | 2 ++ Python/optimizer_bytecodes.c | 7 ++++-- Python/optimizer_cases.c.h | 7 ++++-- 4 files changed, 34 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-05-21-58-30.gh-issue-131798.nt5Ab7.rst diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index a292ebcc7f4aed..ee8d261685d463 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1778,11 +1778,12 @@ def testfunc(n): self.assertNotIn("_GUARD_TOS_UNICODE", uops) self.assertIn("_BINARY_OP_ADD_UNICODE", uops) - def test_call_type_1(self): + def test_call_type_1_guards_removed(self): def testfunc(n): x = 0 for _ in range(n): - x += type(42) is int + foo = eval('42') + x += type(foo) is int return x res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) @@ -1793,6 +1794,25 @@ def testfunc(n): self.assertNotIn("_GUARD_NOS_NULL", uops) self.assertNotIn("_GUARD_CALLABLE_TYPE_1", uops) + def test_call_type_1_known_type(self): + def testfunc(n): + x = 0 + for _ in range(n): + x += type(42) is int + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # When the result of type(...) is known, _CALL_TYPE_1 is replaced with + # _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW which is optimized away in + # remove_unneeded_uops. + self.assertNotIn("_CALL_TYPE_1", uops) + self.assertNotIn("_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + def test_call_type_1_result_is_const(self): def testfunc(n): x = 0 @@ -1806,7 +1826,6 @@ def testfunc(n): self.assertEqual(res, TIER2_THRESHOLD) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_CALL_TYPE_1", uops) self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) def test_call_str_1(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-05-21-58-30.gh-issue-131798.nt5Ab7.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-05-21-58-30.gh-issue-131798.nt5Ab7.rst new file mode 100644 index 00000000000000..e4b5f610353f94 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-05-21-58-30.gh-issue-131798.nt5Ab7.rst @@ -0,0 +1,2 @@ +Optimize away ``_CALL_TYPE_1`` in the JIT when the return type is known. +Patch by Tomas Roun diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index b4220e2c627ecb..12efaacd8f0dfc 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -937,8 +937,11 @@ dummy_func(void) { } op(_CALL_TYPE_1, (unused, unused, arg -- res)) { - if (sym_has_type(arg)) { - res = sym_new_const(ctx, (PyObject *)sym_get_type(arg)); + PyObject* type = (PyObject *)sym_get_type(arg); + if (type) { + res = sym_new_const(ctx, type); + REPLACE_OP(this_instr, _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, 0, + (uintptr_t)type); } else { res = sym_new_not_null(ctx); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 960c683800455e..1a2d49973ee916 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2056,8 +2056,11 @@ JitOptSymbol *arg; JitOptSymbol *res; arg = stack_pointer[-1]; - if (sym_has_type(arg)) { - res = sym_new_const(ctx, (PyObject *)sym_get_type(arg)); + PyObject* type = (PyObject *)sym_get_type(arg); + if (type) { + res = sym_new_const(ctx, type); + REPLACE_OP(this_instr, _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, 0, + (uintptr_t)type); } else { res = sym_new_not_null(ctx); From 24069fbca861a5904ee7718469919e84828f22e7 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sat, 7 Jun 2025 10:56:43 +0200 Subject: [PATCH 106/116] Revert "gh-133390: Support SQL keyword completion for sqlite3 CLI (#133393)" temporarily (GH-135232) This reverts commit 62b3d2d443785c4ea5262edb4f9f7040440f9463, which broke buildbots --- Doc/whatsnew/3.15.rst | 7 -- Lib/sqlite3/__main__.py | 11 ++- Lib/sqlite3/_completer.py | 42 -------- Lib/test/test_sqlite3/test_cli.py | 98 ------------------- Misc/ACKS | 1 - ...-05-05-03-14-08.gh-issue-133390.AuTggn.rst | 1 - Modules/_sqlite/module.c | 39 -------- 7 files changed, 6 insertions(+), 193 deletions(-) delete mode 100644 Lib/sqlite3/_completer.py delete mode 100644 Misc/NEWS.d/next/Library/2025-05-05-03-14-08.gh-issue-133390.AuTggn.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 2f8335a895c70b..e5bccf17eb63e2 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -134,13 +134,6 @@ shelve (Contributed by Andrea Oliveri in :gh:`134004`.) -sqlite3 -------- - -* Support SQL keyword completion in the :mod:`sqlite3` command-line interface. - (Contributed by Long Tan in :gh:`133393`.) - - ssl --- diff --git a/Lib/sqlite3/__main__.py b/Lib/sqlite3/__main__.py index 9e74b49ee828bc..c2fa23c46cf990 100644 --- a/Lib/sqlite3/__main__.py +++ b/Lib/sqlite3/__main__.py @@ -12,8 +12,6 @@ from textwrap import dedent from _colorize import get_theme, theme_no_color -from ._completer import completer - def execute(c, sql, suppress_errors=True, theme=theme_no_color): """Helper that wraps execution of SQL code. @@ -138,9 +136,12 @@ def main(*args): execute(con, args.sql, suppress_errors=False, theme=theme) else: # No SQL provided; start the REPL. - with completer(): - console = SqliteInteractiveConsole(con, use_color=True) - console.interact(banner, exitmsg="") + console = SqliteInteractiveConsole(con, use_color=True) + try: + import readline # noqa: F401 + except ImportError: + pass + console.interact(banner, exitmsg="") finally: con.close() diff --git a/Lib/sqlite3/_completer.py b/Lib/sqlite3/_completer.py deleted file mode 100644 index f21ef69cad6439..00000000000000 --- a/Lib/sqlite3/_completer.py +++ /dev/null @@ -1,42 +0,0 @@ -from contextlib import contextmanager - -try: - from _sqlite3 import SQLITE_KEYWORDS -except ImportError: - SQLITE_KEYWORDS = () - -_completion_matches = [] - - -def _complete(text, state): - global _completion_matches - - if state == 0: - text_upper = text.upper() - _completion_matches = [c for c in SQLITE_KEYWORDS if c.startswith(text_upper)] - try: - return _completion_matches[state] + " " - except IndexError: - return None - - -@contextmanager -def completer(): - try: - import readline - except ImportError: - yield - return - - old_completer = readline.get_completer() - try: - readline.set_completer(_complete) - if readline.backend == "editline": - # libedit uses "^I" instead of "tab" - command_string = "bind ^I rl_complete" - else: - command_string = "tab: complete" - readline.parse_and_bind(command_string) - yield - finally: - readline.set_completer(old_completer) diff --git a/Lib/test/test_sqlite3/test_cli.py b/Lib/test/test_sqlite3/test_cli.py index 7f0b0f3650535a..37e0f74f688659 100644 --- a/Lib/test/test_sqlite3/test_cli.py +++ b/Lib/test/test_sqlite3/test_cli.py @@ -1,19 +1,14 @@ """sqlite3 CLI tests.""" import sqlite3 -import sys -import textwrap import unittest from sqlite3.__main__ import main as cli -from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink -from test.support.pty_helper import run_pty from test.support import ( captured_stdout, captured_stderr, captured_stdin, force_not_colorized_test_class, - requires_subprocess, ) @@ -205,98 +200,5 @@ def test_color(self): self.assertIn('\x1b[1;35mOperationalError (SQLITE_ERROR)\x1b[0m: ' '\x1b[35mnear "sel": syntax error\x1b[0m', err) - -@requires_subprocess() -@force_not_colorized_test_class -class Completion(unittest.TestCase): - PS1 = "sqlite> " - - @classmethod - def setUpClass(cls): - _sqlite3 = import_module("_sqlite3") - if not hasattr(_sqlite3, "SQLITE_KEYWORDS"): - raise unittest.SkipTest("unable to determine SQLite keywords") - - readline = import_module("readline") - if readline.backend == "editline": - raise unittest.SkipTest("libedit readline is not supported") - - def write_input(self, input_, env=None): - script = textwrap.dedent(""" - import readline - from sqlite3.__main__ import main - - readline.parse_and_bind("set colored-completion-prefix off") - main() - """) - return run_pty(script, input_, env) - - def test_complete_sql_keywords(self): - # List candidates starting with 'S', there should be multiple matches. - input_ = b"S\t\tEL\t 1;\n.quit\n" - output = self.write_input(input_) - self.assertIn(b"SELECT", output) - self.assertIn(b"SET", output) - self.assertIn(b"SAVEPOINT", output) - self.assertIn(b"(1,)", output) - - # Keywords are completed in upper case for even lower case user input. - input_ = b"sel\t\t 1;\n.quit\n" - output = self.write_input(input_) - self.assertIn(b"SELECT", output) - self.assertIn(b"(1,)", output) - - @unittest.skipIf(sys.platform.startswith("freebsd"), - "Two actual tabs are inserted when there are no matching" - " completions in the pseudo-terminal opened by run_pty()" - " on FreeBSD") - def test_complete_no_match(self): - input_ = b"xyzzy\t\t\b\b\b\b\b\b\b.quit\n" - # Set NO_COLOR to disable coloring for self.PS1. - output = self.write_input(input_, env={"NO_COLOR": "1"}) - lines = output.decode().splitlines() - indices = ( - i for i, line in enumerate(lines, 1) - if line.startswith(f"{self.PS1}xyzzy") - ) - line_num = next(indices, -1) - self.assertNotEqual(line_num, -1) - # Completions occupy lines, assert no extra lines when there is nothing - # to complete. - self.assertEqual(line_num, len(lines)) - - def test_complete_no_input(self): - from _sqlite3 import SQLITE_KEYWORDS - - script = textwrap.dedent(""" - import readline - from sqlite3.__main__ import main - - # Configure readline to ...: - # - hide control sequences surrounding each candidate - # - hide "Display all xxx possibilities? (y or n)" - # - hide "--More--" - # - show candidates one per line - readline.parse_and_bind("set colored-completion-prefix off") - readline.parse_and_bind("set colored-stats off") - readline.parse_and_bind("set completion-query-items 0") - readline.parse_and_bind("set page-completions off") - readline.parse_and_bind("set completion-display-width 0") - - main() - """) - input_ = b"\t\t.quit\n" - output = run_pty(script, input_, env={"NO_COLOR": "1"}) - lines = output.decode().splitlines() - indices = [ - i for i, line in enumerate(lines) - if line.startswith(self.PS1) - ] - self.assertEqual(len(indices), 2) - start, end = indices - candidates = [l.strip() for l in lines[start+1:end]] - self.assertEqual(candidates, sorted(SQLITE_KEYWORDS)) - - if __name__ == "__main__": unittest.main() diff --git a/Misc/ACKS b/Misc/ACKS index 739af8d9e11a10..2435943f1bb2bd 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1868,7 +1868,6 @@ Neil Tallim Geoff Talvola Anish Tambe Musashi Tamura -Long Tan William Tanksley Christian Tanzer Steven Taschuk diff --git a/Misc/NEWS.d/next/Library/2025-05-05-03-14-08.gh-issue-133390.AuTggn.rst b/Misc/NEWS.d/next/Library/2025-05-05-03-14-08.gh-issue-133390.AuTggn.rst deleted file mode 100644 index 38d5c311b1d437..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-05-05-03-14-08.gh-issue-133390.AuTggn.rst +++ /dev/null @@ -1 +0,0 @@ -Support keyword completion in the :mod:`sqlite3` command-line interface. diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 5464fd1227ad20..909ddd1f990e19 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -32,7 +32,6 @@ #include "microprotocols.h" #include "row.h" #include "blob.h" -#include "util.h" #if SQLITE_VERSION_NUMBER < 3015002 #error "SQLite 3.15.2 or higher required" @@ -405,40 +404,6 @@ pysqlite_error_name(int rc) return NULL; } -static int -add_keyword_tuple(PyObject *module) -{ -#if SQLITE_VERSION_NUMBER >= 3024000 - int count = sqlite3_keyword_count(); - PyObject *keywords = PyTuple_New(count); - if (keywords == NULL) { - return -1; - } - for (int i = 0; i < count; i++) { - const char *keyword; - int size; - int result = sqlite3_keyword_name(i, &keyword, &size); - if (result != SQLITE_OK) { - pysqlite_state *state = pysqlite_get_state(module); - set_error_from_code(state, result); - goto error; - } - PyObject *kwd = PyUnicode_FromStringAndSize(keyword, size); - if (!kwd) { - goto error; - } - PyTuple_SET_ITEM(keywords, i, kwd); - } - return PyModule_Add(module, "SQLITE_KEYWORDS", keywords); - -error: - Py_DECREF(keywords); - return -1; -#else - return 0; -#endif -} - static int add_integer_constants(PyObject *module) { #define ADD_INT(ival) \ @@ -737,10 +702,6 @@ module_exec(PyObject *module) goto error; } - if (add_keyword_tuple(module) < 0) { - goto error; - } - if (PyModule_AddStringConstant(module, "sqlite_version", sqlite3_libversion())) { goto error; } From ac9c3431cc5916a795c42b3e2b965233ceffe6f0 Mon Sep 17 00:00:00 2001 From: Daniel Golding Date: Sat, 7 Jun 2025 20:32:06 +0200 Subject: [PATCH 107/116] gh-134876: Add fallback for when process_vm_readv fails with ENOSYS (#134878) --- Misc/ACKS | 1 + ...-05-31-10-26-46.gh-issue-134876.8mBGJI.rst | 2 + Python/remote_debug.h | 70 +++++++++++++++++++ Python/remote_debugging.c | 39 +++++++++++ 4 files changed, 112 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-31-10-26-46.gh-issue-134876.8mBGJI.rst diff --git a/Misc/ACKS b/Misc/ACKS index 2435943f1bb2bd..0be31560387ccb 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -658,6 +658,7 @@ Michael Goderbauer Karan Goel Jeroen Van Goey Christoph Gohlke +Daniel Golding Tim Golden Yonatan Goldschmidt Mark Gollahon diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-31-10-26-46.gh-issue-134876.8mBGJI.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-31-10-26-46.gh-issue-134876.8mBGJI.rst new file mode 100644 index 00000000000000..1da76561469a41 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-31-10-26-46.gh-issue-134876.8mBGJI.rst @@ -0,0 +1,2 @@ +Add support to :pep:`768` remote debugging for Linux kernels which don't +have CONFIG_CROSS_MEMORY_ATTACH configured. diff --git a/Python/remote_debug.h b/Python/remote_debug.h index 6cbf1c8deaaed9..0a817bdbd488e0 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -116,6 +116,8 @@ typedef struct { mach_port_t task; #elif defined(MS_WINDOWS) HANDLE hProcess; +#elif defined(__linux__) + int memfd; #endif page_cache_entry_t pages[MAX_PAGES]; Py_ssize_t page_size; @@ -162,6 +164,8 @@ _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { _set_debug_exception_cause(PyExc_RuntimeError, "Failed to initialize Windows process handle"); return -1; } +#elif defined(__linux__) + handle->memfd = -1; #endif handle->page_size = get_page_size(); for (int i = 0; i < MAX_PAGES; i++) { @@ -179,6 +183,11 @@ _Py_RemoteDebug_CleanupProcHandle(proc_handle_t *handle) { CloseHandle(handle->hProcess); handle->hProcess = NULL; } +#elif defined(__linux__) + if (handle->memfd != -1) { + close(handle->memfd); + handle->memfd = -1; + } #endif handle->pid = 0; _Py_RemoteDebug_FreePageCache(handle); @@ -907,6 +916,61 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle) return address; } +#if defined(__linux__) && HAVE_PROCESS_VM_READV + +static int +open_proc_mem_fd(proc_handle_t *handle) +{ + char mem_file_path[64]; + sprintf(mem_file_path, "/proc/%d/mem", handle->pid); + + handle->memfd = open(mem_file_path, O_RDWR); + if (handle->memfd == -1) { + PyErr_SetFromErrno(PyExc_OSError); + _set_debug_exception_cause(PyExc_OSError, + "failed to open file %s: %s", mem_file_path, strerror(errno)); + return -1; + } + return 0; +} + +// Why is pwritev not guarded? Except on Android API level 23 (no longer +// supported), HAVE_PROCESS_VM_READV is sufficient. +static int +read_remote_memory_fallback(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst) +{ + if (handle->memfd == -1) { + if (open_proc_mem_fd(handle) < 0) { + return -1; + } + } + + struct iovec local[1]; + Py_ssize_t result = 0; + Py_ssize_t read_bytes = 0; + + do { + local[0].iov_base = (char*)dst + result; + local[0].iov_len = len - result; + off_t offset = remote_address + result; + + read_bytes = preadv(handle->memfd, local, 1, offset); + if (read_bytes < 0) { + PyErr_SetFromErrno(PyExc_OSError); + _set_debug_exception_cause(PyExc_OSError, + "preadv failed for PID %d at address 0x%lx " + "(size %zu, partial read %zd bytes): %s", + handle->pid, remote_address + result, len - result, result, strerror(errno)); + return -1; + } + + result += read_bytes; + } while ((size_t)read_bytes != local[0].iov_len); + return 0; +} + +#endif // __linux__ + // Platform-independent memory read function static int _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst) @@ -928,6 +992,9 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address } while (result < len); return 0; #elif defined(__linux__) && HAVE_PROCESS_VM_READV + if (handle->memfd != -1) { + return read_remote_memory_fallback(handle, remote_address, len, dst); + } struct iovec local[1]; struct iovec remote[1]; Py_ssize_t result = 0; @@ -941,6 +1008,9 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address read_bytes = process_vm_readv(handle->pid, local, 1, remote, 1, 0); if (read_bytes < 0) { + if (errno == ENOSYS) { + return read_remote_memory_fallback(handle, remote_address, len, dst); + } PyErr_SetFromErrno(PyExc_OSError); _set_debug_exception_cause(PyExc_OSError, "process_vm_readv failed for PID %d at address 0x%lx " diff --git a/Python/remote_debugging.c b/Python/remote_debugging.c index dd55b7812d4dee..7aee87ef05a407 100644 --- a/Python/remote_debugging.c +++ b/Python/remote_debugging.c @@ -24,6 +24,39 @@ read_memory(proc_handle_t *handle, uint64_t remote_address, size_t len, void* ds return _Py_RemoteDebug_ReadRemoteMemory(handle, remote_address, len, dst); } +// Why is pwritev not guarded? Except on Android API level 23 (no longer +// supported), HAVE_PROCESS_VM_READV is sufficient. +#if defined(__linux__) && HAVE_PROCESS_VM_READV +static int +write_memory_fallback(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) +{ + if (handle->memfd == -1) { + if (open_proc_mem_fd(handle) < 0) { + return -1; + } + } + + struct iovec local[1]; + Py_ssize_t result = 0; + Py_ssize_t written = 0; + + do { + local[0].iov_base = (char*)src + result; + local[0].iov_len = len - result; + off_t offset = remote_address + result; + + written = pwritev(handle->memfd, local, 1, offset); + if (written < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + result += written; + } while ((size_t)written != local[0].iov_len); + return 0; +} +#endif // __linux__ + static int write_memory(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) { @@ -39,6 +72,9 @@ write_memory(proc_handle_t *handle, uintptr_t remote_address, size_t len, const } while (result < len); return 0; #elif defined(__linux__) && HAVE_PROCESS_VM_READV + if (handle->memfd != -1) { + return write_memory_fallback(handle, remote_address, len, src); + } struct iovec local[1]; struct iovec remote[1]; Py_ssize_t result = 0; @@ -52,6 +88,9 @@ write_memory(proc_handle_t *handle, uintptr_t remote_address, size_t len, const written = process_vm_writev(handle->pid, local, 1, remote, 1, 0); if (written < 0) { + if (errno == ENOSYS) { + return write_memory_fallback(handle, remote_address, len, src); + } PyErr_SetFromErrno(PyExc_OSError); return -1; } From 8fdbbf8b18f1405abe677d0e04874c1c86ccdb4a Mon Sep 17 00:00:00 2001 From: Amit Lavon Date: Sat, 7 Jun 2025 14:08:44 -0700 Subject: [PATCH 108/116] GH-131798: Type-propagate string/list/tuple slices (GH-134671) --- Lib/test/test_capi/test_opt.py | 30 +++++++++++++++++-- ...-05-25-19-32-15.gh-issue-131798.f5h8aI.rst | 1 + Python/optimizer_bytecodes.c | 14 +++++++++ Python/optimizer_cases.c.h | 13 +++++++- 4 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-25-19-32-15.gh-issue-131798.f5h8aI.rst diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index ee8d261685d463..8a3819dabe44ce 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1666,13 +1666,11 @@ def testfunc(n): self.assertIn("_CONTAINS_OP_DICT", uops) self.assertNotIn("_TO_BOOL_BOOL", uops) - def test_remove_guard_for_known_type_str(self): def f(n): for i in range(n): false = i == TIER2_THRESHOLD empty = "X"[:false] - empty += "" # Make JIT realize this is a string. if empty: return 1 return 0 @@ -2249,6 +2247,34 @@ def f(n): self.assertNotIn("_LOAD_ATTR_METHOD_NO_DICT", uops) self.assertNotIn("_LOAD_ATTR_METHOD_LAZY_DICT", uops) + def test_remove_guard_for_slice_list(self): + def f(n): + for i in range(n): + false = i == TIER2_THRESHOLD + sliced = [1, 2, 3][:false] + if sliced: + return 1 + return 0 + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, 0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_TO_BOOL_LIST", uops) + self.assertNotIn("_GUARD_TOS_LIST", uops) + + def test_remove_guard_for_slice_tuple(self): + def f(n): + for i in range(n): + false = i == TIER2_THRESHOLD + a, b = (1, 2, 3)[: false + 2] + + _, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_UNPACK_SEQUENCE_TWO_TUPLE", uops) + self.assertNotIn("_GUARD_TOS_TUPLE", uops) + def global_identity(x): return x diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-25-19-32-15.gh-issue-131798.f5h8aI.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-25-19-32-15.gh-issue-131798.f5h8aI.rst new file mode 100644 index 00000000000000..6ecbfb8d9cf7df --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-25-19-32-15.gh-issue-131798.f5h8aI.rst @@ -0,0 +1 @@ +Make the JIT optimizer understand that slicing a string/list/tuple returns the same type. diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 12efaacd8f0dfc..fbf4dfd3db629c 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1237,6 +1237,20 @@ dummy_func(void) { sym_set_const(callable, list_append); } + op(_BINARY_SLICE, (container, start, stop -- res)) { + // Slicing a string/list/tuple always returns the same type. + PyTypeObject *type = sym_get_type(container); + if (type == &PyUnicode_Type || + type == &PyList_Type || + type == &PyTuple_Type) + { + res = sym_new_type(ctx, type); + } + else { + res = sym_new_not_null(ctx); + } + } + // END BYTECODES // } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 1a2d49973ee916..b42f47c75eaf50 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -568,8 +568,19 @@ } case _BINARY_SLICE: { + JitOptSymbol *container; JitOptSymbol *res; - res = sym_new_not_null(ctx); + container = stack_pointer[-3]; + PyTypeObject *type = sym_get_type(container); + if (type == &PyUnicode_Type || + type == &PyList_Type || + type == &PyTuple_Type) + { + res = sym_new_type(ctx, type); + } + else { + res = sym_new_not_null(ctx); + } stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); From d610f11d21241d353b25843f66e51098a5c0ddad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 8 Jun 2025 09:10:52 +0200 Subject: [PATCH 109/116] gh-133579: correctly report C curses errors in `_curses_panel` (#134629) This is a follow-up to ee36db550076e5a9185444ffbc53eaf8157ef04c. --- Modules/_curses_panel.c | 369 ++++++++++++++++++++----------- Modules/clinic/_curses_panel.c.h | 159 ++++--------- 2 files changed, 278 insertions(+), 250 deletions(-) diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index eecf7a1c8a1e56..d7acfc6a06a974 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -17,6 +17,7 @@ static const char PyCursesVersion[] = "2.1"; #include "Python.h" +#define CURSES_PANEL_MODULE #include "py_curses.h" #if defined(HAVE_NCURSESW_PANEL_H) @@ -28,10 +29,12 @@ static const char PyCursesVersion[] = "2.1"; #endif typedef struct { - PyObject *PyCursesError; + PyObject *error; PyTypeObject *PyCursesPanel_Type; } _curses_panel_state; +typedef struct PyCursesPanelObject PyCursesPanelObject; + static inline _curses_panel_state * get_curses_panel_state(PyObject *module) { @@ -40,11 +43,30 @@ get_curses_panel_state(PyObject *module) return (_curses_panel_state *)state; } +static inline _curses_panel_state * +get_curses_panel_state_by_panel(PyCursesPanelObject *panel) +{ + /* + * Note: 'state' may be NULL if Py_TYPE(panel) is not a heap + * type associated with this module, but the compiler would + * have likely already complained with an "invalid pointer + * type" at compile-time. + * + * To make it more robust, all functions recovering a module's + * state from an object should expect to return NULL with an + * exception set (in contrast to functions recovering a module's + * state from a module itself). + */ + void *state = PyType_GetModuleState(Py_TYPE(panel)); + assert(state != NULL); + return (_curses_panel_state *)state; +} + static int _curses_panel_clear(PyObject *mod) { _curses_panel_state *state = get_curses_panel_state(mod); - Py_CLEAR(state->PyCursesError); + Py_CLEAR(state->error); Py_CLEAR(state->PyCursesPanel_Type); return 0; } @@ -54,7 +76,7 @@ _curses_panel_traverse(PyObject *mod, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(mod)); _curses_panel_state *state = get_curses_panel_state(mod); - Py_VISIT(state->PyCursesError); + Py_VISIT(state->error); Py_VISIT(state->PyCursesPanel_Type); return 0; } @@ -65,28 +87,149 @@ _curses_panel_free(void *mod) (void)_curses_panel_clear((PyObject *)mod); } +/* Utility Error Procedures + * + * The naming and implementations are identical to those in _cursesmodule.c. + * Functions that are not yet needed (for instance, reporting an ERR value + * from a module-wide function, namely curses_panel_set_error()) are + * omitted and should only be added if needed. + */ + +static void +_curses_panel_format_error(_curses_panel_state *state, + const char *curses_funcname, + const char *python_funcname, + const char *return_value, + const char *default_message) +{ + assert(!PyErr_Occurred()); + if (python_funcname == NULL && curses_funcname == NULL) { + PyErr_SetString(state->error, default_message); + } + else if (python_funcname == NULL) { + (void)PyErr_Format(state->error, CURSES_ERROR_FORMAT, + curses_funcname, return_value); + } + else { + assert(python_funcname != NULL); + (void)PyErr_Format(state->error, CURSES_ERROR_VERBOSE_FORMAT, + curses_funcname, python_funcname, return_value); + } +} + +/* + * Format a curses error for a function that returned ERR. + * + * Specify a non-NULL 'python_funcname' when the latter differs from + * 'curses_funcname'. If both names are NULL, uses the 'catchall_ERR' + * message instead. + */ +static void +_curses_panel_set_error(_curses_panel_state *state, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_panel_format_error(state, curses_funcname, python_funcname, + "ERR", catchall_ERR); +} + +/* + * Format a curses error for a function that returned NULL. + * + * Specify a non-NULL 'python_funcname' when the latter differs from + * 'curses_funcname'. If both names are NULL, uses the 'catchall_NULL' + * message instead. + */ +static void +_curses_panel_set_null_error(_curses_panel_state *state, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_panel_format_error(state, curses_funcname, python_funcname, + "NULL", catchall_NULL); +} + +/* Same as _curses_panel_set_null_error() for a module object. */ +static void +curses_panel_set_null_error(PyObject *module, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_panel_state *state = get_curses_panel_state(module); + _curses_panel_set_null_error(state, curses_funcname, python_funcname); +} + +/* Same as _curses_panel_set_error() for a panel object. */ +static void +curses_panel_panel_set_error(PyCursesPanelObject *panel, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_panel_state *state = get_curses_panel_state_by_panel(panel); + _curses_panel_set_error(state, curses_funcname, python_funcname); +} + +/* Same as _curses_panel_set_null_error() for a panel object. */ +static void +curses_panel_panel_set_null_error(PyCursesPanelObject *panel, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_panel_state *state = get_curses_panel_state_by_panel(panel); + _curses_panel_set_null_error(state, curses_funcname, python_funcname); +} + +/* + * Indicate that a panel object couldn't be found. + * + * Use it for the following constructions: + * + * PROC caller_funcname: + * pan = called_funcname() + * find_po(panel) + * + * PROC caller_funcname: + * find_po(self->pan) +*/ +static void +curses_panel_notfound_error(const char *called_funcname, + const char *caller_funcname) +{ + assert(!(called_funcname == NULL && caller_funcname == NULL)); + if (caller_funcname == NULL) { + (void)PyErr_Format(PyExc_RuntimeError, + "%s(): cannot find panel object", + called_funcname); + } + else { + (void)PyErr_Format(PyExc_RuntimeError, + "%s() (called by %s()): cannot find panel object", + called_funcname, caller_funcname); + } +} + /* Utility Functions */ /* - * Check the return code from a curses function and return None - * or raise an exception as appropriate. + * Check the return code from a curses function, returning None + * on success and setting an exception on error. */ +/* + * Return None if 'code' is different from ERR (implementation-defined). + * Otherwise, set an exception using curses_panel_panel_set_error() and + * the remaining arguments, and return NULL. + */ static PyObject * -PyCursesCheckERR(_curses_panel_state *state, int code, const char *fname) +curses_panel_panel_check_err(PyCursesPanelObject *panel, int code, + const char *curses_funcname, + const char *python_funcname) { if (code != ERR) { Py_RETURN_NONE; } - else { - if (fname == NULL) { - PyErr_SetString(state->PyCursesError, catchall_ERR); - } - else { - PyErr_Format(state->PyCursesError, "%s() returned ERR", fname); - } - return NULL; - } + curses_panel_panel_set_error(panel, curses_funcname, python_funcname); + return NULL; } /***************************************************************************** @@ -95,7 +238,7 @@ PyCursesCheckERR(_curses_panel_state *state, int code, const char *fname) /* Definition of the panel object and panel type */ -typedef struct { +typedef struct PyCursesPanelObject { PyObject_HEAD PANEL *pan; PyCursesWindowObject *wo; /* for reference counts */ @@ -144,8 +287,11 @@ insert_lop(PyCursesPanelObject *po) return 0; } -/* Remove the panel object from lop */ -static void +/* Remove the panel object from lop. + * + * Return -1 on error but do NOT set an exception; otherwise return 0. + */ +static int remove_lop(PyCursesPanelObject *po) { list_of_panels *temp, *n; @@ -154,25 +300,23 @@ remove_lop(PyCursesPanelObject *po) if (temp->po == po) { lop = temp->next; PyMem_Free(temp); - return; + return 0; } while (temp->next == NULL || temp->next->po != po) { if (temp->next == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "remove_lop: can't find Panel Object"); - return; + return -1; } temp = temp->next; } n = temp->next->next; PyMem_Free(temp->next); temp->next = n; - return; + return 0; } /* Return the panel object that corresponds to pan */ static PyCursesPanelObject * -find_po(PANEL *pan) +find_po_impl(PANEL *pan) { list_of_panels *temp; for (temp = lop; temp->po->pan != pan; temp = temp->next) @@ -180,6 +324,17 @@ find_po(PANEL *pan) return temp->po; } +/* Same as find_po_impl() but with caller context information. */ +static PyCursesPanelObject * +find_po(PANEL *pan, const char *called_funcname, const char *caller_funcname) +{ + PyCursesPanelObject *res = find_po_impl(pan); + if (res == NULL) { + curses_panel_notfound_error(called_funcname, caller_funcname); + } + return res; +} + /*[clinic input] module _curses_panel class _curses_panel.panel "PyCursesPanelObject *" "&PyCursesPanel_Type" @@ -193,67 +348,59 @@ class _curses_panel.panel "PyCursesPanelObject *" "&PyCursesPanel_Type" /*[clinic input] _curses_panel.panel.bottom - cls: defining_class - Push the panel to the bottom of the stack. [clinic start generated code]*/ static PyObject * -_curses_panel_panel_bottom_impl(PyCursesPanelObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=8ec7fbbc08554021 input=6b7d2c0578b5a1c4]*/ +_curses_panel_panel_bottom_impl(PyCursesPanelObject *self) +/*[clinic end generated code: output=7aa7d14d7e1d1ce6 input=b6c920c071b61e2e]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, bottom_panel(self->pan), "bottom"); + int rtn = bottom_panel(self->pan); + return curses_panel_panel_check_err(self, rtn, "bottom_panel", "bottom"); } /*[clinic input] _curses_panel.panel.hide - cls: defining_class - Hide the panel. This does not delete the object, it just makes the window on screen invisible. [clinic start generated code]*/ static PyObject * -_curses_panel_panel_hide_impl(PyCursesPanelObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=cc6ab7203cdc1450 input=1bfc741f473e6055]*/ +_curses_panel_panel_hide_impl(PyCursesPanelObject *self) +/*[clinic end generated code: output=a7bbbd523e1eab49 input=f6ab884e99386118]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, hide_panel(self->pan), "hide"); + int rtn = hide_panel(self->pan); + return curses_panel_panel_check_err(self, rtn, "hide_panel", "hide"); } /*[clinic input] _curses_panel.panel.show - cls: defining_class - Display the panel (which might have been hidden). [clinic start generated code]*/ static PyObject * -_curses_panel_panel_show_impl(PyCursesPanelObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=dc3421de375f0409 input=8122e80151cb4379]*/ +_curses_panel_panel_show_impl(PyCursesPanelObject *self) +/*[clinic end generated code: output=6b4553ab45c97769 input=57b167bbefaa3755]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, show_panel(self->pan), "show"); + int rtn = show_panel(self->pan); + return curses_panel_panel_check_err(self, rtn, "show_panel", "show"); } /*[clinic input] _curses_panel.panel.top - cls: defining_class - Push panel to the top of the stack. [clinic start generated code]*/ static PyObject * -_curses_panel_panel_top_impl(PyCursesPanelObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=10a072e511e873f7 input=1f372d597dda3379]*/ +_curses_panel_panel_top_impl(PyCursesPanelObject *self) +/*[clinic end generated code: output=0f5f2f8cdd2d1777 input=be33975ec3ca0e9a]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, top_panel(self->pan), "top"); + int rtn = top_panel(self->pan); + return curses_panel_panel_check_err(self, rtn, "top_panel", "top"); } /* Allocation and deallocation of Panel Objects */ @@ -287,13 +434,22 @@ PyCursesPanel_Dealloc(PyObject *self) tp = (PyObject *) Py_TYPE(po); obj = (PyObject *) panel_userptr(po->pan); if (obj) { - (void)set_panel_userptr(po->pan, NULL); Py_DECREF(obj); + if (set_panel_userptr(po->pan, NULL) == ERR) { + curses_panel_panel_set_error(po, "set_panel_userptr", "__del__"); + PyErr_FormatUnraisable("Exception ignored in PyCursesPanel_Dealloc()"); + } + } + if (del_panel(po->pan) == ERR && !PyErr_Occurred()) { + curses_panel_panel_set_error(po, "del_panel", "__del__"); + PyErr_FormatUnraisable("Exception ignored in PyCursesPanel_Dealloc()"); } - (void)del_panel(po->pan); if (po->wo != NULL) { Py_DECREF(po->wo); - remove_lop(po); + if (remove_lop(po) < 0) { + PyErr_SetString(PyExc_RuntimeError, "__del__: no panel object to delete"); + PyErr_FormatUnraisable("Exception ignored in PyCursesPanel_Dealloc()"); + } } PyObject_Free(po); Py_DECREF(tp); @@ -315,18 +471,11 @@ _curses_panel_panel_above_impl(PyCursesPanelObject *self) PyCursesPanelObject *po; pan = panel_above(self->pan); - - if (pan == NULL) { /* valid output, it means the calling panel - is on top of the stack */ + if (pan == NULL) { /* valid output: it means no panel exists yet */ Py_RETURN_NONE; } - po = find_po(pan); - if (po == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "panel_above: can't find Panel Object"); - return NULL; - } - return Py_NewRef(po); + po = find_po(pan, "panel_above", "above"); + return Py_XNewRef(po); } /* panel_below(NULL) returns the top panel in the stack. To get @@ -345,18 +494,11 @@ _curses_panel_panel_below_impl(PyCursesPanelObject *self) PyCursesPanelObject *po; pan = panel_below(self->pan); - - if (pan == NULL) { /* valid output, it means the calling panel - is on the bottom of the stack */ + if (pan == NULL) { /* valid output: it means no panel exists yet */ Py_RETURN_NONE; } - po = find_po(pan); - if (po == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "panel_below: can't find Panel Object"); - return NULL; - } - return Py_NewRef(po); + po = find_po(pan, "panel_below", "below"); + return Py_XNewRef(po); } /*[clinic input] @@ -378,7 +520,6 @@ _curses_panel_panel_hidden_impl(PyCursesPanelObject *self) /*[clinic input] _curses_panel.panel.move - cls: defining_class y: int x: int / @@ -387,12 +528,11 @@ Move the panel to the screen coordinates (y, x). [clinic start generated code]*/ static PyObject * -_curses_panel_panel_move_impl(PyCursesPanelObject *self, PyTypeObject *cls, - int y, int x) -/*[clinic end generated code: output=ce546c93e56867da input=60a0e7912ff99849]*/ +_curses_panel_panel_move_impl(PyCursesPanelObject *self, int y, int x) +/*[clinic end generated code: output=d867535a89777415 input=e0b36b78acc03fba]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, move_panel(self->pan, y, x), "move_panel"); + int rtn = move_panel(self->pan, y, x); + return curses_panel_panel_check_err(self, rtn, "move_panel", "move"); } /*[clinic input] @@ -411,7 +551,6 @@ _curses_panel_panel_window_impl(PyCursesPanelObject *self) /*[clinic input] _curses_panel.panel.replace - cls: defining_class win: object(type="PyCursesWindowObject *", subclass_of="&PyCursesWindow_Type") / @@ -420,22 +559,17 @@ Change the window associated with the panel to the window win. static PyObject * _curses_panel_panel_replace_impl(PyCursesPanelObject *self, - PyTypeObject *cls, PyCursesWindowObject *win) -/*[clinic end generated code: output=c71f95c212d58ae7 input=dbec7180ece41ff5]*/ +/*[clinic end generated code: output=2253a95f7b287255 input=4b1c4283987d9dfa]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - - PyCursesPanelObject *po = find_po(self->pan); + PyCursesPanelObject *po = find_po(self->pan, "replace", NULL); if (po == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "replace_panel: can't find Panel Object"); return NULL; } int rtn = replace_panel(self->pan, win->win); if (rtn == ERR) { - PyErr_SetString(state->PyCursesError, "replace_panel() returned ERR"); + curses_panel_panel_set_error(self, "replace_panel", "replace"); return NULL; } Py_SETREF(po->wo, (PyCursesWindowObject*)Py_NewRef(win)); @@ -445,7 +579,6 @@ _curses_panel_panel_replace_impl(PyCursesPanelObject *self, /*[clinic input] _curses_panel.panel.set_userptr - cls: defining_class obj: object / @@ -454,8 +587,8 @@ Set the panel's user pointer to obj. static PyObject * _curses_panel_panel_set_userptr_impl(PyCursesPanelObject *self, - PyTypeObject *cls, PyObject *obj) -/*[clinic end generated code: output=db74f3db07b28080 input=e3fee2ff7b1b8e48]*/ + PyObject *obj) +/*[clinic end generated code: output=7fa1fd23f69db71e input=d2c6a9dbefabbf39]*/ { PyCursesInitialised; Py_INCREF(obj); @@ -464,34 +597,27 @@ _curses_panel_panel_set_userptr_impl(PyCursesPanelObject *self, if (rc == ERR) { /* In case of an ncurses error, decref the new object again */ Py_DECREF(obj); + curses_panel_panel_set_error(self, "set_panel_userptr", "set_userptr"); + return NULL; } - else { - Py_XDECREF(oldobj); - } - - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, rc, "set_panel_userptr"); + Py_XDECREF(oldobj); + Py_RETURN_NONE; } /*[clinic input] _curses_panel.panel.userptr - cls: defining_class - Return the user pointer for the panel. [clinic start generated code]*/ static PyObject * -_curses_panel_panel_userptr_impl(PyCursesPanelObject *self, - PyTypeObject *cls) -/*[clinic end generated code: output=eea6e6f39ffc0179 input=f22ca4f115e30a80]*/ +_curses_panel_panel_userptr_impl(PyCursesPanelObject *self) +/*[clinic end generated code: output=e849c307b5dc9237 input=f78b7a47aef0fd50]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - PyCursesInitialised; PyObject *obj = (PyObject *) panel_userptr(self->pan); if (obj == NULL) { - PyErr_SetString(state->PyCursesError, "no userptr set"); + curses_panel_panel_set_null_error(self, "panel_userptr", "userptr"); return NULL; } @@ -552,18 +678,11 @@ _curses_panel_bottom_panel_impl(PyObject *module) PyCursesInitialised; pan = panel_above(NULL); - - if (pan == NULL) { /* valid output, it means - there's no panel at all */ + if (pan == NULL) { /* valid output: it means no panel exists yet */ Py_RETURN_NONE; } - po = find_po(pan); - if (po == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "panel_above: can't find Panel Object"); - return NULL; - } - return Py_NewRef(po); + po = find_po(pan, "panel_above", "bottom_panel"); + return Py_XNewRef(po); } /*[clinic input] @@ -579,14 +698,13 @@ static PyObject * _curses_panel_new_panel_impl(PyObject *module, PyCursesWindowObject *win) /*[clinic end generated code: output=45e948e0176a9bd2 input=74d4754e0ebe4800]*/ { - _curses_panel_state *state = get_curses_panel_state(module); - PANEL *pan = new_panel(win->win); if (pan == NULL) { - PyErr_SetString(state->PyCursesError, catchall_NULL); + curses_panel_set_null_error(module, "new_panel", NULL); return NULL; } - return (PyObject *)PyCursesPanel_New(state, pan, win); + _curses_panel_state *state = get_curses_panel_state(module); + return PyCursesPanel_New(state, pan, win); } @@ -610,18 +728,11 @@ _curses_panel_top_panel_impl(PyObject *module) PyCursesInitialised; pan = panel_below(NULL); - - if (pan == NULL) { /* valid output, it means - there's no panel at all */ + if (pan == NULL) { /* valid output: it means no panel exists yet */ Py_RETURN_NONE; } - po = find_po(pan); - if (po == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "panel_below: can't find Panel Object"); - return NULL; - } - return Py_NewRef(po); + po = find_po(pan, "panel_below", "top_panel"); + return Py_XNewRef(po); } /*[clinic input] @@ -673,10 +784,10 @@ _curses_panel_exec(PyObject *mod) } /* For exception _curses_panel.error */ - state->PyCursesError = PyErr_NewException( + state->error = PyErr_NewException( "_curses_panel.error", NULL, NULL); - if (PyModule_AddObjectRef(mod, "error", state->PyCursesError) < 0) { + if (PyModule_AddObjectRef(mod, "error", state->error) < 0) { return -1; } diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index 6f4966825ec4bf..75cf067c8aa822 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -2,10 +2,7 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_runtime.h" // _Py_SINGLETON() -#endif -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_curses_panel_panel_bottom__doc__, "bottom($self, /)\n" @@ -14,19 +11,15 @@ PyDoc_STRVAR(_curses_panel_panel_bottom__doc__, "Push the panel to the bottom of the stack."); #define _CURSES_PANEL_PANEL_BOTTOM_METHODDEF \ - {"bottom", _PyCFunction_CAST(_curses_panel_panel_bottom), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_bottom__doc__}, + {"bottom", (PyCFunction)_curses_panel_panel_bottom, METH_NOARGS, _curses_panel_panel_bottom__doc__}, static PyObject * -_curses_panel_panel_bottom_impl(PyCursesPanelObject *self, PyTypeObject *cls); +_curses_panel_panel_bottom_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_bottom(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_bottom(PyObject *self, PyObject *Py_UNUSED(ignored)) { - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "bottom() takes no arguments"); - return NULL; - } - return _curses_panel_panel_bottom_impl((PyCursesPanelObject *)self, cls); + return _curses_panel_panel_bottom_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_panel_hide__doc__, @@ -38,19 +31,15 @@ PyDoc_STRVAR(_curses_panel_panel_hide__doc__, "This does not delete the object, it just makes the window on screen invisible."); #define _CURSES_PANEL_PANEL_HIDE_METHODDEF \ - {"hide", _PyCFunction_CAST(_curses_panel_panel_hide), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_hide__doc__}, + {"hide", (PyCFunction)_curses_panel_panel_hide, METH_NOARGS, _curses_panel_panel_hide__doc__}, static PyObject * -_curses_panel_panel_hide_impl(PyCursesPanelObject *self, PyTypeObject *cls); +_curses_panel_panel_hide_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_hide(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_hide(PyObject *self, PyObject *Py_UNUSED(ignored)) { - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "hide() takes no arguments"); - return NULL; - } - return _curses_panel_panel_hide_impl((PyCursesPanelObject *)self, cls); + return _curses_panel_panel_hide_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_panel_show__doc__, @@ -60,19 +49,15 @@ PyDoc_STRVAR(_curses_panel_panel_show__doc__, "Display the panel (which might have been hidden)."); #define _CURSES_PANEL_PANEL_SHOW_METHODDEF \ - {"show", _PyCFunction_CAST(_curses_panel_panel_show), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_show__doc__}, + {"show", (PyCFunction)_curses_panel_panel_show, METH_NOARGS, _curses_panel_panel_show__doc__}, static PyObject * -_curses_panel_panel_show_impl(PyCursesPanelObject *self, PyTypeObject *cls); +_curses_panel_panel_show_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_show(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_show(PyObject *self, PyObject *Py_UNUSED(ignored)) { - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "show() takes no arguments"); - return NULL; - } - return _curses_panel_panel_show_impl((PyCursesPanelObject *)self, cls); + return _curses_panel_panel_show_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_panel_top__doc__, @@ -82,19 +67,15 @@ PyDoc_STRVAR(_curses_panel_panel_top__doc__, "Push panel to the top of the stack."); #define _CURSES_PANEL_PANEL_TOP_METHODDEF \ - {"top", _PyCFunction_CAST(_curses_panel_panel_top), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_top__doc__}, + {"top", (PyCFunction)_curses_panel_panel_top, METH_NOARGS, _curses_panel_panel_top__doc__}, static PyObject * -_curses_panel_panel_top_impl(PyCursesPanelObject *self, PyTypeObject *cls); +_curses_panel_panel_top_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_top(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_top(PyObject *self, PyObject *Py_UNUSED(ignored)) { - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "top() takes no arguments"); - return NULL; - } - return _curses_panel_panel_top_impl((PyCursesPanelObject *)self, cls); + return _curses_panel_panel_top_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_panel_above__doc__, @@ -158,36 +139,19 @@ PyDoc_STRVAR(_curses_panel_panel_move__doc__, "Move the panel to the screen coordinates (y, x)."); #define _CURSES_PANEL_PANEL_MOVE_METHODDEF \ - {"move", _PyCFunction_CAST(_curses_panel_panel_move), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_move__doc__}, + {"move", _PyCFunction_CAST(_curses_panel_panel_move), METH_FASTCALL, _curses_panel_panel_move__doc__}, static PyObject * -_curses_panel_panel_move_impl(PyCursesPanelObject *self, PyTypeObject *cls, - int y, int x); +_curses_panel_panel_move_impl(PyCursesPanelObject *self, int y, int x); static PyObject * -_curses_panel_panel_move(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_move(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) - #else - # define KWTUPLE NULL - #endif - - static const char * const _keywords[] = {"", "", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "move", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; int y; int x; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { + if (!_PyArg_CheckPositional("move", nargs, 2, 2)) { goto exit; } y = PyLong_AsInt(args[0]); @@ -198,7 +162,7 @@ _curses_panel_panel_move(PyObject *self, PyTypeObject *cls, PyObject *const *arg if (x == -1 && PyErr_Occurred()) { goto exit; } - return_value = _curses_panel_panel_move_impl((PyCursesPanelObject *)self, cls, y, x); + return_value = _curses_panel_panel_move_impl((PyCursesPanelObject *)self, y, x); exit: return return_value; @@ -229,44 +193,24 @@ PyDoc_STRVAR(_curses_panel_panel_replace__doc__, "Change the window associated with the panel to the window win."); #define _CURSES_PANEL_PANEL_REPLACE_METHODDEF \ - {"replace", _PyCFunction_CAST(_curses_panel_panel_replace), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_replace__doc__}, + {"replace", (PyCFunction)_curses_panel_panel_replace, METH_O, _curses_panel_panel_replace__doc__}, static PyObject * _curses_panel_panel_replace_impl(PyCursesPanelObject *self, - PyTypeObject *cls, PyCursesWindowObject *win); static PyObject * -_curses_panel_panel_replace(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_replace(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) - #else - # define KWTUPLE NULL - #endif - - static const char * const _keywords[] = {"", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "replace", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; PyCursesWindowObject *win; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { - goto exit; - } - if (!PyObject_TypeCheck(args[0], &PyCursesWindow_Type)) { - _PyArg_BadArgument("replace", "argument 1", (&PyCursesWindow_Type)->tp_name, args[0]); + if (!PyObject_TypeCheck(arg, &PyCursesWindow_Type)) { + _PyArg_BadArgument("replace", "argument", (&PyCursesWindow_Type)->tp_name, arg); goto exit; } - win = (PyCursesWindowObject *)args[0]; - return_value = _curses_panel_panel_replace_impl((PyCursesPanelObject *)self, cls, win); + win = (PyCursesWindowObject *)arg; + return_value = _curses_panel_panel_replace_impl((PyCursesPanelObject *)self, win); exit: return return_value; @@ -279,41 +223,19 @@ PyDoc_STRVAR(_curses_panel_panel_set_userptr__doc__, "Set the panel\'s user pointer to obj."); #define _CURSES_PANEL_PANEL_SET_USERPTR_METHODDEF \ - {"set_userptr", _PyCFunction_CAST(_curses_panel_panel_set_userptr), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_set_userptr__doc__}, + {"set_userptr", (PyCFunction)_curses_panel_panel_set_userptr, METH_O, _curses_panel_panel_set_userptr__doc__}, static PyObject * _curses_panel_panel_set_userptr_impl(PyCursesPanelObject *self, - PyTypeObject *cls, PyObject *obj); + PyObject *obj); static PyObject * -_curses_panel_panel_set_userptr(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_set_userptr(PyObject *self, PyObject *obj) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) - #else - # define KWTUPLE NULL - #endif - - static const char * const _keywords[] = {"", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "set_userptr", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject *obj; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { - goto exit; - } - obj = args[0]; - return_value = _curses_panel_panel_set_userptr_impl((PyCursesPanelObject *)self, cls, obj); -exit: + return_value = _curses_panel_panel_set_userptr_impl((PyCursesPanelObject *)self, obj); + return return_value; } @@ -324,20 +246,15 @@ PyDoc_STRVAR(_curses_panel_panel_userptr__doc__, "Return the user pointer for the panel."); #define _CURSES_PANEL_PANEL_USERPTR_METHODDEF \ - {"userptr", _PyCFunction_CAST(_curses_panel_panel_userptr), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_userptr__doc__}, + {"userptr", (PyCFunction)_curses_panel_panel_userptr, METH_NOARGS, _curses_panel_panel_userptr__doc__}, static PyObject * -_curses_panel_panel_userptr_impl(PyCursesPanelObject *self, - PyTypeObject *cls); +_curses_panel_panel_userptr_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_userptr(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_userptr(PyObject *self, PyObject *Py_UNUSED(ignored)) { - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "userptr() takes no arguments"); - return NULL; - } - return _curses_panel_panel_userptr_impl((PyCursesPanelObject *)self, cls); + return _curses_panel_panel_userptr_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_bottom_panel__doc__, @@ -424,4 +341,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=36853ecb4a979814 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=db2fe491582784aa input=a9049054013a1b77]*/ From bcb6b45cb86a2f9f65b6c41f27c36059ba86a50b Mon Sep 17 00:00:00 2001 From: "Jiucheng(Oliver)" Date: Sun, 8 Jun 2025 03:13:21 -0400 Subject: [PATCH 110/116] gh-134151 Fix `TypeError` in `email.utils.decode_params` when sorting RFC 2231 continuations (#134687) - Fix sorting logic in `email.utils.decode_params` to handle None values. - Update tests for RFC 2231 continuation sorting. --- Lib/email/utils.py | 10 ++++++++-- Lib/test/test_email/test_email.py | 18 ++++++++++++++++++ ...5-05-25-23-23-05.gh-issue-134151.13Wwsb.rst | 2 ++ 3 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-25-23-23-05.gh-issue-134151.13Wwsb.rst diff --git a/Lib/email/utils.py b/Lib/email/utils.py index 7eab74dc0db9df..3de1f0d24a15b0 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -417,8 +417,14 @@ def decode_params(params): for name, continuations in rfc2231_params.items(): value = [] extended = False - # Sort by number - continuations.sort() + # Sort by number, treating None as 0 if there is no 0, + # and ignore it if there is already a 0. + has_zero = any(x[0] == 0 for x in continuations) + if has_zero: + continuations = [x for x in continuations if x[0] is not None] + else: + continuations = [(x[0] or 0, x[1], x[2]) for x in continuations] + continuations.sort(key=lambda x: x[0]) # And now append all values in numerical order, converting # %-encodings for the encoded segments. If any of the # continuation names ends in a *, then the entire string, after diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 7b14305f997e5d..8765d121fd0813 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -389,6 +389,24 @@ def test_bad_param(self): msg = email.message_from_string("Content-Type: blarg; baz; boo\n") self.assertEqual(msg.get_param('baz'), '') + def test_continuation_sorting_part_order(self): + msg = email.message_from_string( + "Content-Disposition: attachment; " + "filename*=\"ignored\"; " + "filename*0*=\"utf-8''foo%20\"; " + "filename*1*=\"bar.txt\"\n" + ) + filename = msg.get_filename() + self.assertEqual(filename, 'foo bar.txt') + + def test_sorting_no_continuations(self): + msg = email.message_from_string( + "Content-Disposition: attachment; " + "filename*=\"bar.txt\"; " + ) + filename = msg.get_filename() + self.assertEqual(filename, 'bar.txt') + def test_missing_filename(self): msg = email.message_from_string("From: foo\n") self.assertEqual(msg.get_filename(), None) diff --git a/Misc/NEWS.d/next/Library/2025-05-25-23-23-05.gh-issue-134151.13Wwsb.rst b/Misc/NEWS.d/next/Library/2025-05-25-23-23-05.gh-issue-134151.13Wwsb.rst new file mode 100644 index 00000000000000..ecdde240b4aadc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-25-23-23-05.gh-issue-134151.13Wwsb.rst @@ -0,0 +1,2 @@ +:mod:`email`: Fix :exc:`TypeError` in :func:`email.utils.decode_params` +when sorting :rfc:`2231` continuations that contain an unnumbered section. From 4372011928b43d369be727ed3eb6d9d4f9610660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 8 Jun 2025 09:43:32 +0200 Subject: [PATCH 111/116] gh-134531: fix `_hashlib` clinic directive post GH-134626 (#135249) --- Modules/_hashopenssl.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 331275076d7937..42821ebe9f6a54 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -260,7 +260,7 @@ static PyModuleDef _hashlibmodule; typedef struct { PyTypeObject *HASH_type; // based on EVP_MD - PyTypeObject *HMACtype; + PyTypeObject *HMAC_type; #ifdef PY_OPENSSL_HAS_SHAKE PyTypeObject *HASHXOF_type; // based on EVP_MD #endif @@ -300,11 +300,11 @@ typedef struct { #include "clinic/_hashopenssl.c.h" /*[clinic input] module _hashlib -class _hashlib.HASH "HASHobject *" "((_hashlibstate *)PyModule_GetState(module))->EVPtype" -class _hashlib.HASHXOF "HASHobject *" "((_hashlibstate *)PyModule_GetState(module))->EVPXOFtype" -class _hashlib.HMAC "HMACobject *" "((_hashlibstate *)PyModule_GetState(module))->HMACtype" +class _hashlib.HASH "HASHobject *" "((_hashlibstate *)PyModule_GetState(module))->HASH_type" +class _hashlib.HASHXOF "HASHobject *" "((_hashlibstate *)PyModule_GetState(module))->HASHXOF_type" +class _hashlib.HMAC "HMACobject *" "((_hashlibstate *)PyModule_GetState(module))->HMAC_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4f6b8873ed13d1ff]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=eb805ce4b90b1b31]*/ /* LCOV_EXCL_START */ @@ -1643,7 +1643,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, } _hashlibstate *state = get_hashlib_state(module); - self = PyObject_New(HMACobject, state->HMACtype); + self = PyObject_New(HMACobject, state->HMAC_type); if (self == NULL) { goto error; } @@ -2204,7 +2204,7 @@ hashlib_traverse(PyObject *m, visitproc visit, void *arg) { _hashlibstate *state = get_hashlib_state(m); Py_VISIT(state->HASH_type); - Py_VISIT(state->HMACtype); + Py_VISIT(state->HMAC_type); #ifdef PY_OPENSSL_HAS_SHAKE Py_VISIT(state->HASHXOF_type); #endif @@ -2218,7 +2218,7 @@ hashlib_clear(PyObject *m) { _hashlibstate *state = get_hashlib_state(m); Py_CLEAR(state->HASH_type); - Py_CLEAR(state->HMACtype); + Py_CLEAR(state->HMAC_type); #ifdef PY_OPENSSL_HAS_SHAKE Py_CLEAR(state->HASHXOF_type); #endif @@ -2296,11 +2296,11 @@ hashlib_init_hmactype(PyObject *module) { _hashlibstate *state = get_hashlib_state(module); - state->HMACtype = (PyTypeObject *)PyType_FromSpec(&HMACtype_spec); - if (state->HMACtype == NULL) { + state->HMAC_type = (PyTypeObject *)PyType_FromSpec(&HMACtype_spec); + if (state->HMAC_type == NULL) { return -1; } - if (PyModule_AddType(module, state->HMACtype) < 0) { + if (PyModule_AddType(module, state->HMAC_type) < 0) { return -1; } return 0; From 1cb716387255a7bdab5b580bcf8ac1b6fa32cc41 Mon Sep 17 00:00:00 2001 From: LamentXU <108666168+LamentXU123@users.noreply.github.com> Date: Sun, 8 Jun 2025 19:46:16 +0800 Subject: [PATCH 112/116] =?UTF-8?q?gh-135244:=20generate=20UUID=20random?= =?UTF-8?q?=20Node=20ID=20with=20a=20CSPRNG=20as=20per=20RFC=209562,=20?= =?UTF-8?q?=C2=A76.10.3=20(#135226)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This aligns with the recommendations of RFC 9562, Section 6.10, paragraph 3 [1]. [1]: https://www.rfc-editor.org/rfc/rfc9562.html#section-6.10-3. --------- Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/uuid.rst | 4 ++++ Lib/uuid.py | 14 ++++++++------ .../2025-06-08-10-22-22.gh-issue-135244.Y2SOTJ.rst | 4 ++++ 3 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-06-08-10-22-22.gh-issue-135244.Y2SOTJ.rst diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index 8cce6b98cbcdb3..747ee3ee0e1951 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -257,6 +257,10 @@ The :mod:`uuid` module defines the following functions: non-specified arguments are substituted for a pseudo-random integer of appropriate size. + By default, *a*, *b* and *c* are generated by a non-cryptographically + secure pseudo-random number generator (CSPRNG). Use :func:`uuid4` when + a UUID needs to be used in a security-sensitive context. + .. versionadded:: 3.14 diff --git a/Lib/uuid.py b/Lib/uuid.py index 06f81a7c338372..313f2fc46cb346 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -656,18 +656,20 @@ def _windll_getnode(): def _random_getnode(): """Get a random node ID.""" - # RFC 4122, $4.1.6 says "For systems with no IEEE address, a randomly or - # pseudo-randomly generated value may be used; see Section 4.5. The - # multicast bit must be set in such addresses, in order that they will - # never conflict with addresses obtained from network cards." + # RFC 9562, §6.10-3 says that + # + # Implementations MAY elect to obtain a 48-bit cryptographic-quality + # random number as per Section 6.9 to use as the Node ID. [...] [and] + # implementations MUST set the least significant bit of the first octet + # of the Node ID to 1. This bit is the unicast or multicast bit, which + # will never be set in IEEE 802 addresses obtained from network cards. # # The "multicast bit" of a MAC address is defined to be "the least # significant bit of the first octet". This works out to be the 41st bit # counting from 1 being the least significant bit, or 1<<40. # # See https://en.wikipedia.org/w/index.php?title=MAC_address&oldid=1128764812#Universal_vs._local_(U/L_bit) - import random - return random.getrandbits(48) | (1 << 40) + return int.from_bytes(os.urandom(6)) | (1 << 40) # _OS_GETTERS, when known, are targeted for a specific OS or platform. diff --git a/Misc/NEWS.d/next/Library/2025-06-08-10-22-22.gh-issue-135244.Y2SOTJ.rst b/Misc/NEWS.d/next/Library/2025-06-08-10-22-22.gh-issue-135244.Y2SOTJ.rst new file mode 100644 index 00000000000000..1f70358e64e2a0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-08-10-22-22.gh-issue-135244.Y2SOTJ.rst @@ -0,0 +1,4 @@ +:mod:`uuid`: when the MAC address cannot be determined, the 48-bit node +ID is now generated with a cryptographically-secure pseudo-random number +generator (CSPRNG) as per :rfc:`RFC 9562, §6.10.3 <9562#section-6.10-3>`. +This affects :func:`~uuid.uuid1` and :func:`~uuid.uuid6`. From 158e5162bfaa8a49178ce2c3f2455c3e03b60157 Mon Sep 17 00:00:00 2001 From: Yongzi Li <204532581+Yzi-Li@users.noreply.github.com> Date: Sun, 8 Jun 2025 20:28:55 +0800 Subject: [PATCH 113/116] gh-134976: document the exception type that can be raised by `s[i]` (#134977) --- Doc/library/stdtypes.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index f0b4b09ff10dce..b75e5ceecf874e 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1018,7 +1018,7 @@ operations have the same priority as the corresponding numeric operations. [3]_ | ``s * n`` or | equivalent to adding *s* to | (2)(7) | | ``n * s`` | itself *n* times | | +--------------------------+--------------------------------+----------+ -| ``s[i]`` | *i*\ th item of *s*, origin 0 | \(3) | +| ``s[i]`` | *i*\ th item of *s*, origin 0 | (3)(9) | +--------------------------+--------------------------------+----------+ | ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) | +--------------------------+--------------------------------+----------+ @@ -1150,6 +1150,9 @@ Notes: without copying any data and with the returned index being relative to the start of the sequence rather than the start of the slice. +(9) + An :exc:`IndexError` is raised if *i* is outside the sequence range. + .. _typesseq-immutable: From aee45fd03f23c29aac57ecf747f9f1ee52235e72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 8 Jun 2025 14:34:57 +0200 Subject: [PATCH 114/116] gh-134531: refactor `_hashlib` logic for handling NIDs and EVP_MDs (#135254) --- Modules/_hashopenssl.c | 168 ++++++++++++++++++++++++++--------------- 1 file changed, 106 insertions(+), 62 deletions(-) diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 42821ebe9f6a54..50cf3c57491049 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -368,41 +368,83 @@ notify_ssl_error_occurred(void) } /* LCOV_EXCL_STOP */ -static const char * -get_openssl_evp_md_utf8name(const EVP_MD *md) -{ - assert(md != NULL); - int nid = EVP_MD_nid(md); - const char *name = NULL; - const py_hashentry_t *h; +/* + * OpenSSL provides a way to go from NIDs to digest names for hash functions + * but lacks this granularity for MAC objects where it is not possible to get + * the underlying digest name (only the block size and digest size are allowed + * to be recovered). + * + * In addition, OpenSSL aliases pollute the list of known digest names + * as OpenSSL appears to have its own definition of alias. In particular, + * the resulting list still contains duplicate and alternate names for several + * algorithms. + * + * Therefore, digest names, whether they are used by hash functions or HMAC, + * are handled through EVP_MD objects or directly by using some NID. + */ - for (h = py_hashes; h->py_name != NULL; h++) { +/* Get a cached entry by OpenSSL NID. */ +static const py_hashentry_t * +get_hashentry_by_nid(int nid) +{ + for (const py_hashentry_t *h = py_hashes; h->py_name != NULL; h++) { if (h->ossl_nid == nid) { - name = h->py_name; - break; + return h; } } + return NULL; +} + +/* + * Convert the NID to a string via OBJ_nid2*() functions. + * + * If 'nid' cannot be resolved, set an exception and return NULL. + */ +static const char * +get_asn1_utf8name_by_nid(int nid) +{ + const char *name = OBJ_nid2ln(nid); if (name == NULL) { - /* Ignore aliased names and only use long, lowercase name. The aliases - * pollute the list and OpenSSL appears to have its own definition of - * alias as the resulting list still contains duplicate and alternate - * names for several algorithms. - */ - name = OBJ_nid2ln(nid); - if (name == NULL) - name = OBJ_nid2sn(nid); + // In OpenSSL 3.0 and later, OBJ_nid*() are thread-safe and may raise. + assert(ERR_peek_last_error() != 0); + if (ERR_GET_REASON(ERR_peek_last_error()) != OBJ_R_UNKNOWN_NID) { + notify_ssl_error_occurred(); + return NULL; + } + // fallback to short name and unconditionally propagate errors + name = OBJ_nid2sn(nid); + if (name == NULL) { + raise_ssl_error(PyExc_ValueError, "cannot resolve NID %d", nid); + } } return name; } -static PyObject * -get_openssl_evp_md_name(const EVP_MD *md) +/* + * Convert the NID to an OpenSSL digest name. + * + * On error, set an exception and return NULL. + */ +static const char * +get_hashlib_utf8name_by_nid(int nid) +{ + const py_hashentry_t *e = get_hashentry_by_nid(nid); + return e ? e->py_name : get_asn1_utf8name_by_nid(nid); +} + +/* Same as get_hashlib_utf8name_by_nid() but using an EVP_MD object. */ +static const char * +get_hashlib_utf8name_by_evp_md(const EVP_MD *md) { - const char *name = get_openssl_evp_md_utf8name(md); - return PyUnicode_FromString(name); + assert(md != NULL); + return get_hashlib_utf8name_by_nid(EVP_MD_nid(md)); } -/* Get EVP_MD by HID and purpose */ +/* + * Get a new reference to an EVP_MD object described by name and purpose. + * + * If 'name' is an OpenSSL indexed name, the return value is cached. + */ static PY_EVP_MD * get_openssl_evp_md_by_utf8name(PyObject *module, const char *name, Py_hash_type py_ht) @@ -471,42 +513,46 @@ get_openssl_evp_md_by_utf8name(PyObject *module, const char *name, return digest; } -/* Get digest EVP_MD from object +/* + * Raise an exception indicating that 'digestmod' is not supported. + */ +static void +raise_unsupported_digestmod_error(PyObject *module, PyObject *digestmod) +{ + _hashlibstate *state = get_hashlib_state(module); + PyErr_Format(state->unsupported_digestmod_error, + "Unsupported digestmod %R", digestmod); +} + +/* + * Get a new reference to an EVP_MD described by 'digestmod' and purpose. + * + * On error, set an exception and return NULL. * - * * string - * * _hashopenssl builtin function + * Parameters * - * on error returns NULL with exception set. + * digestmod A digest name or a _hashopenssl builtin function + * py_ht The message digest purpose. */ static PY_EVP_MD * -get_openssl_evp_md(PyObject *module, PyObject *digestmod, - Py_hash_type py_ht) +get_openssl_evp_md(PyObject *module, PyObject *digestmod, Py_hash_type py_ht) { - PyObject *name_obj = NULL; const char *name; - if (PyUnicode_Check(digestmod)) { - name_obj = digestmod; - } else { - _hashlibstate *state = get_hashlib_state(module); - // borrowed ref - name_obj = PyDict_GetItemWithError(state->constructs, digestmod); + name = PyUnicode_AsUTF8(digestmod); } - if (name_obj == NULL) { - if (!PyErr_Occurred()) { - _hashlibstate *state = get_hashlib_state(module); - PyErr_Format( - state->unsupported_digestmod_error, - "Unsupported digestmod %R", digestmod); - } - return NULL; + else { + PyObject *dict = get_hashlib_state(module)->constructs; + assert(dict != NULL); + PyObject *borrowed_ref = PyDict_GetItemWithError(dict, digestmod); + name = borrowed_ref == NULL ? NULL : PyUnicode_AsUTF8(borrowed_ref); } - - name = PyUnicode_AsUTF8(name_obj); if (name == NULL) { + if (!PyErr_Occurred()) { + raise_unsupported_digestmod_error(module, digestmod); + } return NULL; } - return get_openssl_evp_md_by_utf8name(module, name, py_ht); } @@ -745,7 +791,9 @@ _hashlib_HASH_get_name(PyObject *op, void *Py_UNUSED(closure)) notify_ssl_error_occurred(); return NULL; } - return get_openssl_evp_md_name(md); + const char *name = get_hashlib_utf8name_by_evp_md(md); + assert(name != NULL || PyErr_Occurred()); + return name == NULL ? NULL : PyUnicode_FromString(name); } static PyGetSetDef HASH_getsets[] = { @@ -1775,20 +1823,15 @@ _hmac_dealloc(PyObject *op) static PyObject * _hmac_repr(PyObject *op) { + const char *digest_name; HMACobject *self = HMACobject_CAST(op); const EVP_MD *md = _hashlib_hmac_get_md(self); - if (md == NULL) { - return NULL; - } - PyObject *digest_name = get_openssl_evp_md_name(md); + digest_name = md == NULL ? NULL : get_hashlib_utf8name_by_evp_md(md); if (digest_name == NULL) { + assert(PyErr_Occurred()); return NULL; } - PyObject *repr = PyUnicode_FromFormat( - "<%U HMAC object @ %p>", digest_name, self - ); - Py_DECREF(digest_name); - return repr; + return PyUnicode_FromFormat("<%s HMAC object @ %p>", digest_name, self); } /*[clinic input] @@ -1900,13 +1943,12 @@ _hashlib_hmac_get_name(PyObject *op, void *Py_UNUSED(closure)) if (md == NULL) { return NULL; } - PyObject *digest_name = get_openssl_evp_md_name(md); + const char *digest_name = get_hashlib_utf8name_by_evp_md(md); if (digest_name == NULL) { + assert(PyErr_Occurred()); return NULL; } - PyObject *name = PyUnicode_FromFormat("hmac-%U", digest_name); - Py_DECREF(digest_name); - return name; + return PyUnicode_FromFormat("hmac-%s", digest_name); } static PyMethodDef HMAC_methods[] = { @@ -1982,7 +2024,9 @@ _openssl_hash_name_mapper(const EVP_MD *md, const char *from, return; } - py_name = get_openssl_evp_md_name(md); + const char *name = get_hashlib_utf8name_by_evp_md(md); + assert(name != NULL || PyErr_Occurred()); + py_name = name == NULL ? NULL : PyUnicode_FromString(name); if (py_name == NULL) { state->error = 1; } else { From 254bdac71121ee8028967388104a3f083250211e Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sun, 8 Jun 2025 17:33:19 +0100 Subject: [PATCH 115/116] Update tutorial for new "Copy" button (#135007) Co-authored-by: Sergey B Kirpichev --- Doc/tutorial/introduction.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index cdb35da7bc95ba..9e06e03991bc96 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -13,10 +13,9 @@ end a multi-line command. .. only:: html - You can toggle the display of prompts and output by clicking on ``>>>`` - in the upper-right corner of an example box. If you hide the prompts - and output for an example, then you can easily copy and paste the input - lines into your interpreter. + You can use the "Copy" button (it appears in the upper-right corner + when hovering over or tapping a code example), which strips prompts + and omits output, to copy and paste the input lines into your interpreter. .. index:: single: # (hash); comment From aac22ea212849f8fffee9e05af7429c503d973ee Mon Sep 17 00:00:00 2001 From: Alper Date: Sun, 8 Jun 2025 09:55:12 -0700 Subject: [PATCH 116/116] Add compile_commands.json to .gitignore (#135111) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index cdb0352e0a8836..7aa6272cf8e382 100644 --- a/.gitignore +++ b/.gitignore @@ -131,6 +131,7 @@ Tools/unicode/data/ /autom4te.cache /build/ /builddir/ +/compile_commands.json /config.cache /config.log /config.status