diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0aea3983fa600d..3d39e0c4ef0417 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,6 +4,9 @@ # It uses the same pattern rule for gitignore file # https://git-scm.com/docs/gitignore#_pattern_format +# GitHub +.github/** @ezio-melotti + # asyncio **/*asyncio* @1st1 @asvetlov @@ -130,7 +133,7 @@ Lib/ast.py @isidentical **/*idlelib* @terryjreedy -**/*typing* @gvanrossum @Fidget-Spinner @JelleZijlstra +**/*typing* @gvanrossum @Fidget-Spinner @JelleZijlstra @AlexWaygood **/*asyncore @giampaolo **/*asynchat @giampaolo diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 627f57070d200b..f4affee76e1d44 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -38,7 +38,7 @@ also suggestions on how you can most effectively help the project. Please be aware that our workflow does deviate slightly from the typical GitHub project. Details on how to properly submit a pull request are covered in -`Lifecycle of a Pull Request `_. +`Lifecycle of a Pull Request `_. We utilize various bots and status checks to help with this, so do follow the comments they leave and their "Details" links, respectively. The key points of our workflow that are not covered by a bot or status check are: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bb08b2ff62d511..328714e877147e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,6 +22,13 @@ on: - '3.8' - '3.7' +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: check_source: name: 'Check for source changes' @@ -35,8 +42,8 @@ jobs: id: check run: | if [ -z "$GITHUB_BASE_REF" ]; then - echo '::set-output name=run_tests::true' - echo '::set-output name=run_ssl_tests::true' + echo "run_tests=true" >> $GITHUB_OUTPUT + echo "run_ssl_tests=true" >> $GITHUB_OUTPUT else git fetch origin $GITHUB_BASE_REF --depth=1 # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more @@ -52,8 +59,8 @@ jobs: # into the PR branch anyway. # # https://github.com/python/core-workflow/issues/373 - git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo '::set-output name=run_tests::true' || true - git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE '(ssl|hashlib|hmac|^.github)' && echo '::set-output name=run_ssl_tests::true' || true + git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true + git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE '(ssl|hashlib|hmac|^.github)' && echo "run_ssl_tests=true" >> $GITHUB_OUTPUT || true fi check_abi: @@ -77,8 +84,7 @@ jobs: make -j4 - name: Check for changes in the ABI run: | - make check-abidump - if [ $? -neq 0 ] ; then + if ! make check-abidump; then echo "Generated ABI file is not up to date." echo "Please, add the release manager of this branch as a reviewer of this PR." echo "" @@ -100,7 +106,7 @@ jobs: - name: Add ccache to PATH run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1 + uses: hendrikmuhs/ccache-action@v1.2 - name: Check Autoconf version 2.69 and aclocal 1.16.3 run: | grep "Generated by GNU Autoconf 2.69" configure @@ -229,7 +235,7 @@ jobs: run: | echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1 + uses: hendrikmuhs/ccache-action@v1.2 - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV @@ -292,7 +298,7 @@ jobs: run: | echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1 + uses: hendrikmuhs/ccache-action@v1.2 - name: Configure CPython run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR - name: Build CPython @@ -336,7 +342,7 @@ jobs: run: | echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1 + uses: hendrikmuhs/ccache-action@v1.2 - name: Configure CPython run: ./configure --with-address-sanitizer --without-pymalloc - name: Build CPython diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml index ec18735e9b9fa6..5243dbba5f6b3a 100644 --- a/.github/workflows/build_msi.yml +++ b/.github/workflows/build_msi.yml @@ -23,6 +23,13 @@ on: paths: - 'Tools/msi/**' +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: build_win32: name: 'Windows (x86) Installer' diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 73a6a50520e02d..af5c5d0ad2e209 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -25,6 +25,13 @@ on: - 'Misc/**' - '.github/workflows/doc.yml' +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: build_doc: name: 'Docs' diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index 8cd834419f00bf..b2b63472d83421 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -5,6 +5,9 @@ on: types: - opened +permissions: + issues: read + jobs: notify-new-bugs-announce: runs-on: ubuntu-latest @@ -39,7 +42,7 @@ jobs: assignee : issue.data.assignees.map(assignee => { return assignee.login }), body : issue.data.body }; - + const data = { from: "CPython Issues ", to: "new-bugs-announce@python.org", diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 61e3d1adf534d5..9f4754f912b09f 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -16,6 +16,10 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: verify: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index b3b22f471c2765..d42e111666f1e9 100644 --- a/.gitignore +++ b/.gitignore @@ -114,6 +114,7 @@ PCbuild/win32/ Tools/unicode/data/ /autom4te.cache /build/ +/builddir/ /config.cache /config.log /config.status diff --git a/Doc/about.rst b/Doc/about.rst index 0ce35667924b92..5e6160ff2700ed 100644 --- a/Doc/about.rst +++ b/Doc/about.rst @@ -7,7 +7,7 @@ These documents are generated from `reStructuredText`_ sources by `Sphinx`_, a document processor specifically written for the Python documentation. .. _reStructuredText: https://docutils.sourceforge.io/rst.html -.. _Sphinx: http://sphinx-doc.org/ +.. _Sphinx: https://www.sphinx-doc.org/ .. In the online version of these documents, you can submit comments and suggest changes directly on the documentation pages. diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 0c15045fcb3a39..85f9eda17a2058 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -335,7 +335,7 @@ Other objects status = converter(object, address); where *object* is the Python object to be converted and *address* is the - :c:type:`void*` argument that was passed to the ``PyArg_Parse*`` function. + :c:expr:`void*` argument that was passed to the ``PyArg_Parse*`` function. The returned *status* should be ``1`` for a successful conversion and ``0`` if the conversion has failed. When the conversion fails, the *converter* function should raise an exception and leave the content of *address* unmodified. diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index 9b9c4ffa4d0343..fdb321fe7ab3f2 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -28,7 +28,8 @@ not. The wrappers ensure that ``str[size-1]`` is always ``'\0'`` upon return. They never write more than *size* bytes (including the trailing ``'\0'``) into str. Both functions require that ``str != NULL``, ``size > 0``, ``format != NULL`` -and ``size < INT_MAX``. +and ``size < INT_MAX``. Note that this means there is no equivalent to the C99 +``n = snprintf(NULL, 0, ...)`` which would determine the necessary buffer size. The return value (*rv*) for these functions should be interpreted as follows: diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 041afedee294ab..819168d48707c1 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -118,7 +118,7 @@ Dictionary Objects .. c:function:: PyObject* PyDict_GetItemString(PyObject *p, const char *key) This is the same as :c:func:`PyDict_GetItem`, but *key* is specified as a - :c:type:`const char*`, rather than a :c:expr:`PyObject*`. + :c:expr:`const char*`, rather than a :c:expr:`PyObject*`. Note that exceptions which occur while calling :meth:`__hash__` and :meth:`__eq__` methods and creating a temporary string object diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 46ce700abf1474..b52f3477ee2316 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -19,6 +19,24 @@ can be used to get a frame object. See also :ref:`Reflection `. +.. c:var:: PyTypeObject PyFrame_Type + + The type of frame objects. + It is the same object as :py:class:`types.FrameType` in the Python layer. + + .. versionchanged:: 3.11 + + Previously, this type was only available after including + ````. + +.. c:function:: int PyFrame_Check(PyObject *obj) + + Return non-zero if *obj* is a frame object. + + .. versionchanged:: 3.11 + + Previously, this function was only available after including + ````. .. c:function:: PyFrameObject* PyFrame_GetBack(PyFrameObject *frame) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index aecc3b877d8faa..55c05096ae91d0 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1741,8 +1741,8 @@ you need to include :file:`pythread.h` to use thread-local storage. .. note:: None of these API functions handle memory management on behalf of the - :c:type:`void*` values. You need to allocate and deallocate them yourself. - If the :c:type:`void*` values happen to be :c:expr:`PyObject*`, these + :c:expr:`void*` values. You need to allocate and deallocate them yourself. + If the :c:expr:`void*` values happen to be :c:expr:`PyObject*`, these functions don't do refcount operations on them either. .. _thread-specific-storage-api: @@ -1800,7 +1800,7 @@ is not possible due to its implementation being opaque at build time. .. note:: A freed key becomes a dangling pointer. You should reset the key to - `NULL`. + ``NULL``. Methods diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 6743045cac37de..24d66201f68dbb 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -254,7 +254,7 @@ PyPreConfig .. c:member:: int configure_locale - Set the LC_CTYPE locale to the user preferred locale? + Set the LC_CTYPE locale to the user preferred locale. If equals to ``0``, set :c:member:`~PyPreConfig.coerce_c_locale` and :c:member:`~PyPreConfig.coerce_c_locale_warn` members to ``0``. diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index 571d68cdf361c2..8e25968c6909fd 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -43,7 +43,7 @@ The following functions allow marshalled values to be read back in. .. c:function:: long PyMarshal_ReadLongFromFile(FILE *file) - Return a C :c:type:`long` from the data stream in a :c:expr:`FILE*` opened + Return a C :c:expr:`long` from the data stream in a :c:expr:`FILE*` opened for reading. Only a 32-bit value can be read in using this function, regardless of the native size of :c:expr:`long`. @@ -53,7 +53,7 @@ The following functions allow marshalled values to be read back in. .. c:function:: int PyMarshal_ReadShortFromFile(FILE *file) - Return a C :c:type:`short` from the data stream in a :c:expr:`FILE*` opened + Return a C :c:expr:`short` from the data stream in a :c:expr:`FILE*` opened for reading. Only a 16-bit value can be read in using this function, regardless of the native size of :c:expr:`short`. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index f726cd48663b1c..7041c15d23fb83 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -95,6 +95,8 @@ for the I/O buffer escapes completely the Python memory manager. Allocator Domains ================= +.. _allocator-domains: + All allocating functions belong to one of three different "domains" (see also :c:type:`PyMemAllocatorDomain`). These domains represent different allocation strategies and are optimized for different purposes. The specific details on @@ -479,6 +481,25 @@ Customize Memory Allocators See also :c:member:`PyPreConfig.allocator` and :ref:`Preinitialize Python with PyPreConfig `. + .. warning:: + + :c:func:`PyMem_SetAllocator` does have the following contract: + + * It can be called after :c:func:`Py_PreInitialize` and before + :c:func:`Py_InitializeFromConfig` to install a custom memory + allocator. There are no restrictions over the installed allocator + other than the ones imposed by the domain (for instance, the Raw + Domain allows the allocator to be called without the GIL held). See + :ref:`the section on allocator domains ` for more + information. + + * If called after Python has finish initializing (after + :c:func:`Py_InitializeFromConfig` has been called) the allocator + **must** wrap the existing allocator. Substituting the current + allocator for some other arbitrary one is **not supported**. + + + .. c:function:: void PyMem_SetupDebugHooks(void) Setup :ref:`debug hooks in the Python memory allocators ` diff --git a/Doc/c-api/typehints.rst b/Doc/c-api/typehints.rst index 88554a346c0dda..4c1957a2a1dbca 100644 --- a/Doc/c-api/typehints.rst +++ b/Doc/c-api/typehints.rst @@ -15,7 +15,7 @@ two types exist -- :ref:`GenericAlias ` and Equivalent to calling the Python class :class:`types.GenericAlias`. The *origin* and *args* arguments set the ``GenericAlias``\ 's ``__origin__`` and ``__args__`` attributes respectively. - *origin* should be a :c:type:`PyTypeObject*`, and *args* can be a + *origin* should be a :c:expr:`PyTypeObject*`, and *args* can be a :c:expr:`PyTupleObject*` or any ``PyObject*``. If *args* passed is not a tuple, a 1-tuple is automatically constructed and ``__args__`` is set to ``(args,)``. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 4f4e239bfa5d9b..dc82620cc3ba53 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -149,10 +149,16 @@ Quick Reference +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ .. [#slots] - A slot name in parentheses indicates it is (effectively) deprecated. - Names in angle brackets should be treated as read-only. - Names in square brackets are for internal use only. - "" (as a prefix) means the field is required (must be non-``NULL``). + + **()**: A slot name in parentheses indicates it is (effectively) deprecated. + + **<>**: Names in angle brackets should be initially set to ``NULL`` and + treated as read-only. + + **[]**: Names in square brackets are for internal use only. + + **** (as a prefix) means the field is required (must be non-``NULL``). + .. [#cols] Columns: **"O"**: set on :c:type:`PyBaseObject_Type` @@ -1213,6 +1219,17 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Inheritance:** This flag is not inherited. + However, subclasses will not be instantiable unless they provide a + non-NULL :c:member:`~PyTypeObject.tp_new` (which is only possible + via the C API). + + .. note:: + + To disallow instantiating a class directly but allow instantiating + its subclasses (e.g. for an :term:`abstract base class`), + do not use this flag. + Instead, make :c:member:`~PyTypeObject.tp_new` only succeed for + subclasses. .. versionadded:: 3.10 @@ -1484,7 +1501,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) If the instances of this type are weakly referenceable, this field is greater than zero and contains the offset in the instance structure of the weak reference list head (ignoring the GC header, if present); this offset is used by - :c:func:`PyObject_ClearWeakRefs` and the :c:func:`PyWeakref_\*` functions. The + :c:func:`PyObject_ClearWeakRefs` and the ``PyWeakref_*`` functions. The instance structure needs to include a field of type :c:expr:`PyObject*` which is initialized to ``NULL``. @@ -1892,8 +1909,19 @@ and :c:type:`PyType_Type` effectively act as defaults.) Tuple of base types. - This is set for types created by a class statement. It should be ``NULL`` for - statically defined types. + This field should be set to ``NULL`` and treated as read-only. + Python will fill it in when the type is :c:func:`initialized `. + + For dynamically created classes, the ``Py_tp_bases`` + :c:type:`slot ` can be used instead of the *bases* argument + of :c:func:`PyType_FromSpecWithBases`. + The argument form is preferred. + + .. warning:: + + Multiple inheritance does not work well for statically defined types. + If you set ``tp_bases`` to a tuple, Python will not raise an error, + but some slots will only be inherited from the first base. **Inheritance:** @@ -1905,6 +1933,8 @@ and :c:type:`PyType_Type` effectively act as defaults.) Tuple containing the expanded set of base types, starting with the type itself and ending with :class:`object`, in Method Resolution Order. + This field should be set to ``NULL`` and treated as read-only. + Python will fill it in when the type is :c:func:`initialized `. **Inheritance:** diff --git a/Doc/conf.py b/Doc/conf.py index e5c989da0b360d..fd4ee2d5eee811 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -234,28 +234,3 @@ # Relative filename of the data files refcount_file = 'data/refcounts.dat' stable_abi_file = 'data/stable_abi.dat' - -# Sphinx 2 and Sphinx 3 compatibility -# ----------------------------------- - -# bpo-40204: Allow Sphinx 2 syntax in the C domain -c_allow_pre_v3 = True - -# bpo-40204: Disable warnings on Sphinx 2 syntax of the C domain since the -# documentation is built with -W (warnings treated as errors). -c_warn_on_allowed_pre_v3 = False - -# Fix '!' not working with C domain when pre_v3 is enabled -import sphinx - -if sphinx.version_info[:2] < (5, 3): - from sphinx.domains.c import CXRefRole - - original_run = CXRefRole.run - - def new_run(self): - if self.disabled: - return super(CXRefRole, self).run() - return original_run(self) - - CXRefRole.run = new_run diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 1694cad6f43ba7..cbd22a3e23ece5 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1010,10 +1010,10 @@ PyImport_Import:PyObject*::+1: PyImport_Import:PyObject*:name:0: PyImport_ImportFrozenModule:int::: -PyImport_ImportFrozenModule:const char*::: +PyImport_ImportFrozenModule:const char*:name:: PyImport_ImportFrozenModuleObject:int::: -PyImport_ImportFrozenModuleObject:PyObject*::+1: +PyImport_ImportFrozenModuleObject:PyObject*:name:+1: PyImport_ImportModule:PyObject*::+1: PyImport_ImportModule:const char*:name:: diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index 5f5abdf9c15067..e64db373344038 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -298,16 +298,16 @@ be directly useful to you: .. code-block:: shell-session - $ /opt/bin/python3.4-config --cflags - -I/opt/include/python3.4m -I/opt/include/python3.4m -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes + $ /opt/bin/python3.11-config --cflags + -I/opt/include/python3.11 -I/opt/include/python3.11 -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -* ``pythonX.Y-config --ldflags`` will give you the recommended flags when - linking: +* ``pythonX.Y-config --ldflags --embed`` will give you the recommended flags + when linking: .. code-block:: shell-session - $ /opt/bin/python3.4-config --ldflags - -L/opt/lib/python3.4/config-3.4m -lpthread -ldl -lutil -lm -lpython3.4m -Xlinker -export-dynamic + $ /opt/bin/python3.11-config --ldflags --embed + -L/opt/lib/python3.11/config-3.11-x86_64-linux-gnu -L/opt/lib -lpython3.11 -lpthread -ldl -lutil -lm .. note:: To avoid confusion between several Python installations (and especially diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 21ef1f04cfbcc1..5ba6383640cc73 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -208,7 +208,7 @@ a special case, for which the new value passed to the handler is ``NULL``. Python supports two pairs of attribute handlers; a type that supports attributes only needs to implement the functions for one pair. The difference is that one pair takes the name of the attribute as a :c:expr:`char\*`, while the other -accepts a :c:type:`PyObject\*`. Each type can use whichever pair makes more +accepts a :c:expr:`PyObject*`. Each type can use whichever pair makes more sense for the implementation's convenience. :: getattrfunc tp_getattr; /* char * version */ @@ -341,7 +341,7 @@ Type-specific Attribute Management For simplicity, only the :c:expr:`char\*` version will be demonstrated here; the type of the name parameter is the only difference between the :c:expr:`char\*` -and :c:type:`PyObject\*` flavors of the interface. This example effectively does +and :c:expr:`PyObject*` flavors of the interface. This example effectively does the same thing as the generic example above, but does not use the generic support added in Python 2.2. It explains how the handler functions are called, so that if you do need to extend their functionality, you'll understand diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index ad4e38a04acdce..489bca76432d85 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -188,7 +188,7 @@ at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions also available at https://docs.python.org/3/download.html. The documentation is written in reStructuredText and processed by `the Sphinx -documentation tool `__. The reStructuredText source for +documentation tool `__. The reStructuredText source for the documentation is part of the Python source distribution. @@ -270,7 +270,7 @@ Where in the world is www.python.org located? --------------------------------------------- The Python project's infrastructure is located all over the world and is managed -by the Python Infrastructure Team. Details `here `__. +by the Python Infrastructure Team. Details `here `__. Why is it called Python? diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 76bd1538b41015..584d33e9622e33 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -25,8 +25,9 @@ Reference Manual `. You can also write your own debugger by using the code for pdb as an example. The IDLE interactive development environment, which is part of the standard -Python distribution (normally available as Tools/scripts/idle), includes a -graphical debugger. +Python distribution (normally available as +`Tools/scripts/idle3 `_), +includes a graphical debugger. PythonWin is a Python IDE that includes a GUI debugger based on pdb. The PythonWin debugger colors breakpoints and has quite a few cool features such as @@ -78,7 +79,8 @@ set of modules required by a program and bind these modules together with a Python binary to produce a single executable. One is to use the freeze tool, which is included in the Python source tree as -``Tools/freeze``. It converts Python byte code to C arrays; with a C compiler you can +`Tools/freeze `_. +It converts Python byte code to C arrays; with a C compiler you can embed all your modules into a new program, which is then linked with the standard Python modules. @@ -114,7 +116,7 @@ Core Language Why am I getting an UnboundLocalError when the variable has a value? -------------------------------------------------------------------- -It can be a surprise to get the UnboundLocalError in previously working +It can be a surprise to get the :exc:`UnboundLocalError` in previously working code when it is modified by adding an assignment statement somewhere in the body of a function. @@ -123,6 +125,7 @@ This code: >>> x = 10 >>> def bar(): ... print(x) + ... >>> bar() 10 @@ -133,7 +136,7 @@ works, but this code: ... print(x) ... x += 1 -results in an UnboundLocalError: +results in an :exc:`!UnboundLocalError`: >>> foo() Traceback (most recent call last): @@ -155,6 +158,7 @@ global: ... global x ... print(x) ... x += 1 + ... >>> foobar() 10 @@ -176,6 +180,7 @@ keyword: ... x += 1 ... bar() ... print(x) + ... >>> foo() 10 11 @@ -273,7 +278,7 @@ main.py:: import mod print(config.x) -Note that using a module is also the basis for implementing the Singleton design +Note that using a module is also the basis for implementing the singleton design pattern, for the same reason. @@ -291,9 +296,9 @@ using multiple imports per line uses less screen space. It's good practice if you import modules in the following order: -1. standard library modules -- e.g. ``sys``, ``os``, ``getopt``, ``re`` +1. standard library modules -- e.g. :mod:`sys`, :mod:`os`, :mod:`argparse`, :mod:`re` 2. third-party library modules (anything installed in Python's site-packages - directory) -- e.g. mx.DateTime, ZODB, PIL.Image, etc. + directory) -- e.g. :mod:`!dateutil`, :mod:`!requests`, :mod:`!PIL.Image` 3. locally developed modules It is sometimes necessary to move imports to a function or class to avoid @@ -471,7 +476,7 @@ object ``x`` refers to). After this assignment we have two objects (the ints Some operations (for example ``y.append(10)`` and ``y.sort()``) mutate the object, whereas superficially similar operations (for example ``y = y + [10]`` -and ``sorted(y)``) create a new object. In general in Python (and in all cases +and :func:`sorted(y) `) create a new object. In general in Python (and in all cases in the standard library) a method that mutates an object will return ``None`` to help avoid getting the two types of operations confused. So if you mistakenly write ``y.sort()`` thinking it will give you a sorted copy of ``y``, @@ -644,7 +649,7 @@ Sequences can be copied by slicing:: How can I find the methods or attributes of an object? ------------------------------------------------------ -For an instance x of a user-defined class, ``dir(x)`` returns an alphabetized +For an instance ``x`` of a user-defined class, :func:`dir(x) ` returns an alphabetized list of the names containing the instance attributes and methods and attributes defined by its class. @@ -669,9 +674,9 @@ callable. Consider the following code:: <__main__.A object at 0x16D07CC> Arguably the class has a name: even though it is bound to two names and invoked -through the name B the created instance is still reported as an instance of -class A. However, it is impossible to say whether the instance's name is a or -b, since both names are bound to the same value. +through the name ``B`` the created instance is still reported as an instance of +class ``A``. However, it is impossible to say whether the instance's name is ``a`` or +``b``, since both names are bound to the same value. Generally speaking it should not be necessary for your code to "know the names" of particular values. Unless you are deliberately writing introspective @@ -841,7 +846,7 @@ How do I get int literal attribute instead of SyntaxError? ---------------------------------------------------------- Trying to lookup an ``int`` literal attribute in the normal manner gives -a syntax error because the period is seen as a decimal point:: +a :exc:`SyntaxError` because the period is seen as a decimal point:: >>> 1.__class__ File "", line 1 @@ -887,7 +892,7 @@ leading '0' in a decimal number (except '0'). How do I convert a number to a string? -------------------------------------- -To convert, e.g., the number 144 to the string '144', use the built-in type +To convert, e.g., the number ``144`` to the string ``'144'``, use the built-in type constructor :func:`str`. If you want a hexadecimal or octal representation, use the built-in functions :func:`hex` or :func:`oct`. For fancy formatting, see the :ref:`f-strings` and :ref:`formatstrings` sections, @@ -1006,11 +1011,11 @@ Not as such. For simple input parsing, the easiest approach is usually to split the line into whitespace-delimited words using the :meth:`~str.split` method of string objects and then convert decimal strings to numeric values using :func:`int` or -:func:`float`. ``split()`` supports an optional "sep" parameter which is useful +:func:`float`. :meth:`!split()` supports an optional "sep" parameter which is useful if the line uses something other than whitespace as a separator. For more complicated input parsing, regular expressions are more powerful -than C's :c:func:`sscanf` and better suited for the task. +than C's ``sscanf`` and better suited for the task. What does 'UnicodeDecodeError' or 'UnicodeEncodeError' error mean? @@ -1206,15 +1211,16 @@ difference is that a Python list can contain objects of many different types. The ``array`` module also provides methods for creating arrays of fixed types with compact representations, but they are slower to index than lists. Also -note that NumPy and other third party packages define array-like structures with +note that `NumPy `_ +and other third party packages define array-like structures with various characteristics as well. -To get Lisp-style linked lists, you can emulate cons cells using tuples:: +To get Lisp-style linked lists, you can emulate *cons cells* using tuples:: lisp_list = ("like", ("this", ("example", None) ) ) If mutability is desired, you could use lists instead of tuples. Here the -analogue of lisp car is ``lisp_list[0]`` and the analogue of cdr is +analogue of a Lisp *car* is ``lisp_list[0]`` and the analogue of *cdr* is ``lisp_list[1]``. Only do this if you're sure you really need to, because it's usually a lot slower than using Python lists. @@ -1270,16 +1276,28 @@ use a list comprehension:: A = [[None] * w for i in range(h)] Or, you can use an extension that provides a matrix datatype; `NumPy -`_ is the best known. +`_ is the best known. -How do I apply a method to a sequence of objects? -------------------------------------------------- +How do I apply a method or function to a sequence of objects? +------------------------------------------------------------- -Use a list comprehension:: +To call a method or function and accumulate the return values is a list, +a :term:`list comprehension` is an elegant solution:: result = [obj.method() for obj in mylist] + result = [function(obj) for obj in mylist] + +To just run the method or function without saving the return values, +a plain :keyword:`for` loop will suffice:: + + for obj in mylist: + obj.method() + + for obj in mylist: + function(obj) + .. _faq-augmented-assignment-tuple-error: Why does a_tuple[i] += ['item'] raise an exception when the addition works? @@ -1334,11 +1352,12 @@ that even though there was an error, the append worked:: ['foo', 'item'] To see why this happens, you need to know that (a) if an object implements an -``__iadd__`` magic method, it gets called when the ``+=`` augmented assignment +:meth:`~object.__iadd__` magic method, it gets called when the ``+=`` augmented +assignment is executed, and its return value is what gets used in the assignment statement; -and (b) for lists, ``__iadd__`` is equivalent to calling ``extend`` on the list +and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`~list.extend` on the list and returning the list. That's why we say that for lists, ``+=`` is a -"shorthand" for ``list.extend``:: +"shorthand" for :meth:`!list.extend`:: >>> a_list = [] >>> a_list += [1] @@ -1363,7 +1382,7 @@ Thus, in our tuple example what is happening is equivalent to:: ... TypeError: 'tuple' object does not support item assignment -The ``__iadd__`` succeeds, and thus the list is extended, but even though +The :meth:`!__iadd__` succeeds, and thus the list is extended, but even though ``result`` points to the same object that ``a_tuple[0]`` already points to, that final assignment still results in an error, because tuples are immutable. @@ -1440,7 +1459,8 @@ See also :ref:`why-self`. How do I check if an object is an instance of a given class or of a subclass of it? ----------------------------------------------------------------------------------- -Use the built-in function ``isinstance(obj, cls)``. You can check if an object +Use the built-in function :func:`isinstance(obj, cls) `. You can +check if an object is an instance of any of a number of classes by providing a tuple instead of a single class, e.g. ``isinstance(obj, (class1, class2, ...))``, and can also check whether an object is one of Python's built-in types, e.g. @@ -1537,13 +1557,13 @@ Here the ``UpperOut`` class redefines the ``write()`` method to convert the argument string to uppercase before calling the underlying ``self._outfile.write()`` method. All other methods are delegated to the underlying ``self._outfile`` object. The delegation is accomplished via the -``__getattr__`` method; consult :ref:`the language reference ` +:meth:`~object.__getattr__` method; consult :ref:`the language reference ` for more information about controlling attribute access. Note that for more general cases delegation can get trickier. When attributes -must be set as well as retrieved, the class must define a :meth:`__setattr__` +must be set as well as retrieved, the class must define a :meth:`~object.__setattr__` method too, and it must do so carefully. The basic implementation of -:meth:`__setattr__` is roughly equivalent to the following:: +:meth:`!__setattr__` is roughly equivalent to the following:: class X: ... @@ -1551,7 +1571,8 @@ method too, and it must do so carefully. The basic implementation of self.__dict__[name] = value ... -Most :meth:`__setattr__` implementations must modify ``self.__dict__`` to store +Most :meth:`!__setattr__` implementations must modify +:meth:`self.__dict__ ` to store local state for self without causing an infinite recursion. @@ -1689,17 +1710,17 @@ My class defines __del__ but it is not called when I delete the object. There are several possible reasons for this. -The del statement does not necessarily call :meth:`__del__` -- it simply +The :keyword:`del` statement does not necessarily call :meth:`~object.__del__` -- it simply decrements the object's reference count, and if this reaches zero -:meth:`__del__` is called. +:meth:`!__del__` is called. If your data structures contain circular links (e.g. a tree where each child has a parent reference and each parent has a list of children) the reference counts will never go back to zero. Once in a while Python runs an algorithm to detect such cycles, but the garbage collector might run some time after the last -reference to your data structure vanishes, so your :meth:`__del__` method may be +reference to your data structure vanishes, so your :meth:`!__del__` method may be called at an inconvenient and random time. This is inconvenient if you're trying -to reproduce a problem. Worse, the order in which object's :meth:`__del__` +to reproduce a problem. Worse, the order in which object's :meth:`!__del__` methods are executed is arbitrary. You can run :func:`gc.collect` to force a collection, but there *are* pathological cases where objects will never be collected. @@ -1707,7 +1728,7 @@ collected. Despite the cycle collector, it's still a good idea to define an explicit ``close()`` method on objects to be called whenever you're done with them. The ``close()`` method can then remove attributes that refer to subobjects. Don't -call :meth:`__del__` directly -- :meth:`__del__` should call ``close()`` and +call :meth:`!__del__` directly -- :meth:`!__del__` should call ``close()`` and ``close()`` should make sure that it can be called more than once for the same object. @@ -1724,7 +1745,7 @@ and sibling references (if they need them!). Normally, calling :func:`sys.exc_clear` will take care of this by clearing the last recorded exception. -Finally, if your :meth:`__del__` method raises an exception, a warning message +Finally, if your :meth:`!__del__` method raises an exception, a warning message is printed to :data:`sys.stderr`. @@ -1852,8 +1873,8 @@ For example, here is the implementation of How can a subclass control what data is stored in an immutable instance? ------------------------------------------------------------------------ -When subclassing an immutable type, override the :meth:`__new__` method -instead of the :meth:`__init__` method. The latter only runs *after* an +When subclassing an immutable type, override the :meth:`~object.__new__` method +instead of the :meth:`~object.__init__` method. The latter only runs *after* an instance is created, which is too late to alter data in an immutable instance. @@ -1955,8 +1976,8 @@ can't be made to work because it cannot detect changes to the attributes. To make the *lru_cache* approach work when the *station_id* is mutable, -the class needs to define the *__eq__* and *__hash__* methods so that -the cache can detect relevant attribute updates:: +the class needs to define the :meth:`~object.__eq__` and :meth:`~object.__hash__` +methods so that the cache can detect relevant attribute updates:: class Weather: "Example with a mutable station identifier" diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index 7768aafd979685..c0c92fdbbc84d1 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -167,7 +167,7 @@ How can I embed Python into a Windows application? Embedding the Python interpreter in a Windows app can be summarized as follows: -1. Do _not_ build Python into your .exe file directly. On Windows, Python must +1. Do **not** build Python into your .exe file directly. On Windows, Python must be a DLL to handle importing modules that are themselves DLL's. (This is the first key undocumented fact.) Instead, link to :file:`python{NN}.dll`; it is typically installed in ``C:\Windows\System``. *NN* is the Python version, a @@ -191,7 +191,7 @@ Embedding the Python interpreter in a Windows app can be summarized as follows: 2. If you use SWIG, it is easy to create a Python "extension module" that will make the app's data and methods available to Python. SWIG will handle just about all the grungy details for you. The result is C code that you link - *into* your .exe file (!) You do _not_ have to create a DLL file, and this + *into* your .exe file (!) You do **not** have to create a DLL file, and this also simplifies linking. 3. SWIG will create an init function (a C function) whose name depends on the @@ -218,10 +218,10 @@ Embedding the Python interpreter in a Windows app can be summarized as follows: 5. There are two problems with Python's C API which will become apparent if you use a compiler other than MSVC, the compiler used to build pythonNN.dll. - Problem 1: The so-called "Very High Level" functions that take FILE * + Problem 1: The so-called "Very High Level" functions that take ``FILE *`` arguments will not work in a multi-compiler environment because each - compiler's notion of a struct FILE will be different. From an implementation - standpoint these are very _low_ level functions. + compiler's notion of a ``struct FILE`` will be different. From an implementation + standpoint these are very low level functions. Problem 2: SWIG generates the following code when generating wrappers to void functions: diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 59f9426f603172..3d74d550dc345a 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -882,7 +882,7 @@ Glossary package A Python :term:`module` which can contain submodules or recursively, - subpackages. Technically, a package is a Python module with an + subpackages. Technically, a package is a Python module with a ``__path__`` attribute. See also :term:`regular package` and :term:`namespace package`. diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index a97d10cfe6bb69..adc2f37371a91a 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -732,9 +732,9 @@ your program, just in case they don't know:: if args.quiet: print(answer) elif args.verbose: - print("{} to the power {} equals {}".format(args.x, args.y, answer)) + print(f"{args.x} to the power {args.y} equals {answer}") else: - print("{}^{} == {}".format(args.x, args.y, answer)) + print(f"{args.x}^{args.y} == {answer}") Note that slight difference in the usage text. Note the ``[-v | -q]``, which tells us that we can either use ``-v`` or ``-q``, diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index e5ed32f69a8efb..4e9b3dee09137c 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1,5 +1,7 @@ .. highlight:: c +.. _howto-clinic: + ********************** Argument Clinic How-To ********************** diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index bad5e508b01900..990d006b3e9824 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -173,6 +173,7 @@ yourself some work and use :func:`auto()` for the values:: ... FRIDAY = auto() ... SATURDAY = auto() ... SUNDAY = auto() + ... WEEKEND = SATURDAY | SUNDAY .. _enum-advanced-tutorial: @@ -305,6 +306,10 @@ Iterating over the members of an enum does not provide the aliases:: >>> list(Shape) [, , ] + >>> list(Weekday) + [, , , , , , ] + +Note that the aliases ``Shape.ALIAS_FOR_SQUARE`` and ``Weekday.WEEKEND`` aren't shown. The special attribute ``__members__`` is a read-only ordered mapping of names to members. It includes all names defined in the enumeration, including the @@ -324,6 +329,11 @@ the enumeration members. For example, finding all the aliases:: >>> [name for name, member in Shape.__members__.items() if member.name != name] ['ALIAS_FOR_SQUARE'] +.. note:: + + Aliases for flags include values with multiple flags set, such as ``3``, + and no flags set, i.e. ``0``. + Comparisons ----------- @@ -751,7 +761,7 @@ flags being set, the boolean evaluation is :data:`False`:: False Individual flags should have values that are powers of two (1, 2, 4, 8, ...), -while combinations of flags won't:: +while combinations of flags will not:: >>> class Color(Flag): ... RED = auto() @@ -1107,8 +1117,8 @@ example of when ``KEEP`` is needed). .. _enum-class-differences: -How are Enums different? ------------------------- +How are Enums and Flags different? +---------------------------------- Enums have a custom metaclass that affects many aspects of both derived :class:`Enum` classes and their instances (members). @@ -1125,6 +1135,13 @@ responsible for ensuring that various other methods on the final :class:`Enum` class are correct (such as :meth:`__new__`, :meth:`__getnewargs__`, :meth:`__str__` and :meth:`__repr__`). +Flag Classes +^^^^^^^^^^^^ + +Flags have an expanded view of aliasing: to be canonical, the value of a flag +needs to be a power-of-two value, and not a duplicate name. So, in addition to the +:class:`Enum` definition of alias, a flag with no value (a.k.a. ``0``) or with more than one +power-of-two value (e.g. ``3``) is considered an alias. Enum Members (aka instances) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1134,9 +1151,35 @@ The most interesting thing about enum members is that they are singletons. and then puts a custom :meth:`__new__` in place to ensure that no new ones are ever instantiated by returning only the existing member instances. +Flag Members +^^^^^^^^^^^^ + +Flag members can be iterated over just like the :class:`Flag` class, and only the +canonical members will be returned. For example:: + + >>> list(Color) + [, , ] + +(Note that ``BLACK``, ``PURPLE``, and ``WHITE`` do not show up.) + +Inverting a flag member returns the corresponding positive value, +rather than a negative value --- for example:: + + >>> ~Color.RED + + +Flag members have a length corresponding to the number of power-of-two values +they contain. For example:: + + >>> len(Color.PURPLE) + 2 + .. _enum-cookbook: +Enum Cookbook +------------- + While :class:`Enum`, :class:`IntEnum`, :class:`StrEnum`, :class:`Flag`, and :class:`IntFlag` are expected to cover the majority of use-cases, they cannot @@ -1310,7 +1353,7 @@ enumerations):: DuplicateFreeEnum ^^^^^^^^^^^^^^^^^ -Raises an error if a duplicate member name is found instead of creating an +Raises an error if a duplicate member value is found instead of creating an alias:: >>> class DuplicateFreeEnum(Enum): diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 1c3bd23f9fee6d..e68bc2ebb1cc82 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -994,7 +994,7 @@ requesting iterator-2 and its corresponding key. The functools module ==================== -The :mod:`functools` module in Python 2.5 contains some higher-order functions. +The :mod:`functools` module contains some higher-order functions. A **higher-order function** takes one or more functions as input and returns a new function. The most useful tool in this module is the :func:`functools.partial` function. diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 2657b4ec6aaf9f..2eddb582da7c24 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -461,7 +461,7 @@ Module State Access from Slot Methods, Getters and Setters .. After adding to limited API: - If you use the `limited API __, + If you use the :ref:`limited API , you must update ``Py_LIMITED_API`` to ``0x030b0000``, losing ABI compatibility with earlier versions. diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index c51071879a4b97..bf6f54a841a7b9 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -765,13 +765,71 @@ serialization. Running a logging socket listener in production ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To run a logging listener in production, you may need to use a process-management tool -such as `Supervisor `_. `Here -`_ is a Gist which -provides the bare-bones files to run the above functionality using Supervisor: you -will need to change the ``/path/to/`` parts in the Gist to reflect the actual paths you -want to use. - +.. _socket-listener-gist: https://gist.github.com/vsajip/4b227eeec43817465ca835ca66f75e2b + +To run a logging listener in production, you may need to use a +process-management tool such as `Supervisor `_. +`Here is a Gist `__ +which provides the bare-bones files to run the above functionality using +Supervisor. It consists of the following files: + ++-------------------------+----------------------------------------------------+ +| File | Purpose | ++=========================+====================================================+ +| :file:`prepare.sh` | A Bash script to prepare the environment for | +| | testing | ++-------------------------+----------------------------------------------------+ +| :file:`supervisor.conf` | The Supervisor configuration file, which has | +| | entries for the listener and a multi-process web | +| | application | ++-------------------------+----------------------------------------------------+ +| :file:`ensure_app.sh` | A Bash script to ensure that Supervisor is running | +| | with the above configuration | ++-------------------------+----------------------------------------------------+ +| :file:`log_listener.py` | The socket listener program which receives log | +| | events and records them to a file | ++-------------------------+----------------------------------------------------+ +| :file:`main.py` | A simple web application which performs logging | +| | via a socket connected to the listener | ++-------------------------+----------------------------------------------------+ +| :file:`webapp.json` | A JSON configuration file for the web application | ++-------------------------+----------------------------------------------------+ +| :file:`client.py` | A Python script to exercise the web application | ++-------------------------+----------------------------------------------------+ + +The web application uses `Gunicorn `_, which is a +popular web application server that starts multiple worker processes to handle +requests. This example setup shows how the workers can write to the same log file +without conflicting with one another --- they all go through the socket listener. + +To test these files, do the following in a POSIX environment: + +#. Download `the Gist `__ + as a ZIP archive using the :guilabel:`Download ZIP` button. + +#. Unzip the above files from the archive into a scratch directory. + +#. In the scratch directory, run ``bash prepare.sh`` to get things ready. + This creates a :file:`run` subdirectory to contain Supervisor-related and + log files, and a :file:`venv` subdirectory to contain a virtual environment + into which ``bottle``, ``gunicorn`` and ``supervisor`` are installed. + +#. Run ``bash ensure_app.sh`` to ensure that Supervisor is running with + the above configuration. + +#. Run ``venv/bin/python client.py`` to exercise the web application, + which will lead to records being written to the log. + +#. Inspect the log files in the :file:`run` subdirectory. You should see the + most recent log lines in files matching the pattern :file:`app.log*`. They won't be in + any particular order, since they have been handled concurrently by different + worker processes in a non-deterministic way. + +#. You can shut down the listener and the web application by running + ``venv/bin/supervisorctl -c supervisor.conf shutdown``. + +You may need to tweak the configuration files in the unlikely event that the +configured ports clash with something else in your test environment. .. _context-info: @@ -3708,10 +3766,75 @@ instance). Then, you'd get this kind of result: WARNING:demo:Bar >>> -Of course, these above examples show output according to the format used by +Of course, the examples above show output according to the format used by :func:`~logging.basicConfig`, but you can use a different formatter when you configure logging. +Note that with the above scheme, you are somewhat at the mercy of buffering and +the sequence of write calls which you are intercepting. For example, with the +definition of ``LoggerWriter`` above, if you have the snippet + +.. code-block:: python + + sys.stderr = LoggerWriter(logger, logging.WARNING) + 1 / 0 + +then running the script results in + +.. code-block:: text + + WARNING:demo:Traceback (most recent call last): + + WARNING:demo: File "/home/runner/cookbook-loggerwriter/test.py", line 53, in + + WARNING:demo: + WARNING:demo:main() + WARNING:demo: File "/home/runner/cookbook-loggerwriter/test.py", line 49, in main + + WARNING:demo: + WARNING:demo:1 / 0 + WARNING:demo:ZeroDivisionError + WARNING:demo:: + WARNING:demo:division by zero + +As you can see, this output isn't ideal. That's because the underlying code +which writes to ``sys.stderr`` makes mutiple writes, each of which results in a +separate logged line (for example, the last three lines above). To get around +this problem, you need to buffer things and only output log lines when newlines +are seen. Let's use a slghtly better implementation of ``LoggerWriter``: + +.. code-block:: python + + class BufferingLoggerWriter(LoggerWriter): + def __init__(self, logger, level): + super().__init__(logger, level) + self.buffer = '' + + def write(self, message): + if '\n' not in message: + self.buffer += message + else: + parts = message.split('\n') + if self.buffer: + s = self.buffer + parts.pop(0) + self.logger.log(self.level, s) + self.buffer = parts.pop() + for part in parts: + self.logger.log(self.level, part) + +This just buffers up stuff until a newline is seen, and then logs complete +lines. With this approach, you get better output: + +.. code-block:: text + + WARNING:demo:Traceback (most recent call last): + WARNING:demo: File "/home/runner/cookbook-loggerwriter/main.py", line 55, in + WARNING:demo: main() + WARNING:demo: File "/home/runner/cookbook-loggerwriter/main.py", line 52, in main + WARNING:demo: 1/0 + WARNING:demo:ZeroDivisionError: division by zero + + .. patterns-to-avoid: Patterns to avoid diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index 588e895b04bde2..decce12bf3faf6 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -247,7 +247,7 @@ To accommodate those situations, Python provides :class:`functools.cmp_to_key` to wrap the comparison function to make it usable as a key function:: - sorted(words, key=cmp_to_key(strcoll) + sorted(words, key=cmp_to_key(strcoll)) # locale-aware sort order Odds and Ends ============= diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index 4969d2420d6ac9..ca09aee72bf879 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -517,7 +517,7 @@ References Some good alternative discussions of Python's Unicode support are: -* `Processing Text Files in Python 3 `_, by Nick Coghlan. +* `Processing Text Files in Python 3 `_, by Nick Coghlan. * `Pragmatic Unicode `_, a PyCon 2012 presentation by Ned Batchelder. The :class:`str` type is described in the Python library reference at diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 76efed95bd4ccb..67ca6e77bc00d1 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -63,7 +63,7 @@ Name Description action_ Specify how an argument should be handled ``'store'``, ``'store_const'``, ``'store_true'``, ``'append'``, ``'append_const'``, ``'count'``, ``'help'``, ``'version'`` choices_ Limit values to a specific set of choices ``['foo', 'bar']``, ``range(1, 10)``, or :class:`~collections.abc.Container` instance const_ Store a constant value -default_ Default value used when an argument is not provided Defaults to *None* +default_ Default value used when an argument is not provided Defaults to ``None`` dest_ Specify the attribute name used in the result namespace help_ Help message for an argument metavar_ Alternate display name for the argument as shown in help @@ -201,9 +201,10 @@ ArgumentParser objects * usage_ - The string describing the program usage (default: generated from arguments added to parser) - * description_ - Text to display before the argument help (default: none) + * description_ - Text to display before the argument help + (by default, no text) - * epilog_ - Text to display after the argument help (default: none) + * epilog_ - Text to display after the argument help (by default, no text) * parents_ - A list of :class:`ArgumentParser` objects whose arguments should also be included @@ -1914,8 +1915,8 @@ FileType objects Namespace(out=<_io.TextIOWrapper name='file.txt' mode='w' encoding='UTF-8'>, raw=<_io.FileIO name='raw.dat' mode='wb'>) FileType objects understand the pseudo-argument ``'-'`` and automatically - convert this into ``sys.stdin`` for readable :class:`FileType` objects and - ``sys.stdout`` for writable :class:`FileType` objects:: + convert this into :data:`sys.stdin` for readable :class:`FileType` objects and + :data:`sys.stdout` for writable :class:`FileType` objects:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('infile', type=argparse.FileType('r')) @@ -1932,7 +1933,7 @@ Argument groups .. method:: ArgumentParser.add_argument_group(title=None, description=None) By default, :class:`ArgumentParser` groups command-line arguments into - "positional arguments" and "optional arguments" when displaying help + "positional arguments" and "options" when displaying help messages. When there is a better conceptual grouping of arguments than this default one, appropriate groups can be created using the :meth:`add_argument_group` method:: diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index da0fe8c3d6b2d3..3ed2c4fa7370e5 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1990,20 +1990,28 @@ and classes for traversing abstract syntax trees: .. function:: literal_eval(node_or_string) - Safely evaluate an expression node or a string containing a Python literal or + Evaluate an expression node or a string containing only a Python literal or container display. The string or node provided may only consist of the following Python literal structures: strings, bytes, numbers, tuples, lists, dicts, sets, booleans, ``None`` and ``Ellipsis``. - This can be used for safely evaluating strings containing Python values from - untrusted sources without the need to parse the values oneself. It is not - capable of evaluating arbitrarily complex expressions, for example involving - operators or indexing. + This can be used for evaluating strings containing Python values without the + need to parse the values oneself. It is not capable of evaluating + arbitrarily complex expressions, for example involving operators or + indexing. + + This function had been documented as "safe" in the past without defining + what that meant. That was misleading. This is specifically designed not to + execute Python code, unlike the more general :func:`eval`. There is no + namespace, no name lookups, or ability to call out. But it is not free from + attack: A relatively small input can lead to memory exhaustion or to C stack + exhaustion, crashing the process. There is also the possibility for + excessive CPU consumption denial of service on some inputs. Calling it on + untrusted data is thus not recommended. .. warning:: - It is possible to crash the Python interpreter with a - sufficiently large/complex string due to stack depth limitations - in Python's AST compiler. + It is possible to crash the Python interpreter due to stack depth + limitations in Python's AST compiler. It can raise :exc:`ValueError`, :exc:`TypeError`, :exc:`SyntaxError`, :exc:`MemoryError` and :exc:`RecursionError` depending on the malformed diff --git a/Doc/library/asyncio-api-index.rst b/Doc/library/asyncio-api-index.rst index 54c1cd6582e494..ad475150fe7d91 100644 --- a/Doc/library/asyncio-api-index.rst +++ b/Doc/library/asyncio-api-index.rst @@ -57,7 +57,7 @@ await on multiple things with timeouts. - Monitor for completion. * - :func:`timeout` - - Run with a timeout. Useful in cases when `wait_for` is not suitable. + - Run with a timeout. Useful in cases when ``wait_for`` is not suitable. * - :func:`to_thread` - Asynchronously run a function in a separate OS thread. diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index 14f2c3533c9712..921a394a59fec7 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -149,7 +149,8 @@ adjusted:: Network logging can block the event loop. It is recommended to use -a separate thread for handling logs or use non-blocking IO. +a separate thread for handling logs or use non-blocking IO. For example, +see :ref:`blocking-handlers`. .. _asyncio-coroutine-not-scheduled: diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 93bca96fff6f10..28b7a900583b84 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -33,7 +33,8 @@ an event loop: Return the running event loop in the current OS thread. - If there is no running event loop a :exc:`RuntimeError` is raised. + Raise a :exc:`RuntimeError` if there is no running event loop. + This function can only be called from a coroutine or a callback. .. versionadded:: 3.7 @@ -42,27 +43,35 @@ an event loop: Get the current event loop. - If there is no current event loop set in the current OS thread, - the OS thread is main, and :func:`set_event_loop` has not yet - been called, asyncio will create a new event loop and set it as the - current one. + When called from a coroutine or a callback (e.g. scheduled with + call_soon or similar API), this function will always return the + running event loop. + + If there is no running event loop set, the function will return + the result of ``get_event_loop_policy().get_event_loop()`` call. Because this function has rather complex behavior (especially when custom event loop policies are in use), using the :func:`get_running_loop` function is preferred to :func:`get_event_loop` in coroutines and callbacks. - Consider also using the :func:`asyncio.run` function instead of using - lower level functions to manually create and close an event loop. + As noted above, consider using the higher-level :func:`asyncio.run` function, + instead of using these lower level functions to manually create and close an + event loop. .. deprecated:: 3.10 - Deprecation warning is emitted if there is no running event loop. - In future Python releases, this function will be an alias of - :func:`get_running_loop`. + Deprecation warning is emitted if there is no current event loop. + In Python 3.12 it will be an error. + + .. note:: + In Python versions 3.10.0--3.10.8 and 3.11.0 this function + (and other functions which used it implicitly) emitted a + :exc:`DeprecationWarning` if there was no running event loop, even if + the current loop was set. .. function:: set_event_loop(loop) - Set *loop* as a current event loop for the current OS thread. + Set *loop* as the current event loop for the current OS thread. .. function:: new_event_loop() @@ -856,9 +865,14 @@ TLS Upgrade Upgrade an existing transport-based connection to TLS. - Return a new transport instance, that the *protocol* must start using - immediately after the *await*. The *transport* instance passed to - the *start_tls* method should never be used again. + Create a TLS coder/decoder instance and insert it between the *transport* + and the *protocol*. The coder/decoder implements both *transport*-facing + protocol and *protocol*-facing transport. + + Return the created two-interface instance. After *await*, the *protocol* + must stop using the original *transport* and communicate with the returned + object only because the coder caches *protocol*-side data and sporadically + exchanges extra TLS session packets with *transport*. Parameters: diff --git a/Doc/library/asyncio-llapi-index.rst b/Doc/library/asyncio-llapi-index.rst index b7ad888a7b67ab..9ce48a24444e66 100644 --- a/Doc/library/asyncio-llapi-index.rst +++ b/Doc/library/asyncio-llapi-index.rst @@ -19,7 +19,7 @@ Obtaining the Event Loop - The **preferred** function to get the running event loop. * - :func:`asyncio.get_event_loop` - - Get an event loop instance (current or via the policy). + - Get an event loop instance (running or current via the current policy). * - :func:`asyncio.set_event_loop` - Set the event loop as current via the current policy. diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index a73e99510f0b4e..d0af45febd14e3 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -7,7 +7,7 @@ Policies ======== -An event loop policy is a global (per-interpreter) object +An event loop policy is a global object used to get and set the current :ref:`event loop `, as well as create new event loops. The default policy can be :ref:`replaced ` with @@ -112,6 +112,11 @@ asyncio ships with the following built-in policies: On Windows, :class:`ProactorEventLoop` is now used by default. + .. deprecated:: 3.11.1 + :meth:`get_event_loop` now emits a :exc:`DeprecationWarning` if there + is no current event loop set and a new event loop has been implicitly + created. In Python 3.12 it will be an error. + .. class:: WindowsSelectorEventLoopPolicy diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 969354ceb163b5..7bc906eaafc1f2 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -156,7 +156,8 @@ Base Transport will be received. After all buffered data is flushed, the protocol's :meth:`protocol.connection_lost() ` method will be called with - :const:`None` as its argument. + :const:`None` as its argument. The transport should not be + used once it is closed. .. method:: BaseTransport.is_closing() diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index ce88d70d6607d9..d87e3c042c9977 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -183,7 +183,8 @@ StreamReader .. class:: StreamReader Represents a reader object that provides APIs to read data - from the IO stream. + from the IO stream. As an :term:`asynchronous iterable`, the + object supports the :keyword:`async for` statement. It is not recommended to instantiate *StreamReader* objects directly; use :func:`open_connection` and :func:`start_server` diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 28d0b21e8180b6..4274638c5e8625 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -175,7 +175,7 @@ their completion. * the :meth:`~asyncio.subprocess.Process.communicate` and :meth:`~asyncio.subprocess.Process.wait` methods don't have a - *timeout* parameter: use the :func:`wait_for` function; + *timeout* parameter: use the :func:`~asyncio.wait_for` function; * the :meth:`Process.wait() ` method is asynchronous, whereas :meth:`subprocess.Popen.wait` method diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index f795f25525938f..bf922fae40ab91 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -18,6 +18,10 @@ and Tasks. Coroutines ========== +**Source code:** :source:`Lib/asyncio/coroutines.py` + +---------------------------------------------------- + :term:`Coroutines ` declared with the async/await syntax is the preferred way of writing asyncio applications. For example, the following snippet of code prints "hello", waits 1 second, @@ -230,6 +234,10 @@ is :meth:`loop.run_in_executor`. Creating Tasks ============== +**Source code:** :source:`Lib/asyncio/tasks.py` + +----------------------------------------------- + .. function:: create_task(coro, *, name=None, context=None) Wrap the *coro* :ref:`coroutine ` into a :class:`Task` diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index a02ba739146aaf..4ca3768f827c6b 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -53,11 +53,13 @@ The modern interface provides: Encode the :term:`bytes-like object` *s* using Base64 and return the encoded :class:`bytes`. - Optional *altchars* must be a :term:`bytes-like object` of at least - length 2 (additional characters are ignored) which specifies an alternative - alphabet for the ``+`` and ``/`` characters. This allows an application to e.g. - generate URL or filesystem safe Base64 strings. The default is ``None``, for - which the standard Base64 alphabet is used. + Optional *altchars* must be a :term:`bytes-like object` of length 2 which + specifies an alternative alphabet for the ``+`` and ``/`` characters. + This allows an application to e.g. generate URL or filesystem safe Base64 + strings. The default is ``None``, for which the standard Base64 alphabet is used. + + May assert or raise a a :exc:`ValueError` if the length of *altchars* is not 2. Raises a + :exc:`TypeError` if *altchars* is not a :term:`bytes-like object`. .. function:: b64decode(s, altchars=None, validate=False) @@ -65,9 +67,9 @@ The modern interface provides: Decode the Base64 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. - Optional *altchars* must be a :term:`bytes-like object` or ASCII string of - at least length 2 (additional characters are ignored) which specifies the - alternative alphabet used instead of the ``+`` and ``/`` characters. + Optional *altchars* must be a :term:`bytes-like object` or ASCII string + of length 2 which specifies the alternative alphabet used instead of the + ``+`` and ``/`` characters. A :exc:`binascii.Error` exception is raised if *s* is incorrectly padded. @@ -80,6 +82,7 @@ The modern interface provides: For more information about the strict base64 check, see :func:`binascii.a2b_base64` + May assert or raise a :exc:`ValueError` if the length of *altchars* is not 2. .. function:: standard_b64encode(s) diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index c2927c1ebd0e1b..9b40f80f5878a4 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -127,7 +127,7 @@ thoughts in mind: .. seealso:: * `Sorted Collections - `_ is a high performance + `_ is a high performance module that uses *bisect* to managed sorted collections of data. * The `SortedCollection recipe diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 4a665f2254f8a6..8225236350d22e 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -189,7 +189,8 @@ wider range of codecs when working with binary files: .. note:: - Underlying encoded files are always opened in binary mode. + If *encoding* is not ``None``, then the + underlying encoded files are always opened in binary mode. No automatic conversion of ``'\n'`` is done on reading and writing. The *mode* argument may be any binary mode acceptable to the built-in :func:`open` function; the ``'b'`` is automatically added. diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 72aa20d73d8bd3..bfd6a7f58b3e50 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -33,13 +33,17 @@ can be customized by end users easily. .. seealso:: + Module :mod:`tomllib` + TOML is a well-specified format for application configuration files. + It is specifically designed to be an improved version of INI. + Module :mod:`shlex` - Support for creating Unix shell-like mini-languages which can be used as - an alternate format for application configuration files. + Support for creating Unix shell-like mini-languages which can also + be used for application configuration files. Module :mod:`json` - The json module implements a subset of JavaScript syntax which can also - be used for this purpose. + The ``json`` module implements a subset of JavaScript syntax which is + sometimes used for configuration, but does not support comments. .. testsetup:: diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index be1dd0c9eb57e8..08a7c7d74eab97 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -110,7 +110,7 @@ Context Variables A read-only property. Set to the value the variable had before the :meth:`ContextVar.set` method call that created the token. - It points to :attr:`Token.MISSING` is the variable was not set + It points to :attr:`Token.MISSING` if the variable was not set before the call. .. attribute:: Token.MISSING diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 9dec7240d9c50f..f7e85f2baa73e8 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -416,7 +416,7 @@ Dialects support the following attributes: .. attribute:: Dialect.skipinitialspace - When :const:`True`, whitespace immediately following the *delimiter* is ignored. + When :const:`True`, spaces immediately following the *delimiter* are ignored. The default is :const:`False`. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 2900f77589e31b..5208f254a5a994 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -6,6 +6,8 @@ .. moduleauthor:: Thomas Heller +**Source code:** :source:`Lib/ctypes` + -------------- :mod:`ctypes` is a foreign function library for Python. It provides C compatible @@ -358,7 +360,7 @@ from within *IDLE* or *PythonWin*:: >>> printf(b"%f bottles of beer\n", 42.5) Traceback (most recent call last): File "", line 1, in - ArgumentError: argument 2: exceptions.TypeError: Don't know how to convert parameter 2 + ArgumentError: argument 2: TypeError: Don't know how to convert parameter 2 >>> As has been mentioned before, all Python types except integers, strings, and @@ -370,6 +372,26 @@ that they can be converted to the required C data type:: 31 >>> +.. _ctypes-calling-variadic-functions: + +Calling varadic functions +^^^^^^^^^^^^^^^^^^^^^^^^^ + +On a lot of platforms calling variadic functions through ctypes is exactly the same +as calling functions with a fixed number of parameters. On some platforms, and in +particular ARM64 for Apple Platforms, the calling convention for variadic functions +is different than that for regular functions. + +On those platforms it is required to specify the *argtypes* attribute for the +regular, non-variadic, function arguments: + +.. code-block:: python3 + + libc.printf.argtypes = [ctypes.c_char_p] + +Because specifying the attribute does inhibit portability it is adviced to always +specify ``argtypes`` for all variadic functions. + .. _ctypes-calling-functions-with-own-custom-data-types: @@ -421,7 +443,7 @@ prototype for a C function), and tries to convert the arguments to valid types:: >>> printf(b"%d %d %d", 1, 2, 3) Traceback (most recent call last): File "", line 1, in - ArgumentError: argument 2: exceptions.TypeError: wrong type + ArgumentError: argument 2: TypeError: wrong type >>> printf(b"%s %d %f\n", b"X", 2, 3) X 2 3.000000 13 @@ -471,7 +493,7 @@ single character Python bytes object into a C char:: >>> strchr(b"abcdef", b"def") Traceback (most recent call last): File "", line 1, in - ArgumentError: argument 2: exceptions.TypeError: one character string expected + ArgumentError: argument 2: TypeError: one character string expected >>> print(strchr(b"abcdef", b"x")) None >>> strchr(b"abcdef", b"d") diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index a69dbb2ac06572..e1d1171927c9e2 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -7,6 +7,8 @@ .. moduleauthor:: Eric S. Raymond .. sectionauthor:: Eric S. Raymond +**Source code:** :source:`Lib/curses/ascii.py` + -------------- The :mod:`curses.ascii` module supplies name constants for ASCII characters and diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 83e19fac6520eb..9b2c3fbd8e4ee3 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -9,6 +9,8 @@ .. sectionauthor:: Moshe Zadka .. sectionauthor:: Eric Raymond +**Source code:** :source:`Lib/curses` + -------------- The :mod:`curses` module provides an interface to the curses library, the diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 4364ac342471eb..32c524a7348719 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -79,9 +79,10 @@ Module contents class C: ... - @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False) + @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, + match_args=True, kw_only=False, slots=False, weakref_slot=False) class C: - ... + ... The parameters to :func:`dataclass` are: @@ -191,7 +192,7 @@ Module contents .. versionchanged:: 3.11 If a field name is already included in the ``__slots__`` of a base class, it will not be included in the generated ``__slots__`` - to prevent `overriding them `_. + to prevent :ref:`overriding them `. Therefore, do not use ``__slots__`` to retrieve the field names of a dataclass. Use :func:`fields` instead. To be able to determine inherited slots, @@ -482,10 +483,10 @@ Module contents @dataclass class Point: - x: float - _: KW_ONLY - y: float - z: float + x: float + _: KW_ONLY + y: float + z: float p = Point(0, y=1.5, z=2.0) @@ -578,8 +579,8 @@ value is not provided when creating the class:: @dataclass class C: i: int - j: int = None - database: InitVar[DatabaseType] = None + j: int | None = None + database: InitVar[DatabaseType | None] = None def __post_init__(self, database): if self.j is None and database is not None: @@ -773,24 +774,24 @@ default value have the following special behaviors: :: class IntConversionDescriptor: - def __init__(self, *, default): - self._default = default + def __init__(self, *, default): + self._default = default - def __set_name__(self, owner, name): - self._name = "_" + name + def __set_name__(self, owner, name): + self._name = "_" + name - def __get__(self, obj, type): - if obj is None: - return self._default + def __get__(self, obj, type): + if obj is None: + return self._default - return getattr(obj, self._name, self._default) + return getattr(obj, self._name, self._default) - def __set__(self, obj, value): - setattr(obj, self._name, int(value)) + def __set__(self, obj, value): + setattr(obj, self._name, int(value)) @dataclass class InventoryItem: - quantity_on_hand: IntConversionDescriptor = IntConversionDescriptor(default=100) + quantity_on_hand: IntConversionDescriptor = IntConversionDescriptor(default=100) i = InventoryItem() print(i.quantity_on_hand) # 100 diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 6bce940f75004b..260108136df7f1 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -114,7 +114,7 @@ reset them before monitoring a calculation. .. seealso:: * IBM's General Decimal Arithmetic Specification, `The General Decimal Arithmetic - Specification `_. + Specification `_. .. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 98866b72a30a93..1e323bd50066b7 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -38,7 +38,9 @@ interpreter. Some instructions are accompanied by one or more inline cache entries, which take the form of :opcode:`CACHE` instructions. These instructions are hidden by default, but can be shown by passing ``show_caches=True`` to - any :mod:`dis` utility. + any :mod:`dis` utility. Furthermore, the interpreter now adapts the + bytecode to specialize it for different runtime conditions. The + adaptive bytecode can be shown by passing ``adaptive=True``. Example: Given the function :func:`myfunc`:: @@ -71,8 +73,8 @@ The bytecode analysis API allows pieces of Python code to be wrapped in a :class:`Bytecode` object that provides easy access to details of the compiled code. -.. class:: Bytecode(x, *, first_line=None, current_offset=None, show_caches=False) - +.. class:: Bytecode(x, *, first_line=None, current_offset=None,\ + show_caches=False, adaptive=False) Analyse the bytecode corresponding to a function, generator, asynchronous generator, coroutine, method, string of source code, or a code object (as @@ -91,6 +93,12 @@ code. disassembled code. Setting this means :meth:`.dis` will display a "current instruction" marker against the specified opcode. + If *show_caches* is ``True``, :meth:`.dis` will display inline cache + entries used by the interpreter to specialize the bytecode. + + If *adaptive* is ``True``, :meth:`.dis` will display specialized bytecode + that may be different from the original bytecode. + .. classmethod:: from_traceback(tb, *, show_caches=False) Construct a :class:`Bytecode` instance from the given traceback, setting @@ -118,7 +126,7 @@ code. This can now handle coroutine and asynchronous generator objects. .. versionchanged:: 3.11 - Added the ``show_caches`` parameter. + Added the *show_caches* and *adaptive* parameters. Example: @@ -174,7 +182,7 @@ operation is being performed, so the intermediate analysis object isn't useful: Added *file* parameter. -.. function:: dis(x=None, *, file=None, depth=None, show_caches=False) +.. function:: dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False) Disassemble the *x* object. *x* can denote either a module, a class, a method, a function, a generator, an asynchronous generator, a coroutine, @@ -195,6 +203,12 @@ operation is being performed, so the intermediate analysis object isn't useful: The maximal depth of recursion is limited by *depth* unless it is ``None``. ``depth=0`` means no recursion. + If *show_caches* is ``True``, this function will display inline cache + entries used by the interpreter to specialize the bytecode. + + If *adaptive* is ``True``, this function will display specialized bytecode + that may be different from the original bytecode. + .. versionchanged:: 3.4 Added *file* parameter. @@ -205,10 +219,10 @@ operation is being performed, so the intermediate analysis object isn't useful: This can now handle coroutine and asynchronous generator objects. .. versionchanged:: 3.11 - Added the ``show_caches`` parameter. + Added the *show_caches* and *adaptive* parameters. -.. function:: distb(tb=None, *, file=None, show_caches=False) +.. function:: distb(tb=None, *, file=None, show_caches=False, adaptive=False) Disassemble the top-of-stack function of a traceback, using the last traceback if none was passed. The instruction causing the exception is @@ -221,11 +235,11 @@ operation is being performed, so the intermediate analysis object isn't useful: Added *file* parameter. .. versionchanged:: 3.11 - Added the ``show_caches`` parameter. + Added the *show_caches* and *adaptive* parameters. -.. function:: disassemble(code, lasti=-1, *, file=None, show_caches=False) - disco(code, lasti=-1, *, file=None, show_caches=False) +.. function:: disassemble(code, lasti=-1, *, file=None, show_caches=False, adaptive=False) + disco(code, lasti=-1, *, file=None, show_caches=False, adaptive=False) Disassemble a code object, indicating the last instruction if *lasti* was provided. The output is divided in the following columns: @@ -248,10 +262,10 @@ operation is being performed, so the intermediate analysis object isn't useful: Added *file* parameter. .. versionchanged:: 3.11 - Added the ``show_caches`` parameter. + Added the *show_caches* and *adaptive* parameters. -.. function:: get_instructions(x, *, first_line=None, show_caches=False) +.. function:: get_instructions(x, *, first_line=None, show_caches=False, adaptive=False) Return an iterator over the instructions in the supplied function, method, source code string or code object. @@ -264,10 +278,12 @@ operation is being performed, so the intermediate analysis object isn't useful: source line information (if any) is taken directly from the disassembled code object. + The *show_caches* and *adaptive* parameters work as they do in :func:`dis`. + .. versionadded:: 3.4 .. versionchanged:: 3.11 - Added the ``show_caches`` parameter. + Added the *show_caches* and *adaptive* parameters. .. function:: findlinestarts(code) diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 75c6ee289a91e9..c106d5a3383a5e 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -696,10 +696,10 @@ special Python comments following an example's source code: .. productionlist:: doctest directive: "#" "doctest:" `directive_options` - directive_options: `directive_option` ("," `directive_option`)\* + directive_options: `directive_option` ("," `directive_option`)* directive_option: `on_or_off` `directive_option_name` - on_or_off: "+" \| "-" - directive_option_name: "DONT_ACCEPT_BLANKLINE" \| "NORMALIZE_WHITESPACE" \| ... + on_or_off: "+" | "-" + directive_option_name: "DONT_ACCEPT_BLANKLINE" | "NORMALIZE_WHITESPACE" | ... Whitespace is not allowed between the ``+`` or ``-`` and the directive option name. The directive option name can be any of the option flag names explained diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index 34f45e20bae9d9..d7f89cf96368b5 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -7,6 +7,8 @@ .. versionadded:: 3.4 +**Source code:** :source:`Lib/ensurepip` + -------------- The :mod:`ensurepip` package provides support for bootstrapping the ``pip`` diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 241d19d3bfb30e..be0686203bd184 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -27,7 +27,8 @@ An enumeration: * is a set of symbolic names (members) bound to unique values -* can be iterated over to return its members in definition order +* can be iterated over to return its canonical (i.e. non-alias) members in + definition order * uses *call* syntax to return members by value * uses *index* syntax to return members by name @@ -92,6 +93,11 @@ Module Contents the bitwise operators without losing their :class:`IntFlag` membership. :class:`IntFlag` members are also subclasses of :class:`int`. (`Notes`_) + :class:`ReprEnum` + + Used by :class:`IntEnum`, :class:`StrEnum`, and :class:`IntFlag` + to keep the :class:`str() ` of the mixed-in type. + :class:`EnumCheck` An enumeration with the values ``CONTINUOUS``, ``NAMED_FLAGS``, and @@ -132,9 +138,20 @@ Module Contents Do not make ``obj`` a member. Can be used as a decorator. + :func:`global_enum` + + Modify the :class:`str() ` and :func:`repr` of an enum + to show its members as belonging to the module instead of its class. + Should only be used if the enum members will be exported to the + module global namespace. + + :func:`show_flag_values` + + Return a list of all power-of-two integers contained in a flag. + .. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` -.. versionadded:: 3.11 ``StrEnum``, ``EnumCheck``, ``FlagBoundary``, ``property``, ``member``, ``nonmember`` +.. versionadded:: 3.11 ``StrEnum``, ``EnumCheck``, ``ReprEnum``, ``FlagBoundary``, ``property``, ``member``, ``nonmember``, ``global_enum``, ``show_flag_values`` --------------- @@ -184,7 +201,7 @@ Data Types .. method:: EnumType.__getitem__(cls, name) - Returns the Enum member in *cls* matching *name*, or raises an :exc:`KeyError`:: + Returns the Enum member in *cls* matching *name*, or raises a :exc:`KeyError`:: >>> Color['BLUE'] @@ -231,10 +248,10 @@ Data Types .. note:: Enum member values - Member values can be anything: :class:`int`, :class:`str`, etc.. If + Member values can be anything: :class:`int`, :class:`str`, etc. If the exact value is unimportant you may use :class:`auto` instances and an - appropriate value will be chosen for you. Care must be taken if you mix - :class:`auto` with other values. + appropriate value will be chosen for you. See :class:`auto` for the + details. .. attribute:: Enum._ignore_ @@ -245,7 +262,7 @@ Data Types names will also be removed from the completed enumeration. See :ref:`TimePeriod ` for an example. - .. method:: Enum.__call__(cls, value, names=None, \*, module=None, qualname=None, type=None, start=1, boundary=None) + .. method:: Enum.__call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) This method is called in two different ways: @@ -262,8 +279,8 @@ Data Types :module: The name of the module the new Enum is created in. :qualname: The actual location in the module where this Enum can be found. :type: A mix-in type for the new Enum. - :start: The first integer value for the Enum (used by :class:`auto`) - :boundary: How to handle out-of-range values from bit operations (:class:`Flag` only) + :start: The first integer value for the Enum (used by :class:`auto`). + :boundary: How to handle out-of-range values from bit operations (:class:`Flag` only). .. method:: Enum.__dir__(self) @@ -305,7 +322,7 @@ Data Types >>> PowersOfThree.SECOND.value 6 - .. method:: Enum.__init_subclass__(cls, \**kwds) + .. method:: Enum.__init_subclass__(cls, **kwds) A *classmethod* that is used to further configure subsequent subclasses. By default, does nothing. @@ -363,7 +380,7 @@ Data Types .. method:: Enum.__format__(self) Returns the string used for *format()* and *f-string* calls. By default, - returns :meth:`__str__` returns, but can be overridden:: + returns :meth:`__str__` return value, but can be overridden:: >>> class OtherStyle(Enum): ... ALTERNATE = auto() @@ -416,19 +433,23 @@ Data Types in most of the same places that a string can be used. The result of any string operation performed on or with a *StrEnum* member is not part of the enumeration. - .. note:: There are places in the stdlib that check for an exact :class:`str` - instead of a :class:`str` subclass (i.e. ``type(unknown) == str`` - instead of ``isinstance(str, unknown)``), and in those locations you - will need to use ``str(StrEnum.member)``. + .. note:: + + There are places in the stdlib that check for an exact :class:`str` + instead of a :class:`str` subclass (i.e. ``type(unknown) == str`` + instead of ``isinstance(unknown, str)``), and in those locations you + will need to use ``str(StrEnum.member)``. .. note:: Using :class:`auto` with :class:`StrEnum` results in the lower-cased member name as the value. - .. note:: :meth:`__str__` is :func:`str.__str__` to better support the - *replacement of existing constants* use-case. :meth:`__format__` is likewise - :func:`str.__format__` for that same reason. + .. note:: + + :meth:`~object.__str__` is :meth:`!str.__str__` to better support the + *replacement of existing constants* use-case. :meth:`~object.__format__` is likewise + :meth:`!str.__format__` for that same reason. .. versionadded:: 3.11 @@ -460,13 +481,17 @@ Data Types .. method:: __iter__(self): - Returns all contained members:: + Returns all contained non-alias members:: >>> list(Color.RED) [] >>> list(purple) [, ] + .. versionchanged:: 3.11 + + Aliases are no longer returned during iteration. + .. method:: __len__(self): Returns number of members in flag:: @@ -534,11 +559,11 @@ Data Types Using :class:`auto` with :class:`Flag` results in integers that are powers of two, starting with ``1``. - .. versionchanged:: 3.11 The *repr()* of zero-valued flags has changed. It + .. versionchanged:: 3.11 The *repr()* of zero-valued flags has changed. It is now:: - >>> Color(0) # doctest: +SKIP - + >>> Color(0) # doctest: +SKIP + .. class:: IntFlag @@ -576,10 +601,30 @@ Data Types Using :class:`auto` with :class:`IntFlag` results in integers that are powers of two, starting with ``1``. - .. versionchanged:: 3.11 :meth:`__str__` is now :func:`int.__str__` to - better support the *replacement of existing constants* use-case. - :meth:`__format__` was already :func:`int.__format__` for that same reason. + .. versionchanged:: 3.11 + + :meth:`~object.__str__` is now :meth:`!int.__str__` to better support the + *replacement of existing constants* use-case. :meth:`~object.__format__` was + already :meth:`!int.__format__` for that same reason. + + Inversion of an :class:`!IntFlag` now returns a positive value that is the + union of all flags not in the given flag, rather than a negative value. + This matches the existing :class:`Flag` behavior. + +.. class:: ReprEnum + :class:`!ReprEum` uses the :meth:`repr() ` of :class:`Enum`, + but the :class:`str() ` of the mixed-in data type: + + * :meth:`!int.__str__` for :class:`IntEnum` and :class:`IntFlag` + * :meth:`!str.__str__` for :class:`StrEnum` + + Inherit from :class:`!ReprEnum` to keep the :class:`str() ` / :func:`format` + of the mixed-in data type instead of using the + :class:`Enum`-default :meth:`str() `. + + + .. versionadded:: 3.11 .. class:: EnumCheck @@ -620,7 +665,7 @@ Data Types .. attribute:: NAMED_FLAGS Ensure that any flag groups/masks contain only named flags -- useful when - values are specified instead of being generated by :func:`auto` + values are specified instead of being generated by :func:`auto`:: >>> from enum import Flag, verify, NAMED_FLAGS >>> @verify(NAMED_FLAGS) @@ -755,7 +800,21 @@ Utilities and Decorators For *Enum* and *IntEnum* that appropriate value will be the last value plus one; for *Flag* and *IntFlag* it will be the first power-of-two greater than the last value; for *StrEnum* it will be the lower-cased version of the - member's name. + member's name. Care must be taken if mixing *auto()* with manually specified + values. + + *auto* instances are only resolved when at the top level of an assignment: + + * ``FIRST = auto()`` will work (auto() is replaced with ``1``); + * ``SECOND = auto(), -2`` will work (auto is replaced with ``2``, so ``2, -2`` is + used to create the ``SECOND`` enum member; + * ``THREE = [auto(), -3]`` will *not* work (``, -3`` is used to + create the ``THREE`` enum member) + + .. versionchanged:: 3.11.1 + + In prior versions, ``auto()`` had to be the only thing + on the assignment line to work properly. ``_generate_next_value_`` can be overridden to customize the values used by *auto*. @@ -815,6 +874,22 @@ Utilities and Decorators .. versionadded:: 3.11 +.. decorator:: global_enum + + A decorator to change the :class:`str() ` and :func:`repr` of an enum + to show its members as belonging to the module instead of its class. + Should only be used when the enum members are exported + to the module global namespace (see :class:`re.RegexFlag` for an example). + + + .. versionadded:: 3.11 + +.. function:: show_flag_values(value) + + Return a list of all power-of-two integers contained in a flag *value*. + + .. versionadded:: 3.11 + --------------- Notes @@ -822,23 +897,23 @@ Notes :class:`IntEnum`, :class:`StrEnum`, and :class:`IntFlag` - These three enum types are designed to be drop-in replacements for existing - integer- and string-based values; as such, they have extra limitations: + These three enum types are designed to be drop-in replacements for existing + integer- and string-based values; as such, they have extra limitations: - - ``__str__`` uses the value and not the name of the enum member + - ``__str__`` uses the value and not the name of the enum member - - ``__format__``, because it uses ``__str__``, will also use the value of - the enum member instead of its name + - ``__format__``, because it uses ``__str__``, will also use the value of + the enum member instead of its name - If you do not need/want those limitations, you can either create your own - base class by mixing in the ``int`` or ``str`` type yourself:: + If you do not need/want those limitations, you can either create your own + base class by mixing in the ``int`` or ``str`` type yourself:: - >>> from enum import Enum - >>> class MyIntEnum(int, Enum): - ... pass + >>> from enum import Enum + >>> class MyIntEnum(int, Enum): + ... pass or you can reassign the appropriate :meth:`str`, etc., in your enum:: - >>> from enum import IntEnum - >>> class MyIntEnum(IntEnum): - ... __str__ = IntEnum.__str__ + >>> from enum import IntEnum + >>> class MyIntEnum(IntEnum): + ... __str__ = IntEnum.__str__ diff --git a/Doc/library/errno.rst b/Doc/library/errno.rst index 035340e256874f..5122c69697ef91 100644 --- a/Doc/library/errno.rst +++ b/Doc/library/errno.rst @@ -657,3 +657,12 @@ defined by the module. The specific list of defined symbols is available as Interface output queue is full .. versionadded:: 3.11 + +.. data:: ENOTCAPABLE + + Capabilities insufficient. This error is mapped to the exception + :exc:`PermissionError`. + + .. availability:: WASI, FreeBSD + + .. versionadded:: 3.11.1 diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 2eccbd17c482c0..1217b817b4e843 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -746,7 +746,12 @@ depending on the system error code. Raised when trying to run an operation without the adequate access rights - for example filesystem permissions. - Corresponds to :c:data:`errno` :py:data:`~errno.EACCES` and :py:data:`~errno.EPERM`. + Corresponds to :c:data:`errno` :py:data:`~errno.EACCES`, + :py:data:`~errno.EPERM`, and :py:data:`~errno.ENOTCAPABLE`. + + .. versionchanged:: 3.11.1 + WASI's :py:data:`~errno.ENOTCAPABLE` is now mapped to + :exc:`PermissionError`. .. exception:: ProcessLookupError @@ -929,21 +934,42 @@ their subgroups based on the types of the contained exceptions. .. method:: derive(excs) - Returns an exception group with the same :attr:`message`, - :attr:`__traceback__`, :attr:`__cause__`, :attr:`__context__` - and :attr:`__notes__` but which wraps the exceptions in ``excs``. + Returns an exception group with the same :attr:`message`, but which + wraps the exceptions in ``excs``. This method is used by :meth:`subgroup` and :meth:`split`. A subclass needs to override it in order to make :meth:`subgroup` and :meth:`split` return instances of the subclass rather - than :exc:`ExceptionGroup`. :: + than :exc:`ExceptionGroup`. + + :meth:`subgroup` and :meth:`split` copy the :attr:`__traceback__`, + :attr:`__cause__`, :attr:`__context__` and :attr:`__notes__` fields from + the original exception group to the one returned by :meth:`derive`, so + these fields do not need to be updated by :meth:`derive`. :: >>> class MyGroup(ExceptionGroup): ... def derive(self, exc): ... return MyGroup(self.message, exc) ... - >>> MyGroup("eg", [ValueError(1), TypeError(2)]).split(TypeError) - (MyGroup('eg', [TypeError(2)]), MyGroup('eg', [ValueError(1)])) + >>> e = MyGroup("eg", [ValueError(1), TypeError(2)]) + >>> e.add_note("a note") + >>> e.__context__ = Exception("context") + >>> e.__cause__ = Exception("cause") + >>> try: + ... raise e + ... except Exception as e: + ... exc = e + ... + >>> match, rest = exc.split(ValueError) + >>> exc, exc.__context__, exc.__cause__, exc.__notes__ + (MyGroup('eg', [ValueError(1), TypeError(2)]), Exception('context'), Exception('cause'), ['a note']) + >>> match, match.__context__, match.__cause__, match.__notes__ + (MyGroup('eg', [ValueError(1)]), Exception('context'), Exception('cause'), ['a note']) + >>> rest, rest.__context__, rest.__cause__, rest.__notes__ + (MyGroup('eg', [TypeError(2)]), Exception('context'), Exception('cause'), ['a note']) + >>> exc.__traceback__ is match.__traceback__ is rest.__traceback__ + True + Note that :exc:`BaseExceptionGroup` defines :meth:`__new__`, so subclasses that need a different constructor signature need to @@ -960,6 +986,10 @@ their subgroups based on the types of the contained exceptions. def derive(self, excs): return Errors(excs, self.exit_code) + Like :exc:`ExceptionGroup`, any subclass of :exc:`BaseExceptionGroup` which + is also a subclass of :exc:`Exception` can only wrap instances of + :exc:`Exception`. + .. versionadded:: 3.11 diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index 9163da57c7b999..46bf0fc2848058 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -48,7 +48,7 @@ patterns. Also note that :func:`functools.lru_cache` with the *maxsize* of 32768 is used to cache the compiled regex patterns in the following functions: :func:`fnmatch`, -:func:`fnmatchcase`, :func:`filter`. +:func:`fnmatchcase`, :func:`.filter`. .. function:: fnmatch(filename, pattern) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 8108c98332e964..110e7e5d7fb9a7 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -54,14 +54,14 @@ are always available. They are listed here in alphabetical order. .. |func-bytearray| replace:: ``bytearray()`` .. |func-bytes| replace:: ``bytes()`` -.. function:: abs(x, /) +.. function:: abs(x) Return the absolute value of a number. The argument may be an integer, a floating point number, or an object implementing :meth:`__abs__`. If the argument is a complex number, its magnitude is returned. -.. function:: aiter(async_iterable, /) +.. function:: aiter(async_iterable) Return an :term:`asynchronous iterator` for an :term:`asynchronous iterable`. Equivalent to calling ``x.__aiter__()``. @@ -70,7 +70,7 @@ are always available. They are listed here in alphabetical order. .. versionadded:: 3.10 -.. function:: all(iterable, /) +.. function:: all(iterable) Return ``True`` if all elements of the *iterable* are true (or if the iterable is empty). Equivalent to:: @@ -82,8 +82,8 @@ are always available. They are listed here in alphabetical order. return True -.. awaitablefunction:: anext(async_iterator, /) - anext(async_iterator, default, /) +.. awaitablefunction:: anext(async_iterator) + anext(async_iterator, default) When awaited, return the next item from the given :term:`asynchronous iterator`, or *default* if given and the iterator is exhausted. @@ -98,7 +98,7 @@ are always available. They are listed here in alphabetical order. .. versionadded:: 3.10 -.. function:: any(iterable, /) +.. function:: any(iterable) Return ``True`` if any element of the *iterable* is true. If the iterable is empty, return ``False``. Equivalent to:: @@ -110,7 +110,7 @@ are always available. They are listed here in alphabetical order. return False -.. function:: ascii(object, /) +.. function:: ascii(object) As :func:`repr`, return a string containing a printable representation of an object, but escape the non-ASCII characters in the string returned by @@ -118,7 +118,7 @@ are always available. They are listed here in alphabetical order. similar to that returned by :func:`repr` in Python 2. -.. function:: bin(x, /) +.. function:: bin(x) Convert an integer number to a binary string prefixed with "0b". The result is a valid Python expression. If *x* is not a Python :class:`int` object, it @@ -140,7 +140,7 @@ are always available. They are listed here in alphabetical order. See also :func:`format` for more information. -.. class:: bool(x=False, /) +.. class:: bool(x=False) Return a Boolean value, i.e. one of ``True`` or ``False``. *x* is converted using the standard :ref:`truth testing procedure `. If *x* is false @@ -222,7 +222,7 @@ are always available. They are listed here in alphabetical order. See also :ref:`binaryseq`, :ref:`typebytes`, and :ref:`bytes-methods`. -.. function:: callable(object, /) +.. function:: callable(object) Return :const:`True` if the *object* argument appears callable, :const:`False` if not. If this returns ``True``, it is still possible that a @@ -235,7 +235,7 @@ are always available. They are listed here in alphabetical order. in Python 3.2. -.. function:: chr(i, /) +.. function:: chr(i) Return the string representing a character whose Unicode code point is the integer *i*. For example, ``chr(97)`` returns the string ``'a'``, while @@ -364,7 +364,7 @@ are always available. They are listed here in alphabetical order. .. class:: complex(real=0, imag=0) - complex(string, /) + complex(string) Return a complex number with the value *real* + *imag*\*1j or convert a string or number to a complex number. If the first parameter is a string, it will @@ -397,7 +397,7 @@ are always available. They are listed here in alphabetical order. :meth:`__float__` are not defined. -.. function:: delattr(object, name, /) +.. function:: delattr(object, name) This is a relative of :func:`setattr`. The arguments are an object and a string. The string must be the name of one of the object's attributes. The @@ -408,8 +408,8 @@ are always available. They are listed here in alphabetical order. .. _func-dict: .. class:: dict(**kwarg) - dict(mapping, /, **kwarg) - dict(iterable, /, **kwarg) + dict(mapping, **kwarg) + dict(iterable, **kwarg) :noindex: Create a new dictionary. The :class:`dict` object is the dictionary class. @@ -420,7 +420,7 @@ are always available. They are listed here in alphabetical order. .. function:: dir() - dir(object, /) + dir(object) Without arguments, return the list of names in the current local scope. With an argument, attempt to return a list of valid attributes for that object. @@ -476,7 +476,7 @@ are always available. They are listed here in alphabetical order. class. -.. function:: divmod(a, b, /) +.. function:: divmod(a, b) Take two (non-complex) numbers as arguments and return a pair of numbers consisting of their quotient and remainder when using integer division. With @@ -619,7 +619,7 @@ are always available. They are listed here in alphabetical order. Added the *closure* parameter. -.. function:: filter(function, iterable, /) +.. function:: filter(function, iterable) Construct an iterator from those elements of *iterable* for which *function* returns true. *iterable* may be either a sequence, a container which @@ -636,7 +636,7 @@ are always available. They are listed here in alphabetical order. elements of *iterable* for which *function* returns false. -.. class:: float(x=0.0, /) +.. class:: float(x=0.0) .. index:: single: NaN @@ -704,7 +704,7 @@ are always available. They are listed here in alphabetical order. single: __format__ single: string; format() (built-in function) -.. function:: format(value, format_spec="", /) +.. function:: format(value, format_spec="") Convert a *value* to a "formatted" representation, as controlled by *format_spec*. The interpretation of *format_spec* will depend on the type @@ -727,7 +727,7 @@ are always available. They are listed here in alphabetical order. .. _func-frozenset: -.. class:: frozenset(iterable=set(), /) +.. class:: frozenset(iterable=set()) :noindex: Return a new :class:`frozenset` object, optionally with elements taken from @@ -739,8 +739,8 @@ are always available. They are listed here in alphabetical order. module. -.. function:: getattr(object, name, /) - getattr(object, name, default, /) +.. function:: getattr(object, name) + getattr(object, name, default) Return the value of the named attribute of *object*. *name* must be a string. If the string is the name of one of the object's attributes, the result is the @@ -764,7 +764,7 @@ are always available. They are listed here in alphabetical order. regardless of where the function is called. -.. function:: hasattr(object, name, /) +.. function:: hasattr(object, name) The arguments are an object and a string. The result is ``True`` if the string is the name of one of the object's attributes, ``False`` if not. (This @@ -772,7 +772,7 @@ are always available. They are listed here in alphabetical order. raises an :exc:`AttributeError` or not.) -.. function:: hash(object, /) +.. function:: hash(object) Return the hash value of the object (if it has one). Hash values are integers. They are used to quickly compare dictionary keys during a @@ -807,7 +807,7 @@ are always available. They are listed here in alphabetical order. signatures for callables are now more comprehensive and consistent. -.. function:: hex(x, /) +.. function:: hex(x) Convert an integer number to a lowercase hexadecimal string prefixed with "0x". If *x* is not a Python :class:`int` object, it has to define an @@ -839,7 +839,7 @@ are always available. They are listed here in alphabetical order. :meth:`float.hex` method. -.. function:: id(object, /) +.. function:: id(object) Return the "identity" of an object. This is an integer which is guaranteed to be unique and constant for this object during its lifetime. @@ -852,7 +852,7 @@ are always available. They are listed here in alphabetical order. .. function:: input() - input(prompt, /) + input(prompt) If the *prompt* argument is present, it is written to standard output without a trailing newline. The function then reads a line from input, converts it @@ -878,8 +878,8 @@ are always available. They are listed here in alphabetical order. with the result after successfully reading input. -.. class:: int(x=0, /) - int(x, /, base=10) +.. class:: int(x=0) + int(x, base=10) Return an integer object constructed from a number or string *x*, or return ``0`` if no arguments are given. If *x* defines :meth:`__int__`, @@ -930,7 +930,7 @@ are always available. They are listed here in alphabetical order. See the :ref:`integer string conversion length limitation ` documentation. -.. function:: isinstance(object, classinfo, /) +.. function:: isinstance(object, classinfo) Return ``True`` if the *object* argument is an instance of the *classinfo* argument, or of a (direct, indirect, or :term:`virtual `) of *classinfo*. A @@ -961,8 +961,8 @@ are always available. They are listed here in alphabetical order. *classinfo* can be a :ref:`types-union`. -.. function:: iter(object, /) - iter(object, sentinel, /) +.. function:: iter(object) + iter(object, sentinel) Return an :term:`iterator` object. The first argument is interpreted very differently depending on the presence of the second argument. Without a @@ -989,7 +989,7 @@ are always available. They are listed here in alphabetical order. process_block(block) -.. function:: len(s, /) +.. function:: len(s) Return the length (the number of items) of an object. The argument may be a sequence (such as a string, bytes, tuple, list, or range) or a collection @@ -1003,7 +1003,7 @@ are always available. They are listed here in alphabetical order. .. _func-list: .. class:: list() - list(iterable, /) + list(iterable) :noindex: Rather than being a function, :class:`list` is actually a mutable @@ -1021,7 +1021,7 @@ are always available. They are listed here in alphabetical order. The contents of this dictionary should not be modified; changes may not affect the values of local and free variables used by the interpreter. -.. function:: map(function, iterable, /, *iterables) +.. function:: map(function, iterable, *iterables) Return an iterator that applies *function* to every item of *iterable*, yielding the results. If additional *iterables* arguments are passed, @@ -1031,9 +1031,9 @@ are always available. They are listed here in alphabetical order. already arranged into argument tuples, see :func:`itertools.starmap`\. -.. function:: max(iterable, /, *, key=None) - max(iterable, /, *, default, key=None) - max(arg1, arg2, /, *args, key=None) +.. function:: max(iterable, *, key=None) + max(iterable, *, default, key=None) + max(arg1, arg2, *args, key=None) Return the largest item in an iterable or the largest of two or more arguments. @@ -1069,9 +1069,9 @@ are always available. They are listed here in alphabetical order. :ref:`typememoryview` for more information. -.. function:: min(iterable, /, *, key=None) - min(iterable, /, *, default, key=None) - min(arg1, arg2, /, *args, key=None) +.. function:: min(iterable, *, key=None) + min(iterable, *, default, key=None) + min(arg1, arg2, *args, key=None) Return the smallest item in an iterable or the smallest of two or more arguments. @@ -1099,8 +1099,8 @@ are always available. They are listed here in alphabetical order. The *key* can be ``None``. -.. function:: next(iterator, /) - next(iterator, default, /) +.. function:: next(iterator) + next(iterator, default) Retrieve the next item from the :term:`iterator` by calling its :meth:`~iterator.__next__` method. If *default* is given, it is returned @@ -1119,7 +1119,7 @@ are always available. They are listed here in alphabetical order. assign arbitrary attributes to an instance of the :class:`object` class. -.. function:: oct(x, /) +.. function:: oct(x) Convert an integer number to an octal string prefixed with "0o". The result is a valid Python expression. If *x* is not a Python :class:`int` object, it @@ -1371,7 +1371,7 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.11 The ``'U'`` mode has been removed. -.. function:: ord(c, /) +.. function:: ord(c) Given a string representing one Unicode character, return an integer representing the Unicode code point of that character. For example, @@ -1419,7 +1419,7 @@ are always available. They are listed here in alphabetical order. supported. -.. function:: print(*objects, sep=' ', end='\n', file=sys.stdout, flush=False) +.. function:: print(*objects, sep=' ', end='\n', file=None, flush=False) Print *objects* to the text stream *file*, separated by *sep* and followed by *end*. *sep*, *end*, *file*, and *flush*, if present, must be given as keyword @@ -1522,15 +1522,15 @@ are always available. They are listed here in alphabetical order. .. _func-range: -.. class:: range(stop, /) - range(start, stop, step=1, /) +.. class:: range(stop) + range(start, stop, step=1) :noindex: Rather than being a function, :class:`range` is actually an immutable sequence type, as documented in :ref:`typesseq-range` and :ref:`typesseq`. -.. function:: repr(object, /) +.. function:: repr(object) Return a string containing a printable representation of an object. For many types, this function makes an attempt to return a string that would yield an @@ -1543,7 +1543,7 @@ are always available. They are listed here in alphabetical order. :exc:`RuntimeError`. -.. function:: reversed(seq, /) +.. function:: reversed(seq) Return a reverse :term:`iterator`. *seq* must be an object which has a :meth:`__reversed__` method or supports the sequence protocol (the @@ -1580,7 +1580,7 @@ are always available. They are listed here in alphabetical order. .. _func-set: .. class:: set() - set(iterable, /) + set(iterable) :noindex: Return a new :class:`set` object, optionally with elements taken from @@ -1592,7 +1592,7 @@ are always available. They are listed here in alphabetical order. module. -.. function:: setattr(object, name, value, /) +.. function:: setattr(object, name, value) This is the counterpart of :func:`getattr`. The arguments are an object, a string, and an arbitrary value. The string may name an existing attribute or a @@ -1614,8 +1614,8 @@ are always available. They are listed here in alphabetical order. :func:`setattr`. -.. class:: slice(stop, /) - slice(start, stop, step=1, /) +.. class:: slice(stop) + slice(start, stop, step=1) Return a :term:`slice` object representing the set of indices specified by ``range(start, stop, step)``. The *start* and *step* arguments default to @@ -1733,7 +1733,7 @@ are always available. They are listed here in alphabetical order. The *start* parameter can be specified as a keyword argument. .. class:: super() - super(type, object_or_type=None, /) + super(type, object_or_type=None) Return a proxy object that delegates method calls to a parent or sibling class of *type*. This is useful for accessing inherited methods that have @@ -1804,15 +1804,15 @@ are always available. They are listed here in alphabetical order. .. _func-tuple: .. class:: tuple() - tuple(iterable, /) + tuple(iterable) :noindex: Rather than being a function, :class:`tuple` is actually an immutable sequence type, as documented in :ref:`typesseq-tuple` and :ref:`typesseq`. -.. class:: type(object, /) - type(name, bases, dict, /, **kwds) +.. class:: type(object) + type(name, bases, dict, **kwds) .. index:: object: type @@ -1853,7 +1853,7 @@ are always available. They are listed here in alphabetical order. longer use the one-argument form to get the type of an object. .. function:: vars() - vars(object, /) + vars(object) Return the :attr:`~object.__dict__` attribute for a module, class, instance, or any other object with a :attr:`~object.__dict__` attribute. diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index e6c59810dc6a87..87ef156a0bed57 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -322,8 +322,8 @@ writing. .. class:: MozillaCookieJar(filename=None, delayload=None, policy=None) A :class:`FileCookieJar` that can load from and save cookies to disk in the - Mozilla ``cookies.txt`` file format (which is also used by the Lynx and Netscape - browsers). + Mozilla ``cookies.txt`` file format (which is also used by curl and the Lynx + and Netscape browsers). .. note:: diff --git a/Doc/library/http.rst b/Doc/library/http.rst index 5895a41d849bd1..bd9dcf68f2fa08 100644 --- a/Doc/library/http.rst +++ b/Doc/library/http.rst @@ -146,16 +146,25 @@ equal to the constant name (i.e. ``http.HTTPStatus.OK`` is also available as Usage:: >>> from http import HTTPMethod - >>> HTTMethod.GET - HTTMethod.GET - >>> HTTMethod.GET == 'GET' + >>> + >>> HTTPMethod.GET + + >>> HTTPMethod.GET == 'GET' True - >>> HTTMethod.GET.value + >>> HTTPMethod.GET.value 'GET' - >>> HTTMethod.GET.description - 'Transfer a current representation of the target resource.' + >>> HTTPMethod.GET.description + 'Retrieve the target.' >>> list(HTTPMethod) - [HTTPMethod.GET, HTTPMethod.HEAD, ...] + [, + , + , + , + , + , + , + , + ] .. _http-methods: diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 81b6bf5373b495..6a564400708186 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -512,3 +512,12 @@ Security Considerations :class:`SimpleHTTPRequestHandler` will follow symbolic links when handling requests, this makes it possible for files outside of the specified directory to be served. + +Earlier versions of Python did not scrub control characters from the +log messages emitted to stderr from ``python -m http.server`` or the +default :class:`BaseHTTPRequestHandler` ``.log_message`` +implementation. This could allow remote clients connecting to your +server to send nefarious control codes to your terminal. + +.. versionadded:: 3.11.1 + Control characters are scrubbed in stderr logs. diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 107b7465c91b11..57991779ccfd62 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -13,29 +13,61 @@ **Source code:** :source:`Lib/importlib/metadata/__init__.py` -``importlib.metadata`` is a library that provides for access to installed -package metadata. Built in part on Python's import system, this library +``importlib_metadata`` is a library that provides access to +the metadata of an installed `Distribution Package `_, +such as its entry points +or its top-level names (`Import Package `_\s, modules, if any). +Built in part on Python's import system, this library intends to replace similar functionality in the `entry point API`_ and `metadata API`_ of ``pkg_resources``. Along with -:mod:`importlib.resources` (with new features backported to the -`importlib_resources`_ package), this can eliminate the need to use the older -and less efficient +:mod:`importlib.resources`, +this package can eliminate the need to use the older and less efficient ``pkg_resources`` package. -By "installed package" we generally mean a third-party package installed into -Python's ``site-packages`` directory via tools such as `pip -`_. Specifically, -it means a package with either a discoverable ``dist-info`` or ``egg-info`` -directory, and metadata defined by :pep:`566` or its older specifications. -By default, package metadata can live on the file system or in zip archives on +``importlib_metadata`` operates on third-party *distribution packages* +installed into Python's ``site-packages`` directory via tools such as +`pip `_. +Specifically, it works with distributions with discoverable +``dist-info`` or ``egg-info`` directories, +and metadata defined by the `Core metadata specifications `_. + +.. important:: + + These are *not* necessarily equivalent to or correspond 1:1 with + the top-level *import package* names + that can be imported inside Python code. + One *distribution package* can contain multiple *import packages* + (and single modules), + and one top-level *import package* + may map to multiple *distribution packages* + if it is a namespace package. + You can use :ref:`package_distributions() ` + to get a mapping between them. + +By default, distribution metadata can live on the file system +or in zip archives on :data:`sys.path`. Through an extension mechanism, the metadata can live almost anywhere. +.. seealso:: + + https://importlib-metadata.readthedocs.io/ + The documentation for ``importlib_metadata``, which supplies a + backport of ``importlib.metadata``. + This includes an `API reference + `__ + for this module's classes and functions, + as well as a `migration guide + `__ + for existing users of ``pkg_resources``. + + Overview ======== -Let's say you wanted to get the version string for a package you've installed +Let's say you wanted to get the version string for a +`Distribution Package `_ you've installed using ``pip``. We start by creating a virtual environment and installing something into it: @@ -54,9 +86,9 @@ You can get the version string for ``wheel`` by running the following: >>> version('wheel') # doctest: +SKIP '0.32.3' -You can also get the set of entry points keyed by group, such as +You can also get a collection of entry points selectable by properties of the EntryPoint (typically 'group' or 'name'), such as ``console_scripts``, ``distutils.commands`` and others. Each group contains a -sequence of :ref:`EntryPoint ` objects. +collection of :ref:`EntryPoint ` objects. You can get the :ref:`metadata for a distribution `:: @@ -91,7 +123,7 @@ Query all entry points:: >>> eps = entry_points() # doctest: +SKIP The ``entry_points()`` function returns an ``EntryPoints`` object, -a sequence of all ``EntryPoint`` objects with ``names`` and ``groups`` +a collection of all ``EntryPoint`` objects with ``names`` and ``groups`` attributes for convenience:: >>> sorted(eps.groups) # doctest: +SKIP @@ -156,7 +188,8 @@ interface to retrieve entry points by group. Distribution metadata --------------------- -Every distribution includes some metadata, which you can extract using the +Every `Distribution Package `_ includes some metadata, +which you can extract using the ``metadata()`` function:: >>> wheel_metadata = metadata('wheel') # doctest: +SKIP @@ -174,6 +207,13 @@ all the metadata in a JSON-compatible form per :PEP:`566`:: >>> wheel_metadata.json['requires_python'] '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' +.. note:: + + The actual type of the object returned by ``metadata()`` is an + implementation detail and should be accessed only through the interface + described by the + `PackageMetadata protocol `_. + .. versionchanged:: 3.10 The ``Description`` is now included in the metadata when presented through the payload. Line continuation characters have been removed. @@ -187,7 +227,8 @@ all the metadata in a JSON-compatible form per :PEP:`566`:: Distribution versions --------------------- -The ``version()`` function is the quickest way to get a distribution's version +The ``version()`` function is the quickest way to get a +`Distribution Package `_'s version number, as a string:: >>> version('wheel') # doctest: +SKIP @@ -200,7 +241,8 @@ Distribution files ------------------ You can also get the full set of files contained within a distribution. The -``files()`` function takes a distribution package name and returns all of the +``files()`` function takes a `Distribution Package `_ name +and returns all of the files installed by this distribution. Each file object returned is a ``PackagePath``, a :class:`pathlib.PurePath` derived object with additional ``dist``, ``size``, and ``hash`` properties as indicated by the metadata. For example:: @@ -245,19 +287,24 @@ distribution is not known to have the metadata present. Distribution requirements ------------------------- -To get the full set of requirements for a distribution, use the ``requires()`` +To get the full set of requirements for a `Distribution Package `_, +use the ``requires()`` function:: >>> requires('wheel') # doctest: +SKIP ["pytest (>=3.0.0) ; extra == 'test'", "pytest-cov ; extra == 'test'"] -Package distributions ---------------------- +.. _package-distributions: +.. _import-distribution-package-mapping: -A convenience method to resolve the distribution or -distributions (in the case of a namespace package) for top-level -Python packages or modules:: +Mapping import to distribution packages +--------------------------------------- + +A convenience method to resolve the `Distribution Package `_ +name (or names, in the case of a namespace package) +that provide each importable top-level +Python module or `Import Package `_:: >>> packages_distributions() {'importlib_metadata': ['importlib-metadata'], 'yaml': ['PyYAML'], 'jaraco': ['jaraco.classes', 'jaraco.functools'], ...} @@ -271,7 +318,8 @@ Distributions While the above API is the most common and convenient usage, you can get all of that information from the ``Distribution`` class. A ``Distribution`` is an -abstract object that represents the metadata for a Python package. You can +abstract object that represents the metadata for +a Python `Distribution Package `_. You can get the ``Distribution`` instance:: >>> from importlib.metadata import distribution # doctest: +SKIP @@ -291,22 +339,36 @@ instance:: >>> dist.metadata['License'] # doctest: +SKIP 'MIT' -The full set of available metadata is not described here. See :pep:`566` -for additional details. +The full set of available metadata is not described here. +See the `Core metadata specifications `_ for additional details. + + +Distribution Discovery +====================== + +By default, this package provides built-in support for discovery of metadata +for file system and zip file `Distribution Package `_\s. +This metadata finder search defaults to ``sys.path``, but varies slightly in how it interprets those values from how other import machinery does. In particular: + +- ``importlib.metadata`` does not honor :class:`bytes` objects on ``sys.path``. +- ``importlib.metadata`` will incidentally honor :py:class:`pathlib.Path` objects on ``sys.path`` even though such values will be ignored for imports. Extending the search algorithm ============================== -Because package metadata is not available through :data:`sys.path` searches, or -package loaders directly, the metadata for a package is found through import -system :ref:`finders `. To find a distribution package's metadata, +Because `Distribution Package `_ metadata +is not available through :data:`sys.path` searches, or +package loaders directly, +the metadata for a distribution is found through import +system `finders`_. To find a distribution package's metadata, ``importlib.metadata`` queries the list of :term:`meta path finders ` on :data:`sys.meta_path`. -The default ``PathFinder`` for Python includes a hook that calls into -``importlib.metadata.MetadataPathFinder`` for finding distributions -loaded from typical file-system-based paths. +By default ``importlib_metadata`` installs a finder for distribution packages +found on the file system. +This finder doesn't actually find any *distributions*, +but it can find their metadata. The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the interface expected of finders by Python's import system. @@ -335,4 +397,4 @@ a custom finder, return instances of this derived ``Distribution`` in the .. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points .. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api -.. _`importlib_resources`: https://importlib-resources.readthedocs.io/en/latest/index.html +.. _`finders`: https://docs.python.org/3/reference/import.html#finders-and-loaders diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index d281ac032216a8..a028537f3cb2e2 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -139,7 +139,7 @@ :class:`importlib.resources.abc.ResourceReader` and provides concrete implementations of the :class:`importlib.resources.abc.ResourceReader`'s abstract methods. Therefore, any loader supplying - :class:`importlib.abc.TraversableReader` also supplies ResourceReader. + :class:`importlib.abc.TraversableResources` also supplies ResourceReader. Loaders that wish to support resource reading are expected to implement this interface. diff --git a/Doc/library/index.rst b/Doc/library/index.rst index 7d2002b37df12c..d064b680f9aaa4 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -27,8 +27,8 @@ as a collection of packages, so it may be necessary to use the packaging tools provided with the operating system to obtain some or all of the optional components. -In addition to the standard library, there is a growing collection of -several thousand components (from individual programs and modules to +In addition to the standard library, there is an active collection of +hundreds of thousands of components (from individual programs and modules to packages and entire application development frameworks), available from the `Python Package Index `_. diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 44f1ae04c9e39e..9cb7a6f94e49cd 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -32,7 +32,7 @@ The :func:`getmembers` function retrieves the members of an object such as a class or module. The functions whose names begin with "is" are mainly provided as convenient choices for the second argument to :func:`getmembers`. They also help you determine when you can expect to find the following special -attributes: +attributes (see :ref:`import-mod-attrs` for module attributes): .. this function name is too big to fit in the ascii-art table below .. |coroutine-origin-link| replace:: :func:`sys.set_coroutine_origin_tracking_depth` @@ -40,11 +40,6 @@ attributes: +-----------+-------------------+---------------------------+ | Type | Attribute | Description | +===========+===================+===========================+ -| module | __doc__ | documentation string | -+-----------+-------------------+---------------------------+ -| | __file__ | filename (missing for | -| | | built-in modules) | -+-----------+-------------------+---------------------------+ | class | __doc__ | documentation string | +-----------+-------------------+---------------------------+ | | __name__ | name with which this | diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 16b12044a18710..3d93c2c0573e09 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -10,6 +10,10 @@ .. testsetup:: from itertools import * + import collections + import math + import operator + import random -------------- @@ -132,10 +136,9 @@ loops that truncate the stream. There are a number of uses for the *func* argument. It can be set to :func:`min` for a running minimum, :func:`max` for a running maximum, or :func:`operator.mul` for a running product. Amortization tables can be - built by accumulating interest and applying payments. First-order - `recurrence relations `_ - can be modeled by supplying the initial value in the iterable and using only - the accumulated total in *func* argument:: + built by accumulating interest and applying payments: + + .. doctest:: >>> data = [3, 4, 6, 2, 1, 9, 0, 7, 5, 8] >>> list(accumulate(data, operator.mul)) # running product @@ -148,17 +151,6 @@ loops that truncate the stream. >>> list(accumulate(cashflows, lambda bal, pmt: bal*1.05 + pmt)) [1000, 960.0, 918.0, 873.9000000000001, 827.5950000000001] - # Chaotic recurrence relation https://en.wikipedia.org/wiki/Logistic_map - >>> logistic_map = lambda x, _: r * x * (1 - x) - >>> r = 3.8 - >>> x0 = 0.4 - >>> inputs = repeat(x0, 36) # only the initial value is used - >>> [format(x, '.2f') for x in accumulate(inputs, logistic_map)] - ['0.40', '0.91', '0.30', '0.81', '0.60', '0.92', '0.29', '0.79', '0.63', - '0.88', '0.39', '0.90', '0.33', '0.84', '0.52', '0.95', '0.18', '0.57', - '0.93', '0.25', '0.71', '0.79', '0.63', '0.88', '0.39', '0.91', '0.32', - '0.83', '0.54', '0.95', '0.20', '0.60', '0.91', '0.30', '0.80', '0.60'] - See :func:`functools.reduce` for a similar function that returns only the final accumulated value. @@ -202,10 +194,10 @@ loops that truncate the stream. The combination tuples are emitted in lexicographic ordering according to the order of the input *iterable*. So, if the input *iterable* is sorted, - the combination tuples will be produced in sorted order. + the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, there will be no repeat + value. So if the input elements are unique, there will be no repeated values in each combination. Roughly equivalent to:: @@ -251,7 +243,7 @@ loops that truncate the stream. The combination tuples are emitted in lexicographic ordering according to the order of the input *iterable*. So, if the input *iterable* is sorted, - the combination tuples will be produced in sorted order. + the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their value. So if the input elements are unique, the generated combinations @@ -410,14 +402,17 @@ loops that truncate the stream. class groupby: # [k for k, g in groupby('AAAABBBCCDAABBB')] --> A B C D A B # [list(g) for k, g in groupby('AAAABBBCCD')] --> AAAA BBB CC D + def __init__(self, iterable, key=None): if key is None: key = lambda x: x self.keyfunc = key self.it = iter(iterable) self.tgtkey = self.currkey = self.currvalue = object() + def __iter__(self): return self + def __next__(self): self.id = object() while self.currkey == self.tgtkey: @@ -425,6 +420,7 @@ loops that truncate the stream. self.currkey = self.keyfunc(self.currvalue) self.tgtkey = self.currkey return (self.currkey, self._grouper(self.tgtkey, self.id)) + def _grouper(self, tgtkey, id): while self.id is id and self.currkey == tgtkey: yield self.currvalue @@ -443,10 +439,17 @@ loops that truncate the stream. Afterward, elements are returned consecutively unless *step* is set higher than one which results in items being skipped. If *stop* is ``None``, then iteration continues until the iterator is exhausted, if at all; otherwise, it stops at the - specified position. Unlike regular slicing, :func:`islice` does not support - negative values for *start*, *stop*, or *step*. Can be used to extract related - fields from data where the internal structure has been flattened (for example, a - multi-line report may list a name field on every third line). Roughly equivalent to:: + specified position. + + If *start* is ``None``, then iteration starts at zero. If *step* is ``None``, + then the step defaults to one. + + Unlike regular slicing, :func:`islice` does not support negative values for + *start*, *stop*, or *step*. Can be used to extract related fields from + data where the internal structure has been flattened (for example, a + multi-line report may list a name field on every third line). + + Roughly equivalent to:: def islice(iterable, *args): # islice('ABCDEFG', 2) --> A B @@ -473,8 +476,6 @@ loops that truncate the stream. for i, element in zip(range(i + 1, stop), iterable): pass - If *start* is ``None``, then iteration starts at zero. If *step* is ``None``, - then the step defaults to one. .. function:: pairwise(iterable) @@ -503,13 +504,13 @@ loops that truncate the stream. of the *iterable* and all possible full-length permutations are generated. - The permutation tuples are emitted in lexicographic ordering according to + The permutation tuples are emitted in lexicographic order according to the order of the input *iterable*. So, if the input *iterable* is sorted, - the combination tuples will be produced in sorted order. + the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, there will be no repeat - values in each permutation. + value. So if the input elements are unique, there will be no repeated + values within a permutation. Roughly equivalent to:: @@ -589,9 +590,7 @@ loops that truncate the stream. .. function:: repeat(object[, times]) Make an iterator that returns *object* over and over again. Runs indefinitely - unless the *times* argument is specified. Used as argument to :func:`map` for - invariant parameters to the called function. Also used with :func:`zip` to - create an invariant part of a tuple record. + unless the *times* argument is specified. Roughly equivalent to:: @@ -605,7 +604,9 @@ loops that truncate the stream. yield object A common use for *repeat* is to supply a stream of constant values to *map* - or *zip*:: + or *zip*: + + .. doctest:: >>> list(map(pow, range(10), repeat(2))) [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] @@ -614,9 +615,12 @@ loops that truncate the stream. Make an iterator that computes the function using arguments obtained from the iterable. Used instead of :func:`map` when argument parameters are already - grouped in tuples from a single iterable (the data has been "pre-zipped"). The - difference between :func:`map` and :func:`starmap` parallels the distinction - between ``function(a,b)`` and ``function(*c)``. Roughly equivalent to:: + grouped in tuples from a single iterable (when the data has been + "pre-zipped"). + + The difference between :func:`map` and :func:`starmap` parallels the + distinction between ``function(a,b)`` and ``function(*c)``. Roughly + equivalent to:: def starmap(function, iterable): # starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000 @@ -644,9 +648,7 @@ loops that truncate the stream. The following Python code helps explain what *tee* does (although the actual implementation is more complex and uses only a single underlying - :abbr:`FIFO (first-in, first-out)` queue). - - Roughly equivalent to:: + :abbr:`FIFO (first-in, first-out)` queue):: def tee(iterable, n=2): it = iter(iterable) @@ -663,7 +665,7 @@ loops that truncate the stream. yield mydeque.popleft() return tuple(gen(d) for d in deques) - Once :func:`tee` has made a split, the original *iterable* should not be + Once a :func:`tee` has been created, the original *iterable* should not be used anywhere else; otherwise, the *iterable* could get advanced without the tee objects being informed. @@ -717,14 +719,28 @@ Itertools Recipes This section shows recipes for creating an extended toolset using the existing itertools as building blocks. +The primary purpose of the itertools recipes is educational. The recipes show +various ways of thinking about individual tools — for example, that +``chain.from_iterable`` is related to the concept of flattening. The recipes +also give ideas about ways that the tools can be combined — for example, how +``compress()`` and ``range()`` can work together. The recipes also show patterns +for using itertools with the :mod:`operator` and :mod:`collections` modules as +well as with the built-in itertools such as ``map()``, ``filter()``, +``reversed()``, and ``enumerate()``. + +A secondary purpose of the recipes is to serve as an incubator. The +``accumulate()``, ``compress()``, and ``pairwise()`` itertools started out as +recipes. Currently, the ``iter_index()`` recipe is being tested to see +whether it proves its worth. + Substantially all of these recipes and many, many others can be installed from the `more-itertools project `_ found on the Python Package Index:: python -m pip install more-itertools -The extended tools offer the same high performance as the underlying toolset. -The superior memory performance is kept by processing elements one at a time +Many of the recipes offer the same high performance as the underlying toolset. +Superior memory performance is kept by processing elements one at a time rather than bringing the whole iterable into memory all at once. Code volume is kept small by linking the tools together in a functional style which helps eliminate temporary variables. High speed is retained by preferring @@ -809,15 +825,36 @@ which incur interpreter overhead. for k in range(len(roots) + 1) ] + def iter_index(iterable, value, start=0): + "Return indices where a value occurs in a sequence or iterable." + # iter_index('AABCADEAF', 'A') --> 0 1 4 7 + try: + seq_index = iterable.index + except AttributeError: + # Slow path for general iterables + it = islice(iterable, start, None) + for i, element in enumerate(it, start): + if element is value or element == value: + yield i + else: + # Fast path for sequences + i = start - 1 + try: + while True: + yield (i := seq_index(value, i+1)) + except ValueError: + pass + def sieve(n): - "Primes less than n" - # sieve(30) --> 2 3 5 7 11 13 17 19 23 29 - data = bytearray([1]) * n - data[:2] = 0, 0 - limit = math.isqrt(n) + 1 - for p in compress(count(), islice(data, limit)): - data[p+p : n : p] = bytearray(len(range(p+p, n, p))) - return compress(count(), data) + "Primes less than n" + # sieve(30) --> 2 3 5 7 11 13 17 19 23 29 + data = bytearray((0, 1)) * (n // 2) + data[:3] = 0, 0, 0 + limit = math.isqrt(n) + 1 + for p in compress(range(limit), data): + data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) + data[2] = 1 + return iter_index(data, 1) if n > 2 else iter([]) def flatten(list_of_lists): "Flatten one level of nesting" @@ -850,6 +887,8 @@ which incur interpreter overhead. def batched(iterable, n): "Batch data into lists of length n. The last batch may be shorter." # batched('ABCDEFG', 3) --> ABC DEF G + if n < 1: + raise ValueError('n must be at least one') it = iter(iterable) while (batch := list(islice(it, n))): yield batch @@ -935,16 +974,19 @@ which incur interpreter overhead. # unique_everseen('AAAABBBCCDAABBB') --> A B C D # unique_everseen('ABBCcAD', str.lower) --> A B C D seen = set() - seen_add = seen.add if key is None: for element in filterfalse(seen.__contains__, iterable): - seen_add(element) + seen.add(element) yield element + # Note: The steps shown above are intended to demonstrate + # filterfalse(). For order preserving deduplication, + # a better solution is: + # yield from dict.fromkeys(iterable) else: for element in iterable: k = key(element) if k not in seen: - seen_add(k) + seen.add(k) yield element def unique_justseen(iterable, key=None): @@ -989,31 +1031,6 @@ which incur interpreter overhead. # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x return next(filter(pred, iterable), default) - def random_product(*args, repeat=1): - "Random selection from itertools.product(*args, **kwds)" - pools = [tuple(pool) for pool in args] * repeat - return tuple(map(random.choice, pools)) - - def random_permutation(iterable, r=None): - "Random selection from itertools.permutations(iterable, r)" - pool = tuple(iterable) - r = len(pool) if r is None else r - return tuple(random.sample(pool, r)) - - def random_combination(iterable, r): - "Random selection from itertools.combinations(iterable, r)" - pool = tuple(iterable) - n = len(pool) - indices = sorted(random.sample(range(n), r)) - return tuple(pool[i] for i in indices) - - def random_combination_with_replacement(iterable, r): - "Random selection from itertools.combinations_with_replacement(iterable, r)" - pool = tuple(iterable) - n = len(pool) - indices = sorted(random.choices(range(n), k=r)) - return tuple(pool[i] for i in indices) - def nth_combination(iterable, r, index): "Equivalent to list(combinations(iterable, r))[index]" pool = tuple(iterable) @@ -1170,8 +1187,32 @@ which incur interpreter overhead. >>> all(factored(x) == expanded(x) for x in range(-10, 11)) True + >>> list(iter_index('AABCADEAF', 'A')) + [0, 1, 4, 7] + >>> list(iter_index('AABCADEAF', 'B')) + [2] + >>> list(iter_index('AABCADEAF', 'X')) + [] + >>> list(iter_index('', 'X')) + [] + >>> list(iter_index('AABCADEAF', 'A', 1)) + [1, 4, 7] + >>> list(iter_index(iter('AABCADEAF'), 'A', 1)) + [1, 4, 7] + >>> list(iter_index('AABCADEAF', 'A', 2)) + [4, 7] + >>> list(iter_index(iter('AABCADEAF'), 'A', 2)) + [4, 7] + >>> list(iter_index('AABCADEAF', 'A', 10)) + [] + >>> list(iter_index(iter('AABCADEAF'), 'A', 10)) + [] + >>> list(sieve(30)) [2, 3, 5, 7, 11, 13, 17, 19, 23, 29] + >>> small_primes = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97] + >>> all(list(sieve(n)) == [p for p in small_primes if p < n] for n in range(101)) + True >>> len(list(sieve(100))) 25 >>> len(list(sieve(1_000))) @@ -1182,6 +1223,9 @@ which incur interpreter overhead. 9592 >>> len(list(sieve(1_000_000))) 78498 + >>> carmichael = {561, 1105, 1729, 2465, 2821, 6601, 8911} # https://oeis.org/A002997 + >>> set(sieve(10_000)).isdisjoint(carmichael) + True >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'] @@ -1230,12 +1274,6 @@ which incur interpreter overhead. [['A', 'B'], ['C', 'D'], ['E', 'F'], ['G']] >>> list(batched('ABCDEFG', 1)) [['A'], ['B'], ['C'], ['D'], ['E'], ['F'], ['G']] - >>> list(batched('ABCDEFG', 0)) - [] - >>> list(batched('ABCDEFG', -1)) - Traceback (most recent call last): - ... - ValueError: Stop argument for islice() must be None or an integer: 0 <= x <= sys.maxsize. >>> s = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' >>> all(list(flatten(batched(s[:n], 5))) == list(s[:n]) for n in range(len(s))) True diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 6abad5ca5ad430..fd5d41a76a1372 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -147,12 +147,12 @@ The :mod:`locale` module defines the following exception and functions: | ``CHAR_MAX`` | Nothing is specified in this locale. | +--------------+-----------------------------------------+ - The function sets temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` + The function temporarily sets the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale or the ``LC_MONETARY`` locale if locales are different and numeric or monetary strings are non-ASCII. This temporary change affects other threads. .. versionchanged:: 3.7 - The function now sets temporarily the ``LC_CTYPE`` locale to the + The function now temporarily sets the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale in some cases. @@ -227,16 +227,18 @@ The :mod:`locale` module defines the following exception and functions: Get a regular expression that can be used with the regex function to recognize a positive response to a yes/no question. - .. note:: - - The expression is in the syntax suitable for the :c:func:`regex` function - from the C library, which might differ from the syntax used in :mod:`re`. - .. data:: NOEXPR Get a regular expression that can be used with the regex(3) function to recognize a negative response to a yes/no question. + .. note:: + + The regular expressions for :const:`YESEXPR` and + :const:`NOEXPR` use syntax suitable for the + :c:func:`regex` function from the C library, which might + differ from the syntax used in :mod:`re`. + .. data:: CRNCYSTR Get the currency symbol, preceded by "-" if the symbol should appear before @@ -399,7 +401,7 @@ The :mod:`locale` module defines the following exception and functions: Formats a number *val* according to the current :const:`LC_NUMERIC` setting. The format follows the conventions of the ``%`` operator. For floating point - values, the decimal point is modified if appropriate. If *grouping* is true, + values, the decimal point is modified if appropriate. If *grouping* is ``True``, also takes the grouping into account. If *monetary* is true, the conversion uses monetary thousands separator and @@ -429,12 +431,14 @@ The :mod:`locale` module defines the following exception and functions: Formats a number *val* according to the current :const:`LC_MONETARY` settings. The returned string includes the currency symbol if *symbol* is true, which is - the default. If *grouping* is true (which is not the default), grouping is done - with the value. If *international* is true (which is not the default), the + the default. If *grouping* is ``True`` (which is not the default), grouping is done + with the value. If *international* is ``True`` (which is not the default), the international currency symbol is used. - Note that this function will not work with the 'C' locale, so you have to set a - locale via :func:`setlocale` first. + .. note:: + + This function will not work with the 'C' locale, so you have to set a + locale via :func:`setlocale` first. .. function:: str(float) @@ -621,4 +625,3 @@ applications that link with additional C libraries which internally invoke :c:func:`gettext` or :c:func:`dcgettext`. For these applications, it may be necessary to bind the text domain, so that the libraries can properly locate their message catalogs. - diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 8ab76ab93b60fb..b6455298278743 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -650,6 +650,17 @@ supports sending logging messages to a remote or local Unix syslog. Closes the socket to the remote host. + .. method:: createSocket() + + Tries to create a socket and, if it's not a datagram socket, connect it + to the other end. This method is called during handler initialization, + but it's not regarded as an error if the other end isn't listening at + this point - the method will be called again when emitting an event, if + but it's not regarded as an error if the other end isn't listening yet + --- the method will be called again when emitting an event, + if there is no socket at that point. + + .. versionadded:: 3.11 .. method:: emit(record) diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index ff3da416a38157..56908dedea1b40 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -614,7 +614,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. seealso:: - `nmh - Message Handling System `_ + `nmh - Message Handling System `_ Home page of :program:`nmh`, an updated version of the original :program:`mh`. `MH & nmh: Email for Users & Programmers `_ diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 5516084780673f..b5ceeb796f8f2f 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1089,10 +1089,14 @@ Miscellaneous .. versionchanged:: 3.11 Accepts a :term:`path-like object`. -.. function:: set_start_method(method) +.. function:: set_start_method(method, force=False) Set the method which should be used to start child processes. - *method* can be ``'fork'``, ``'spawn'`` or ``'forkserver'``. + The *method* argument can be ``'fork'``, ``'spawn'`` or ``'forkserver'``. + Raises :exc:`RuntimeError` if the start method has already been set and *force* + is not ``True``. If *method* is ``None`` and *force* is ``True`` then the start + method is set to ``None``. If *method* is ``None`` and *force* is ``False`` + then the context is set to the default context. Note that this should be called at most once, and it should be protected inside the ``if __name__ == '__main__'`` clause of the diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 127a82d47aa195..76046b34610abe 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -125,7 +125,7 @@ instances:: The following example demonstrates a practical use of the :class:`SharedMemory` -class with `NumPy arrays `_, accessing the +class with `NumPy arrays `_, accessing the same ``numpy.ndarray`` from two distinct Python shells: .. doctest:: diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 633cd0869d8311..d119e8dc5e84fe 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2326,7 +2326,7 @@ features: .. function:: remove(path, *, dir_fd=None) Remove (delete) the file *path*. If *path* is a directory, an - :exc:`IsADirectoryError` is raised. Use :func:`rmdir` to remove directories. + :exc:`OSError` is raised. Use :func:`rmdir` to remove directories. If the file does not exist, a :exc:`FileNotFoundError` is raised. This function can support :ref:`paths relative to directory descriptors @@ -3208,7 +3208,8 @@ features: filenames)``. *dirpath* is a string, the path to the directory. *dirnames* is a list of the - names of the subdirectories in *dirpath* (excluding ``'.'`` and ``'..'``). + names of the subdirectories in *dirpath* (including symlinks to directories, + and excluding ``'.'`` and ``'..'``). *filenames* is a list of the names of the non-directory files in *dirpath*. Note that the names in the lists contain no path components. To get a full path (which begins with *top*) to a file or directory in *dirpath*, do @@ -4375,6 +4376,9 @@ written in Python, such as a mail server's external command delivery program. number is zero); the high bit of the low byte is set if a core file was produced. + If there are no children that could be waited for, :exc:`ChildProcessError` + is raised. + :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exit code. @@ -4382,76 +4386,40 @@ written in Python, such as a mail server's external command delivery program. .. seealso:: - :func:`waitpid` can be used to wait for the completion of a specific - child process and has more options. + The other :func:`!wait*` functions documented below can be used to wait for the + completion of a specific child process and have more options. + :func:`waitpid` is the only one also available on Windows. -.. function:: waitid(idtype, id, options, /) - Wait for the completion of one or more child processes. - *idtype* can be :data:`P_PID`, :data:`P_PGID`, :data:`P_ALL`, or - :data:`P_PIDFD` on Linux. - *id* specifies the pid to wait on. - *options* is constructed from the ORing of one or more of :data:`WEXITED`, - :data:`WSTOPPED` or :data:`WCONTINUED` and additionally may be ORed with - :data:`WNOHANG` or :data:`WNOWAIT`. The return value is an object - representing the data contained in the :c:type:`siginfo_t` structure, namely: - :attr:`si_pid`, :attr:`si_uid`, :attr:`si_signo`, :attr:`si_status`, - :attr:`si_code` or ``None`` if :data:`WNOHANG` is specified and there are no - children in a waitable state. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - -.. data:: P_PID - P_PGID - P_ALL - - These are the possible values for *idtype* in :func:`waitid`. They affect - how *id* is interpreted. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - -.. data:: P_PIDFD - - This is a Linux-specific *idtype* that indicates that *id* is a file - descriptor that refers to a process. - - .. availability:: Linux >= 5.4 - - .. versionadded:: 3.9 - -.. data:: WEXITED - WSTOPPED - WNOWAIT +.. function:: waitid(idtype, id, options, /) - Flags that can be used in *options* in :func:`waitid` that specify what - child signal to wait for. + Wait for the completion of a child process. - .. availability:: Unix, not Emscripten, not WASI. + *idtype* can be :data:`P_PID`, :data:`P_PGID`, :data:`P_ALL`, or (on Linux) :data:`P_PIDFD`. + The interpretation of *id* depends on it; see their individual descriptions. - .. versionadded:: 3.3 + *options* is an OR combination of flags. At least one of :data:`WEXITED`, + :data:`WSTOPPED` or :data:`WCONTINUED` is required; + :data:`WNOHANG` and :data:`WNOWAIT` are additional optional flags. + The return value is an object representing the data contained in the + :c:type:`!siginfo_t` structure with the following attributes: -.. data:: CLD_EXITED - CLD_KILLED - CLD_DUMPED - CLD_TRAPPED - CLD_STOPPED - CLD_CONTINUED + * :attr:`!si_pid` (process ID) + * :attr:`!si_uid` (real user ID of the child) + * :attr:`!si_signo` (always :data:`~signal.SIGCHLD`) + * :attr:`!si_status` (the exit status or signal number, depending on :attr:`!si_code`) + * :attr:`!si_code` (see :data:`CLD_EXITED` for possible values) - These are the possible values for :attr:`si_code` in the result returned by - :func:`waitid`. + If :data:`WNOHANG` is specified and there are no matching children in the + requested state, ``None`` is returned. + Otherwise, if there are no matching children + that could be waited for, :exc:`ChildProcessError` is raised. .. availability:: Unix, not Emscripten, not WASI. .. versionadded:: 3.3 - .. versionchanged:: 3.9 - Added :data:`CLD_KILLED` and :data:`CLD_STOPPED` values. - .. function:: waitpid(pid, options, /) @@ -4469,8 +4437,11 @@ written in Python, such as a mail server's external command delivery program. ``-1``, status is requested for any process in the process group ``-pid`` (the absolute value of *pid*). - An :exc:`OSError` is raised with the value of errno when the syscall - returns -1. + *options* is an OR combination of flags. If it contains :data:`WNOHANG` and + there are no matching children in the requested state, ``(0, 0)`` is + returned. Otherwise, if there are no matching children that could be waited + for, :exc:`ChildProcessError` is raised. Other options that can be used are + :data:`WUNTRACED` and :data:`WCONTINUED`. On Windows: Wait for completion of a process given by process handle *pid*, and return a tuple containing *pid*, and its exit status shifted left by 8 bits @@ -4483,7 +4454,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exit code. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, Windows, not Emscripten, not WASI. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -4496,9 +4467,9 @@ written in Python, such as a mail server's external command delivery program. Similar to :func:`waitpid`, except no process id argument is given and a 3-element tuple containing the child's process id, exit status indication, and resource usage information is returned. Refer to - :mod:`resource`.\ :func:`~resource.getrusage` for details on resource usage - information. The option argument is the same as that provided to - :func:`waitpid` and :func:`wait4`. + :func:`resource.getrusage` for details on resource usage information. The + *options* argument is the same as that provided to :func:`waitpid` and + :func:`wait4`. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exitcode. @@ -4509,10 +4480,10 @@ written in Python, such as a mail server's external command delivery program. .. function:: wait4(pid, options) Similar to :func:`waitpid`, except a 3-element tuple, containing the child's - process id, exit status indication, and resource usage information is returned. - Refer to :mod:`resource`.\ :func:`~resource.getrusage` for details on - resource usage information. The arguments to :func:`wait4` are the same - as those provided to :func:`waitpid`. + process id, exit status indication, and resource usage information is + returned. Refer to :func:`resource.getrusage` for details on resource usage + information. The arguments to :func:`wait4` are the same as those provided + to :func:`waitpid`. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exitcode. @@ -4520,6 +4491,111 @@ written in Python, such as a mail server's external command delivery program. .. availability:: Unix, not Emscripten, not WASI. +.. data:: P_PID + P_PGID + P_ALL + P_PIDFD + + These are the possible values for *idtype* in :func:`waitid`. They affect + how *id* is interpreted: + + * :data:`!P_PID` - wait for the child whose PID is *id*. + * :data:`!P_PGID` - wait for any child whose progress group ID is *id*. + * :data:`!P_ALL` - wait for any child; *id* is ignored. + * :data:`!P_PIDFD` - wait for the child identified by the file descriptor + *id* (a process file descriptor created with :func:`pidfd_open`). + + .. availability:: Unix, not Emscripten, not WASI. + + .. note:: :data:`!P_PIDFD` is only available on Linux >= 5.4. + + .. versionadded:: 3.3 + .. versionadded:: 3.9 + The :data:`!P_PIDFD` constant. + + +.. data:: WCONTINUED + + This *options* flag for :func:`waitpid`, :func:`wait3`, :func:`wait4`, and + :func:`waitid` causes child processes to be reported if they have been + continued from a job control stop since they were last reported. + + .. availability:: Unix, not Emscripten, not WASI. + + +.. data:: WEXITED + + This *options* flag for :func:`waitid` causes child processes that have terminated to + be reported. + + The other ``wait*`` functions always report children that have terminated, + so this option is not available for them. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.3 + + +.. data:: WSTOPPED + + This *options* flag for :func:`waitid` causes child processes that have been stopped + by the delivery of a signal to be reported. + + This option is not available for the other ``wait*`` functions. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.3 + + +.. data:: WUNTRACED + + This *options* flag for :func:`waitpid`, :func:`wait3`, and :func:`wait4` causes + child processes to also be reported if they have been stopped but their + current state has not been reported since they were stopped. + + This option is not available for :func:`waitid`. + + .. availability:: Unix, not Emscripten, not WASI. + + +.. data:: WNOHANG + + This *options* flag causes :func:`waitpid`, :func:`wait3`, :func:`wait4`, and + :func:`waitid` to return right away if no child process status is available + immediately. + + .. availability:: Unix, not Emscripten, not WASI. + + +.. data:: WNOWAIT + + This *options* flag causes :func:`waitid` to leave the child in a waitable state, so that + a later :func:`!wait*` call can be used to retrieve the child status information again. + + This option is not available for the other ``wait*`` functions. + + .. availability:: Unix, not Emscripten, not WASI. + + +.. data:: CLD_EXITED + CLD_KILLED + CLD_DUMPED + CLD_TRAPPED + CLD_STOPPED + CLD_CONTINUED + + These are the possible values for :attr:`!si_code` in the result returned by + :func:`waitid`. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.3 + + .. versionchanged:: 3.9 + Added :data:`CLD_KILLED` and :data:`CLD_STOPPED` values. + + .. function:: waitstatus_to_exitcode(status) Convert a wait status to an exit code. @@ -4552,32 +4628,6 @@ written in Python, such as a mail server's external command delivery program. .. versionadded:: 3.9 -.. data:: WNOHANG - - The option for :func:`waitpid` to return immediately if no child process status - is available immediately. The function returns ``(0, 0)`` in this case. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: WCONTINUED - - This option causes child processes to be reported if they have been continued - from a job control stop since their status was last reported. - - .. availability:: Unix, not Emscripten, not WASI. - - Some Unix systems. - - -.. data:: WUNTRACED - - This option causes child processes to be reported if they have been stopped but - their current state has not been reported since they were stopped. - - .. availability:: Unix, not Emscripten, not WASI. - - The following functions take a process status code as returned by :func:`system`, :func:`wait`, or :func:`waitpid` as a parameter. They may be used to determine the disposition of a process. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 5012150b86cefa..843513c5fc54ab 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -391,7 +391,7 @@ Pure paths provide the following methods and properties: If you want to walk an arbitrary filesystem path upwards, it is recommended to first call :meth:`Path.resolve` so as to resolve - symlinks and eliminate `".."` components. + symlinks and eliminate ``".."`` components. .. data:: PurePath.name diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 41b0f48f4611da..79476b04cd914d 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -90,7 +90,7 @@ Comparison with ``json`` ^^^^^^^^^^^^^^^^^^^^^^^^ There are fundamental differences between the pickle protocols and -`JSON (JavaScript Object Notation) `_: +`JSON (JavaScript Object Notation) `_: * JSON is a text serialization format (it outputs unicode text, although most of the time it is then encoded to ``utf-8``), while pickle is diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 661f7c95483c5c..2b87a36f7c5200 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -548,7 +548,7 @@ Simulation of arrival times and service deliveries for a multiserver queue:: including simulation, sampling, shuffling, and cross-validation. `Economics Simulation - `_ + `_ a simulation of a marketplace by `Peter Norvig `_ that shows effective use of many of the tools and distributions provided by this module @@ -564,6 +564,37 @@ Simulation of arrival times and service deliveries for a multiserver queue:: Recipes ------- +These recipes show how to efficiently make random selections +from the combinatoric iterators in the :mod:`itertools` module: + +.. testcode:: + import random + + def random_product(*args, repeat=1): + "Random selection from itertools.product(*args, **kwds)" + pools = [tuple(pool) for pool in args] * repeat + return tuple(map(random.choice, pools)) + + def random_permutation(iterable, r=None): + "Random selection from itertools.permutations(iterable, r)" + pool = tuple(iterable) + r = len(pool) if r is None else r + return tuple(random.sample(pool, r)) + + def random_combination(iterable, r): + "Random selection from itertools.combinations(iterable, r)" + pool = tuple(iterable) + n = len(pool) + indices = sorted(random.sample(range(n), r)) + return tuple(pool[i] for i in indices) + + def random_combination_with_replacement(iterable, r): + "Random selection from itertools.combinations_with_replacement(iterable, r)" + pool = tuple(iterable) + n = len(pool) + indices = sorted(random.choices(range(n), k=r)) + return tuple(pool[i] for i in indices) + The default :func:`.random` returns multiples of 2⁻⁵³ in the range *0.0 ≤ x < 1.0*. All such numbers are evenly spaced and are exactly representable as Python floats. However, many other representable diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 3a6e2e7f89081c..8e279049a1c2d2 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -481,6 +481,9 @@ The special characters are: some fixed length. Patterns which start with negative lookbehind assertions may match at the beginning of the string being searched. +.. _re-conditional-expression: +.. index:: single: (?(; in regular expressions + ``(?(id/name)yes-pattern|no-pattern)`` Will try to match with ``yes-pattern`` if the group with given *id* or *name* exists, and with ``no-pattern`` if it doesn't. ``no-pattern`` is @@ -1559,16 +1562,22 @@ search() vs. match() .. sectionauthor:: Fred L. Drake, Jr. -Python offers two different primitive operations based on regular expressions: -:func:`re.match` checks for a match only at the beginning of the string, while -:func:`re.search` checks for a match anywhere in the string (this is what Perl -does by default). +Python offers different primitive operations based on regular expressions: + ++ :func:`re.match` checks for a match only at the beginning of the string ++ :func:`re.search` checks for a match anywhere in the string + (this is what Perl does by default) ++ :func:`re.fullmatch` checks for entire string to be a match + For example:: >>> re.match("c", "abcdef") # No match >>> re.search("c", "abcdef") # Match + >>> re.fullmatch("p.*n", "python") # Match + + >>> re.fullmatch("r.*n", "python") # No match Regular expressions beginning with ``'^'`` can be used with :func:`search` to restrict the match at the beginning of the string:: @@ -1582,8 +1591,8 @@ Note however that in :const:`MULTILINE` mode :func:`match` only matches at the beginning of the string, whereas using :func:`search` with a regular expression beginning with ``'^'`` will match at the beginning of each line. :: - >>> re.match('X', 'A\nB\nX', re.MULTILINE) # No match - >>> re.search('^X', 'A\nB\nX', re.MULTILINE) # Match + >>> re.match("X", "A\nB\nX", re.MULTILINE) # No match + >>> re.search("^X", "A\nB\nX", re.MULTILINE) # Match diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index dc8e5f46fb581e..4405dfc0535973 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -128,7 +128,9 @@ Other functions .. function:: compare_digest(a, b) - Return ``True`` if strings *a* and *b* are equal, otherwise ``False``, + Return ``True`` if strings or + :term:`bytes-like objects ` + *a* and *b* are equal, otherwise ``False``, using a "constant-time compare" to reduce the risk of `timing attacks `_. See :func:`hmac.compare_digest` for additional details. diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 2269f50cbaff99..523d1ac5001360 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -4,6 +4,8 @@ .. module:: signal :synopsis: Set handlers for asynchronous events. +**Source code:** :source:`Lib/signal.py` + -------------- This module provides mechanisms to use signal handlers in Python. @@ -362,9 +364,9 @@ The :mod:`signal` module defines the following functions: .. function:: strsignal(signalnum) - Return the system description of the signal *signalnum*, such as - "Interrupt", "Segmentation fault", etc. Returns :const:`None` if the signal - is not recognized. + Returns the description of signal *signalnum*, such as "Interrupt" + for :const:`SIGINT`. Returns :const:`None` if *signalnum* has no + description. Raises :exc:`ValueError` if *signalnum* is invalid. .. versionadded:: 3.8 diff --git a/Doc/library/sndhdr.rst b/Doc/library/sndhdr.rst index e1dbe4a1a34483..fa9323e18dc348 100644 --- a/Doc/library/sndhdr.rst +++ b/Doc/library/sndhdr.rst @@ -54,3 +54,51 @@ be the sample size in bits or ``'A'`` for A-LAW or ``'U'`` for u-LAW. .. versionchanged:: 3.5 Result changed from a tuple to a namedtuple. +The following sound header types are recognized, as listed below with the return value +from :func:`whathdr`: and :func:`what`: + ++------------+------------------------------------+ +| Value | Sound header format | ++============+====================================+ +| ``'aifc'`` | Compressed Audio Interchange Files | ++------------+------------------------------------+ +| ``'aiff'`` | Audio Interchange Files | ++------------+------------------------------------+ +| ``'au'`` | Au Files | ++------------+------------------------------------+ +| ``'hcom'`` | HCOM Files | ++------------+------------------------------------+ +| ``'sndt'`` | Sndtool Sound Files | ++------------+------------------------------------+ +| ``'voc'`` | Creative Labs Audio Files | ++------------+------------------------------------+ +| ``'wav'`` | Waveform Audio File Format Files | ++------------+------------------------------------+ +| ``'8svx'`` | 8-Bit Sampled Voice Files | ++------------+------------------------------------+ +| ``'sb'`` | Signed Byte Audio Data Files | ++------------+------------------------------------+ +| ``'ub'`` | UB Files | ++------------+------------------------------------+ +| ``'ul'`` | uLAW Audio Files | ++------------+------------------------------------+ + +.. data:: tests + + A list of functions performing the individual tests. Each function takes two + arguments: the byte-stream and an open file-like object. When :func:`what` is + called with a byte-stream, the file-like object will be ``None``. + + The test function should return a string describing the image type if the test + succeeded, or ``None`` if it failed. + +Example: + +.. code-block:: pycon + + >>> import sndhdr + >>> imghdr.what('bass.wav') + 'wav' + >>> imghdr.whathdr('bass.wav') + 'wav' + diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 70d56a1dad94f2..a409c99b1f6a45 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -96,8 +96,7 @@ synchronous servers of four types:: Note that :class:`UnixDatagramServer` derives from :class:`UDPServer`, not from :class:`UnixStreamServer` --- the only difference between an IP and a Unix -stream server is the address family, which is simply repeated in both Unix -server classes. +server is the address family. .. class:: ForkingMixIn @@ -432,11 +431,8 @@ Request Handler Objects The :attr:`self.rfile` and :attr:`self.wfile` attributes can be read or written, respectively, to get the request data or return data to the client. - - The :attr:`rfile` attributes of both classes support the - :class:`io.BufferedIOBase` readable interface, and - :attr:`DatagramRequestHandler.wfile` supports the - :class:`io.BufferedIOBase` writable interface. + The :attr:`!rfile` attributes support the :class:`io.BufferedIOBase` readable interface, + and :attr:`!wfile` attributes support the :class:`!io.BufferedIOBase` writable interface. .. versionchanged:: 3.6 :attr:`StreamRequestHandler.wfile` also supports the diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 8fdb75349540c9..b720ac995c446b 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -239,6 +239,7 @@ inserted data and retrieved values from it in multiple ways. * :ref:`sqlite3-adapters` * :ref:`sqlite3-converters` * :ref:`sqlite3-connection-context-manager` + * :ref:`sqlite3-howto-row-factory` * :ref:`sqlite3-explanation` for in-depth background on transaction control. @@ -475,9 +476,10 @@ Module constants .. note:: - The :mod:`!sqlite3` module supports both ``qmark`` and ``numeric`` DB-API - parameter styles, because that is what the underlying SQLite library - supports. However, the DB-API does not allow multiple values for + The :mod:`!sqlite3` module supports ``qmark``, ``numeric``, + and ``named`` DB-API parameter styles, + because that is what the underlying SQLite library supports. + However, the DB-API does not allow multiple values for the ``paramstyle`` attribute. .. data:: sqlite_version @@ -564,7 +566,7 @@ Connection objects supplied, this must be a callable returning an instance of :class:`Cursor` or its subclasses. - .. method:: blobopen(table, column, row, /, \*, readonly=False, name="main") + .. method:: blobopen(table, column, row, /, *, readonly=False, name="main") Open a :class:`Blob` handle to an existing :abbr:`BLOB (Binary Large OBject)`. @@ -634,7 +636,7 @@ Connection objects :meth:`~Cursor.executescript` on it with the given *sql_script*. Return the new cursor object. - .. method:: create_function(name, narg, func, \*, deterministic=False) + .. method:: create_function(name, narg, func, *, deterministic=False) Create or remove a user-defined SQL function. @@ -1027,7 +1029,7 @@ Connection objects con.close() - .. method:: backup(target, \*, pages=-1, progress=None, name="main", sleep=0.250) + .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250) Create a backup of an SQLite database. @@ -1156,7 +1158,7 @@ Connection objects .. _SQLite limit category: https://www.sqlite.org/c3ref/c_limit_attached.html - .. method:: serialize(\*, name="main") + .. method:: serialize(*, name="main") Serialize a database into a :class:`bytes` object. For an ordinary on-disk database file, the serialization is just a copy of the @@ -1178,7 +1180,7 @@ Connection objects .. versionadded:: 3.11 - .. method:: deserialize(data, /, \*, name="main") + .. method:: deserialize(data, /, *, name="main") Deserialize a :meth:`serialized ` database into a :class:`Connection`. @@ -1235,31 +1237,14 @@ Connection objects .. attribute:: row_factory - A callable that accepts two arguments, - a :class:`Cursor` object and the raw row results as a :class:`tuple`, - and returns a custom object representing an SQLite row. - - Example: - - .. doctest:: - - >>> def dict_factory(cursor, row): - ... col_names = [col[0] for col in cursor.description] - ... return {key: value for key, value in zip(col_names, row)} - >>> con = sqlite3.connect(":memory:") - >>> con.row_factory = dict_factory - >>> for row in con.execute("SELECT 1 AS a, 2 AS b"): - ... print(row) - {'a': 1, 'b': 2} + The initial :attr:`~Cursor.row_factory` + for :class:`Cursor` objects created from this connection. + Assigning to this attribute does not affect the :attr:`!row_factory` + of existing cursors belonging to this connection, only new ones. + Is ``None`` by default, + meaning each row is returned as a :class:`tuple`. - If returning a tuple doesn't suffice and you want name-based access to - columns, you should consider setting :attr:`row_factory` to the - highly optimized :class:`sqlite3.Row` type. :class:`Row` provides both - index-based and case-insensitive name-based access to columns with almost no - memory overhead. It will probably be better than your own custom - dictionary-based approach or even a db_row based solution. - - .. XXX what's a db_row-based solution? + See :ref:`sqlite3-howto-row-factory` for more details. .. attribute:: text_factory @@ -1412,7 +1397,7 @@ Cursor objects .. method:: fetchone() - If :attr:`~Connection.row_factory` is ``None``, + If :attr:`~Cursor.row_factory` is ``None``, return the next row query result set as a :class:`tuple`. Else, pass it to the row factory and return its result. Return ``None`` if no more data is available. @@ -1506,6 +1491,22 @@ Cursor objects including :abbr:`CTE (Common Table Expression)` queries. It is only updated by the :meth:`execute` and :meth:`executemany` methods. + .. attribute:: row_factory + + Control how a row fetched from this :class:`!Cursor` is represented. + If ``None``, a row is represented as a :class:`tuple`. + Can be set to the included :class:`sqlite3.Row`; + or a :term:`callable` that accepts two arguments, + a :class:`Cursor` object and the :class:`!tuple` of row values, + and returns a custom object representing an SQLite row. + + Defaults to what :attr:`Connection.row_factory` was set to + when the :class:`!Cursor` was created. + Assigning to this attribute does not affect + :attr:`Connection.row_factory` of the parent connection. + + See :ref:`sqlite3-howto-row-factory` for more details. + .. The sqlite3.Row example used to be a how-to. It has now been incorporated into the Row reference. We keep the anchor here in order not to break @@ -1524,7 +1525,10 @@ Row objects It supports iteration, equality testing, :func:`len`, and :term:`mapping` access by column name and index. - Two row objects compare equal if have equal columns and equal members. + Two :class:`!Row` objects compare equal + if they have identical column names and values. + + See :ref:`sqlite3-howto-row-factory` for more details. .. method:: keys @@ -1535,21 +1539,6 @@ Row objects .. versionchanged:: 3.5 Added support of slicing. - Example: - - .. doctest:: - - >>> con = sqlite3.connect(":memory:") - >>> con.row_factory = sqlite3.Row - >>> res = con.execute("SELECT 'Earth' AS name, 6378 AS radius") - >>> row = res.fetchone() - >>> row.keys() - ['name', 'radius'] - >>> row[0], row["name"] # Access by index and name. - ('Earth', 'Earth') - >>> row["RADIUS"] # Column names are case-insensitive. - 6378 - .. _sqlite3-blob-objects: @@ -2002,7 +1991,7 @@ The following example illustrates the implicit and explicit approaches: return f"Point({self.x}, {self.y})" def adapt_point(point): - return f"{point.x};{point.y}".encode("utf-8") + return f"{point.x};{point.y}" def convert_point(s): x, y = list(map(float, s.split(b";"))) @@ -2068,20 +2057,39 @@ This section shows recipes for common adapters and converters. def convert_date(val): """Convert ISO 8601 date to datetime.date object.""" - return datetime.date.fromisoformat(val) + return datetime.date.fromisoformat(val.decode()) def convert_datetime(val): """Convert ISO 8601 datetime to datetime.datetime object.""" - return datetime.datetime.fromisoformat(val) + return datetime.datetime.fromisoformat(val.decode()) def convert_timestamp(val): """Convert Unix epoch timestamp to datetime.datetime object.""" - return datetime.datetime.fromtimestamp(val) + return datetime.datetime.fromtimestamp(int(val)) sqlite3.register_converter("date", convert_date) sqlite3.register_converter("datetime", convert_datetime) sqlite3.register_converter("timestamp", convert_timestamp) +.. testcode:: + :hide: + + dt = datetime.datetime(2019, 5, 18, 15, 17, 8, 123456) + + assert adapt_date_iso(dt.date()) == "2019-05-18" + assert convert_date(b"2019-05-18") == dt.date() + + assert adapt_datetime_iso(dt) == "2019-05-18T15:17:08.123456" + assert convert_datetime(b"2019-05-18T15:17:08.123456") == dt + + # Using current time as fromtimestamp() returns local date/time. + # Droping microseconds as adapt_datetime_epoch truncates fractional second part. + now = datetime.datetime.now().replace(microsecond=0) + current_timestamp = int(now.timestamp()) + + assert adapt_datetime_epoch(now) == current_timestamp + assert convert_timestamp(str(current_timestamp).encode()) == now + .. _sqlite3-connection-shortcuts: @@ -2221,6 +2229,96 @@ can be found in the `SQLite URI documentation`_. .. _SQLite URI documentation: https://www.sqlite.org/uri.html +.. _sqlite3-howto-row-factory: + +How to create and use row factories +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, :mod:`!sqlite3` represents each row as a :class:`tuple`. +If a :class:`!tuple` does not suit your needs, +you can use the :class:`sqlite3.Row` class +or a custom :attr:`~Cursor.row_factory`. + +While :attr:`!row_factory` exists as an attribute both on the +:class:`Cursor` and the :class:`Connection`, +it is recommended to set :class:`Connection.row_factory`, +so all cursors created from the connection will use the same row factory. + +:class:`!Row` provides indexed and case-insensitive named access to columns, +with minimal memory overhead and performance impact over a :class:`!tuple`. +To use :class:`!Row` as a row factory, +assign it to the :attr:`!row_factory` attribute: + +.. doctest:: + + >>> con = sqlite3.connect(":memory:") + >>> con.row_factory = sqlite3.Row + +Queries now return :class:`!Row` objects: + +.. doctest:: + + >>> res = con.execute("SELECT 'Earth' AS name, 6378 AS radius") + >>> row = res.fetchone() + >>> row.keys() + ['name', 'radius'] + >>> row[0] # Access by index. + 'Earth' + >>> row["name"] # Access by name. + 'Earth' + >>> row["RADIUS"] # Column names are case-insensitive. + 6378 + +You can create a custom :attr:`~Cursor.row_factory` +that returns each row as a :class:`dict`, with column names mapped to values: + +.. testcode:: + + def dict_factory(cursor, row): + fields = [column[0] for column in cursor.description] + return {key: value for key, value in zip(fields, row)} + +Using it, queries now return a :class:`!dict` instead of a :class:`!tuple`: + +.. doctest:: + + >>> con = sqlite3.connect(":memory:") + >>> con.row_factory = dict_factory + >>> for row in con.execute("SELECT 1 AS a, 2 AS b"): + ... print(row) + {'a': 1, 'b': 2} + +The following row factory returns a :term:`named tuple`: + +.. testcode:: + + from collections import namedtuple + + def namedtuple_factory(cursor, row): + fields = [column[0] for column in cursor.description] + cls = namedtuple("Row", fields) + return cls._make(row) + +:func:`!namedtuple_factory` can be used as follows: + +.. doctest:: + + >>> con = sqlite3.connect(":memory:") + >>> con.row_factory = namedtuple_factory + >>> cur = con.execute("SELECT 1 AS a, 2 AS b") + >>> row = cur.fetchone() + >>> row + Row(a=1, b=2) + >>> row[0] # Indexed access. + 1 + >>> row.b # Attribute access. + 2 + +With some adjustments, the above recipe can be adapted to use a +:class:`~dataclasses.dataclass`, or any other custom class, +instead of a :class:`~collections.namedtuple`. + + .. _sqlite3-explanation: Explanation diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 14d2a27a87bac9..75550d6e469831 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1617,6 +1617,9 @@ expression support in the :mod:`re` module). range [*start*, *end*]. Optional arguments *start* and *end* are interpreted as in slice notation. + If *sub* is empty, returns the number of empty strings between characters + which is the length of the string plus one. + .. method:: str.encode(encoding="utf-8", errors="strict") @@ -2698,6 +2701,9 @@ arbitrary binary data. The subsequence to search for may be any :term:`bytes-like object` or an integer in the range 0 to 255. + If *sub* is empty, returns the number of empty slices between characters + which is the length of the bytes object plus one. + .. versionchanged:: 3.3 Also accept an integer in the range 0 to 255 as the subsequence. @@ -4370,11 +4376,9 @@ type, the :dfn:`dictionary`. (For other containers see the built-in A dictionary's keys are *almost* arbitrary values. Values that are not :term:`hashable`, that is, values containing lists, dictionaries or other mutable types (that are compared by value rather than by object identity) may -not be used as keys. Numeric types used for keys obey the normal rules for -numeric comparison: if two numbers compare equal (such as ``1`` and ``1.0``) -then they can be used interchangeably to index the same dictionary entry. (Note -however, that since computers store floating-point numbers as approximations it -is usually unwise to use them as dictionary keys.) +not be used as keys. +Values that compare equal (such as ``1``, ``1.0``, and ``True``) +can be used interchangeably to index the same dictionary entry. .. class:: dict(**kwargs) dict(mapping, **kwargs) diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index d12a5732fa4a0d..50d70731f77523 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -12,21 +12,25 @@ -------------- -This module performs conversions between Python values and C structs represented -as Python :class:`bytes` objects. This can be used in handling binary data -stored in files or from network connections, among other sources. It uses -:ref:`struct-format-strings` as compact descriptions of the layout of the C -structs and the intended conversion to/from Python values. +This module converts between Python values and C structs represented +as Python :class:`bytes` objects. Compact :ref:`format strings ` +describe the intended conversions to/from Python values. +The module's functions and objects can be used for two largely +distinct applications, data exchange with external sources (files or +network connections), or data transfer between the Python application +and the C layer. .. note:: - By default, the result of packing a given C struct includes pad bytes in - order to maintain proper alignment for the C types involved; similarly, - alignment is taken into account when unpacking. This behavior is chosen so - that the bytes of a packed struct correspond exactly to the layout in memory - of the corresponding C struct. To handle platform-independent data formats - or omit implicit pad bytes, use ``standard`` size and alignment instead of - ``native`` size and alignment: see :ref:`struct-alignment` for details. + When no prefix character is given, native mode is the default. It + packs or unpacks data based on the platform and compiler on which + the Python interpreter was built. + The result of packing a given C struct includes pad bytes which + maintain proper alignment for the C types involved; similarly, + alignment is taken into account when unpacking. In contrast, when + communicating data between external sources, the programmer is + responsible for defining byte ordering and padding between elements. + See :ref:`struct-alignment` for details. Several :mod:`struct` functions (and methods of :class:`Struct`) take a *buffer* argument. This refers to objects that implement the :ref:`bufferobjects` and @@ -102,10 +106,13 @@ The module defines the following exception and functions: Format Strings -------------- -Format strings are the mechanism used to specify the expected layout when -packing and unpacking data. They are built up from :ref:`format-characters`, -which specify the type of data being packed/unpacked. In addition, there are -special characters for controlling the :ref:`struct-alignment`. +Format strings describe the data layout when +packing and unpacking data. They are built up from :ref:`format characters`, +which specify the type of data being packed/unpacked. In addition, +special characters control the :ref:`byte order, size and alignment`. +Each format string consists of an optional prefix character which +describes the overall properties of the data and one or more format +characters which describe the actual data values and padding. .. _struct-alignment: @@ -116,6 +123,11 @@ Byte Order, Size, and Alignment By default, C types are represented in the machine's native format and byte order, and properly aligned by skipping pad bytes if necessary (according to the rules used by the C compiler). +This behavior is chosen so +that the bytes of a packed struct correspond exactly to the memory layout +of the corresponding C struct. +Whether to use native byte ordering +and padding or standard formats depends on the application. .. index:: single: @ (at); in struct format strings @@ -144,12 +156,10 @@ following table: If the first character is not one of these, ``'@'`` is assumed. -Native byte order is big-endian or little-endian, depending on the host -system. For example, Intel x86 and AMD64 (x86-64) are little-endian; -IBM z and most legacy architectures are big-endian; -and ARM, RISC-V and IBM Power feature switchable endianness -(bi-endian, though the former two are nearly always little-endian in practice). -Use ``sys.byteorder`` to check the endianness of your system. +Native byte order is big-endian or little-endian, depending on the +host system. For example, Intel x86, AMD64 (x86-64), and Apple M1 are +little-endian; IBM z and many legacy architectures are big-endian. +Use :data:`sys.byteorder` to check the endianness of your system. Native size and alignment are determined using the C compiler's ``sizeof`` expression. This is always combined with native byte order. @@ -194,7 +204,7 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | Format | C Type | Python type | Standard size | Notes | +========+==========================+====================+================+============+ -| ``x`` | pad byte | no value | | | +| ``x`` | pad byte | no value | | \(7) | +--------+--------------------------+--------------------+----------------+------------+ | ``c`` | :c:expr:`char` | bytes of length 1 | 1 | | +--------+--------------------------+--------------------+----------------+------------+ @@ -231,9 +241,9 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``d`` | :c:expr:`double` | float | 8 | \(4) | +--------+--------------------------+--------------------+----------------+------------+ -| ``s`` | :c:expr:`char[]` | bytes | | | +| ``s`` | :c:expr:`char[]` | bytes | | \(9) | +--------+--------------------------+--------------------+----------------+------------+ -| ``p`` | :c:expr:`char[]` | bytes | | | +| ``p`` | :c:expr:`char[]` | bytes | | \(8) | +--------+--------------------------+--------------------+----------------+------------+ | ``P`` | :c:expr:`void \*` | integer | | \(5) | +--------+--------------------------+--------------------+----------------+------------+ @@ -291,6 +301,34 @@ Notes: operations. See the Wikipedia page on the `half-precision floating-point format `_ for more information. +(7) + When packing, ``'x'`` inserts one NUL byte. + +(8) + The ``'p'`` format character encodes a "Pascal string", meaning a short + variable-length string stored in a *fixed number of bytes*, given by the count. + The first byte stored is the length of the string, or 255, whichever is + smaller. The bytes of the string follow. If the string passed in to + :func:`pack` is too long (longer than the count minus 1), only the leading + ``count-1`` bytes of the string are stored. If the string is shorter than + ``count-1``, it is padded with null bytes so that exactly count bytes in all + are used. Note that for :func:`unpack`, the ``'p'`` format character consumes + ``count`` bytes, but that the string returned can never contain more than 255 + bytes. + +(9) + For the ``'s'`` format character, the count is interpreted as the length of the + bytes, not a repeat count like for the other format characters; for example, + ``'10s'`` means a single 10-byte string mapping to or from a single + Python byte string, while ``'10c'`` means 10 + separate one byte character elements (e.g., ``cccccccccc``) mapping + to or from ten different Python byte objects. (See :ref:`struct-examples` + for a concrete demonstration of the difference.) + If a count is not given, it defaults to 1. For packing, the string is + truncated or padded with null bytes as appropriate to make it fit. For + unpacking, the resulting bytes object always has exactly the specified number + of bytes. As a special case, ``'0s'`` means a single, empty string (while + ``'0c'`` means 0 characters). A format character may be preceded by an integral repeat count. For example, the format string ``'4h'`` means exactly the same as ``'hhhh'``. @@ -298,15 +336,6 @@ the format string ``'4h'`` means exactly the same as ``'hhhh'``. Whitespace characters between formats are ignored; a count and its format must not contain whitespace though. -For the ``'s'`` format character, the count is interpreted as the length of the -bytes, not a repeat count like for the other format characters; for example, -``'10s'`` means a single 10-byte string, while ``'10c'`` means 10 characters. -If a count is not given, it defaults to 1. For packing, the string is -truncated or padded with null bytes as appropriate to make it fit. For -unpacking, the resulting bytes object always has exactly the specified number -of bytes. As a special case, ``'0s'`` means a single, empty string (while -``'0c'`` means 0 characters). - When packing a value ``x`` using one of the integer formats (``'b'``, ``'B'``, ``'h'``, ``'H'``, ``'i'``, ``'I'``, ``'l'``, ``'L'``, ``'q'``, ``'Q'``), if ``x`` is outside the valid range for that format @@ -316,17 +345,6 @@ then :exc:`struct.error` is raised. Previously, some of the integer formats wrapped out-of-range values and raised :exc:`DeprecationWarning` instead of :exc:`struct.error`. -The ``'p'`` format character encodes a "Pascal string", meaning a short -variable-length string stored in a *fixed number of bytes*, given by the count. -The first byte stored is the length of the string, or 255, whichever is -smaller. The bytes of the string follow. If the string passed in to -:func:`pack` is too long (longer than the count minus 1), only the leading -``count-1`` bytes of the string are stored. If the string is shorter than -``count-1``, it is padded with null bytes so that exactly count bytes in all -are used. Note that for :func:`unpack`, the ``'p'`` format character consumes -``count`` bytes, but that the string returned can never contain more than 255 -bytes. - .. index:: single: ? (question mark); in struct format strings For the ``'?'`` format character, the return value is either :const:`True` or @@ -342,18 +360,36 @@ Examples ^^^^^^^^ .. note:: - All examples assume a native byte order, size, and alignment with a - big-endian machine. + Native byte order examples (designated by the ``'@'`` format prefix or + lack of any prefix character) may not match what the reader's + machine produces as + that depends on the platform and compiler. + +Pack and unpack integers of three different sizes, using big endian +ordering:: + + >>> from struct import * + >>> pack(">bhl", 1, 2, 3) + b'\x01\x00\x02\x00\x00\x00\x03' + >>> unpack('>bhl', b'\x01\x00\x02\x00\x00\x00\x03' + (1, 2, 3) + >>> calcsize('>bhl') + 7 -A basic example of packing/unpacking three integers:: +Attempt to pack an integer which is too large for the defined field:: - >>> from struct import * - >>> pack('hhl', 1, 2, 3) - b'\x00\x01\x00\x02\x00\x00\x00\x03' - >>> unpack('hhl', b'\x00\x01\x00\x02\x00\x00\x00\x03') - (1, 2, 3) - >>> calcsize('hhl') - 8 + >>> pack(">h", 99999) + Traceback (most recent call last): + File "", line 1, in + struct.error: 'h' format requires -32768 <= number <= 32767 + +Demonstrate the difference between ``'s'`` and ``'c'`` format +characters:: + + >>> pack("@ccc", b'1', b'2', b'3') + b'123' + >>> pack("@3s", b'123') + b'123' Unpacked fields can be named by assigning them to variables or by wrapping the result in a named tuple:: @@ -366,35 +402,132 @@ the result in a named tuple:: >>> Student._make(unpack('<10sHHb', record)) Student(name=b'raymond ', serialnum=4658, school=264, gradelevel=8) -The ordering of format characters may have an impact on size since the padding -needed to satisfy alignment requirements is different:: - - >>> pack('ci', b'*', 0x12131415) - b'*\x00\x00\x00\x12\x13\x14\x15' - >>> pack('ic', 0x12131415, b'*') - b'\x12\x13\x14\x15*' - >>> calcsize('ci') +The ordering of format characters may have an impact on size in native +mode since padding is implicit. In standard mode, the user is +responsible for inserting any desired padding. +Note in +the first ``pack`` call below that three NUL bytes were added after the +packed ``'#'`` to align the following integer on a four-byte boundary. +In this example, the output was produced on a little endian machine:: + + >>> pack('@ci', b'#', 0x12131415) + b'#\x00\x00\x00\x15\x14\x13\x12' + >>> pack('@ic', 0x12131415, b'#') + b'\x15\x14\x13\x12#' + >>> calcsize('@ci') 8 - >>> calcsize('ic') + >>> calcsize('@ic') 5 -The following format ``'llh0l'`` specifies two pad bytes at the end, assuming -longs are aligned on 4-byte boundaries:: +The following format ``'llh0l'`` results in two pad bytes being added +at the end, assuming the platform's longs are aligned on 4-byte boundaries:: - >>> pack('llh0l', 1, 2, 3) + >>> pack('@llh0l', 1, 2, 3) b'\x00\x00\x00\x01\x00\x00\x00\x02\x00\x03\x00\x00' -This only works when native size and alignment are in effect; standard size and -alignment does not enforce any alignment. - .. seealso:: Module :mod:`array` Packed binary storage of homogeneous data. - Module :mod:`xdrlib` - Packing and unpacking of XDR data. + Module :mod:`json` + JSON encoder and decoder. + + Module :mod:`pickle` + Python object serialization. + + +.. _applications: + +Applications +------------ + +Two main applications for the :mod:`struct` module exist, data +interchange between Python and C code within an application or another +application compiled using the same compiler (:ref:`native formats`), and +data interchange between applications using agreed upon data layout +(:ref:`standard formats`). Generally speaking, the format strings +constructed for these two domains are distinct. + + +.. _struct-native-formats: + +Native Formats +^^^^^^^^^^^^^^ + +When constructing format strings which mimic native layouts, the +compiler and machine architecture determine byte ordering and padding. +In such cases, the ``@`` format character should be used to specify +native byte ordering and data sizes. Internal pad bytes are normally inserted +automatically. It is possible that a zero-repeat format code will be +needed at the end of a format string to round up to the correct +byte boundary for proper alignment of consective chunks of data. + +Consider these two simple examples (on a 64-bit, little-endian +machine):: + + >>> calcsize('@lhl') + 24 + >>> calcsize('@llh') + 18 + +Data is not padded to an 8-byte boundary at the end of the second +format string without the use of extra padding. A zero-repeat format +code solves that problem:: + + >>> calcsize('@llh0l') + 24 + +The ``'x'`` format code can be used to specify the repeat, but for +native formats it is better to use a zero-repeat format like ``'0l'``. + +By default, native byte ordering and alignment is used, but it is +better to be explicit and use the ``'@'`` prefix character. + + +.. _struct-standard-formats: + +Standard Formats +^^^^^^^^^^^^^^^^ + +When exchanging data beyond your process such as networking or storage, +be precise. Specify the exact byte order, size, and alignment. Do +not assume they match the native order of a particular machine. +For example, network byte order is big-endian, while many popular CPUs +are little-endian. By defining this explicitly, the user need not +care about the specifics of the platform their code is running on. +The first character should typically be ``<`` or ``>`` +(or ``!``). Padding is the responsibility of the programmer. The +zero-repeat format character won't work. Instead, the user must +explicitly add ``'x'`` pad bytes where needed. Revisiting the +examples from the previous section, we have:: + + >>> calcsize('>> pack('>> calcsize('@llh') + 18 + >>> pack('@llh', 1, 2, 3) == pack('>> calcsize('>> calcsize('@llh0l') + 24 + >>> pack('@llh0l', 1, 2, 3) == pack('>> calcsize('>> calcsize('@llh0l') + 12 + >>> pack('@llh0l', 1, 2, 3) == pack('` now supports ``[]``. See :pep:`585` and - :ref:`types-genericalias`. + :class:`builtins.tuple ` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. data:: Union @@ -849,8 +849,8 @@ These can be used as types in annotations using ``[]``, each having a unique syn respectively. .. deprecated:: 3.9 - :class:`collections.abc.Callable` now supports ``[]``. See :pep:`585` and - :ref:`types-genericalias`. + :class:`collections.abc.Callable` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. versionchanged:: 3.10 ``Callable`` now supports :class:`ParamSpec` and :data:`Concatenate`. @@ -957,8 +957,8 @@ These can be used as types in annotations using ``[]``, each having a unique syn .. versionadded:: 3.5.2 .. deprecated:: 3.9 - :class:`builtins.type ` now supports ``[]``. See :pep:`585` and - :ref:`types-genericalias`. + :class:`builtins.type ` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. data:: Literal @@ -1339,7 +1339,7 @@ These are not used in annotations. They are building blocks for creating generic ``Unpack[Ts]``.) Type variable tuples must *always* be unpacked. This helps distinguish type - variable types from normal type variables:: + variable tuples from normal type variables:: x: Ts # Not valid x: tuple[Ts] # Not valid @@ -1351,7 +1351,7 @@ These are not used in annotations. They are building blocks for creating generic Shape = TypeVarTuple('Shape') class Array(Generic[*Shape]): def __getitem__(self, key: tuple[*Shape]) -> float: ... - def __abs__(self) -> Array[*Shape]: ... + def __abs__(self) -> "Array[*Shape]": ... def get_shape(self) -> tuple[*Shape]: ... Type variable tuples can be happily combined with normal type variables:: @@ -1896,8 +1896,8 @@ Corresponding to built-in types ... .. deprecated:: 3.9 - :class:`builtins.dict ` now supports ``[]``. See :pep:`585` and - :ref:`types-genericalias`. + :class:`builtins.dict ` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: List(list, MutableSequence[T]) @@ -1917,8 +1917,8 @@ Corresponding to built-in types return [item for item in vector if item > 0] .. deprecated:: 3.9 - :class:`builtins.list ` now supports ``[]``. See :pep:`585` and - :ref:`types-genericalias`. + :class:`builtins.list ` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: Set(set, MutableSet[T]) @@ -1927,16 +1927,17 @@ Corresponding to built-in types to use an abstract collection type such as :class:`AbstractSet`. .. deprecated:: 3.9 - :class:`builtins.set ` now supports ``[]``. See :pep:`585` and - :ref:`types-genericalias`. + :class:`builtins.set ` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: FrozenSet(frozenset, AbstractSet[T_co]) A generic version of :class:`builtins.frozenset `. .. deprecated:: 3.9 - :class:`builtins.frozenset ` now supports ``[]``. See - :pep:`585` and :ref:`types-genericalias`. + :class:`builtins.frozenset ` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. note:: :data:`Tuple` is a special form. @@ -1950,8 +1951,8 @@ Corresponding to types in :mod:`collections` .. versionadded:: 3.5.2 .. deprecated:: 3.9 - :class:`collections.defaultdict` now supports ``[]``. See :pep:`585` and - :ref:`types-genericalias`. + :class:`collections.defaultdict` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: OrderedDict(collections.OrderedDict, MutableMapping[KT, VT]) @@ -1960,8 +1961,8 @@ Corresponding to types in :mod:`collections` .. versionadded:: 3.7.2 .. deprecated:: 3.9 - :class:`collections.OrderedDict` now supports ``[]``. See :pep:`585` and - :ref:`types-genericalias`. + :class:`collections.OrderedDict` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) @@ -1971,8 +1972,8 @@ Corresponding to types in :mod:`collections` .. versionadded:: 3.6.1 .. deprecated:: 3.9 - :class:`collections.ChainMap` now supports ``[]``. See :pep:`585` and - :ref:`types-genericalias`. + :class:`collections.ChainMap` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: Counter(collections.Counter, Dict[T, int]) @@ -1982,8 +1983,8 @@ Corresponding to types in :mod:`collections` .. versionadded:: 3.6.1 .. deprecated:: 3.9 - :class:`collections.Counter` now supports ``[]``. See :pep:`585` and - :ref:`types-genericalias`. + :class:`collections.Counter` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: Deque(deque, MutableSequence[T]) @@ -1993,8 +1994,8 @@ Corresponding to types in :mod:`collections` .. versionadded:: 3.6.1 .. deprecated:: 3.9 - :class:`collections.deque` now supports ``[]``. See :pep:`585` and - :ref:`types-genericalias`. + :class:`collections.deque` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. Other concrete types """""""""""""""""""" @@ -2008,7 +2009,7 @@ Other concrete types represent the types of I/O streams such as returned by :func:`open`. - .. deprecated-removed:: 3.8 3.12 + .. deprecated-removed:: 3.8 3.13 The ``typing.io`` namespace is deprecated and will be removed. These types should be directly imported from ``typing`` instead. @@ -2022,7 +2023,7 @@ Other concrete types ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or ``Match[bytes]``. - .. deprecated-removed:: 3.8 3.12 + .. deprecated-removed:: 3.8 3.13 The ``typing.re`` namespace is deprecated and will be removed. These types should be directly imported from ``typing`` instead. @@ -2056,13 +2057,13 @@ Abstract Base Classes Corresponding to collections in :mod:`collections.abc` """""""""""""""""""""""""""""""""""""""""""""""""""""" -.. class:: AbstractSet(Sized, Collection[T_co]) +.. class:: AbstractSet(Collection[T_co]) A generic version of :class:`collections.abc.Set`. .. deprecated:: 3.9 - :class:`collections.abc.Set` now supports ``[]``. See :pep:`585` and - :ref:`types-genericalias`. + :class:`collections.abc.Set` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: ByteString(Sequence[int]) @@ -2075,8 +2076,8 @@ Corresponding to collections in :mod:`collections.abc` annotate arguments of any of the types mentioned above. .. deprecated:: 3.9 - :class:`collections.abc.ByteString` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.ByteString` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: Collection(Sized, Iterable[T_co], Container[T_co]) @@ -2085,34 +2086,34 @@ Corresponding to collections in :mod:`collections.abc` .. versionadded:: 3.6.0 .. deprecated:: 3.9 - :class:`collections.abc.Collection` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.Collection` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: Container(Generic[T_co]) A generic version of :class:`collections.abc.Container`. .. deprecated:: 3.9 - :class:`collections.abc.Container` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.Container` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. -.. class:: ItemsView(MappingView, Generic[KT_co, VT_co]) +.. class:: ItemsView(MappingView, AbstractSet[tuple[KT_co, VT_co]]) A generic version of :class:`collections.abc.ItemsView`. .. deprecated:: 3.9 - :class:`collections.abc.ItemsView` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.ItemsView` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. -.. class:: KeysView(MappingView[KT_co], AbstractSet[KT_co]) +.. class:: KeysView(MappingView, AbstractSet[KT_co]) A generic version of :class:`collections.abc.KeysView`. .. deprecated:: 3.9 - :class:`collections.abc.KeysView` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.KeysView` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. -.. class:: Mapping(Sized, Collection[KT], Generic[VT_co]) +.. class:: Mapping(Collection[KT], Generic[KT, VT_co]) A generic version of :class:`collections.abc.Mapping`. This type can be used as follows:: @@ -2121,56 +2122,58 @@ Corresponding to collections in :mod:`collections.abc` return word_list[word] .. deprecated:: 3.9 - :class:`collections.abc.Mapping` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.Mapping` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. -.. class:: MappingView(Sized, Iterable[T_co]) +.. class:: MappingView(Sized) A generic version of :class:`collections.abc.MappingView`. .. deprecated:: 3.9 - :class:`collections.abc.MappingView` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.MappingView` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: MutableMapping(Mapping[KT, VT]) A generic version of :class:`collections.abc.MutableMapping`. .. deprecated:: 3.9 - :class:`collections.abc.MutableMapping` now supports ``[]``. See - :pep:`585` and :ref:`types-genericalias`. + :class:`collections.abc.MutableMapping` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: MutableSequence(Sequence[T]) A generic version of :class:`collections.abc.MutableSequence`. .. deprecated:: 3.9 - :class:`collections.abc.MutableSequence` now supports ``[]``. See - :pep:`585` and :ref:`types-genericalias`. + :class:`collections.abc.MutableSequence` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: MutableSet(AbstractSet[T]) A generic version of :class:`collections.abc.MutableSet`. .. deprecated:: 3.9 - :class:`collections.abc.MutableSet` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.MutableSet` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: Sequence(Reversible[T_co], Collection[T_co]) A generic version of :class:`collections.abc.Sequence`. .. deprecated:: 3.9 - :class:`collections.abc.Sequence` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.Sequence` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. -.. class:: ValuesView(MappingView[VT_co]) +.. class:: ValuesView(MappingView, Collection[_VT_co]) A generic version of :class:`collections.abc.ValuesView`. .. deprecated:: 3.9 - :class:`collections.abc.ValuesView` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.ValuesView` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. Corresponding to other types in :mod:`collections.abc` """""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -2180,16 +2183,16 @@ Corresponding to other types in :mod:`collections.abc` A generic version of :class:`collections.abc.Iterable`. .. deprecated:: 3.9 - :class:`collections.abc.Iterable` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.Iterable` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: Iterator(Iterable[T_co]) A generic version of :class:`collections.abc.Iterator`. .. deprecated:: 3.9 - :class:`collections.abc.Iterator` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.Iterator` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) @@ -2223,8 +2226,8 @@ Corresponding to other types in :mod:`collections.abc` start += 1 .. deprecated:: 3.9 - :class:`collections.abc.Generator` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.Generator` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: Hashable @@ -2235,8 +2238,8 @@ Corresponding to other types in :mod:`collections.abc` A generic version of :class:`collections.abc.Reversible`. .. deprecated:: 3.9 - :class:`collections.abc.Reversible` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.Reversible` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: Sized @@ -2260,8 +2263,8 @@ Asynchronous programming .. versionadded:: 3.5.3 .. deprecated:: 3.9 - :class:`collections.abc.Coroutine` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.Coroutine` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: AsyncGenerator(AsyncIterator[T_co], Generic[T_co, T_contra]) @@ -2297,8 +2300,9 @@ Asynchronous programming .. versionadded:: 3.6.1 .. deprecated:: 3.9 - :class:`collections.abc.AsyncGenerator` now supports ``[]``. See - :pep:`585` and :ref:`types-genericalias`. + :class:`collections.abc.AsyncGenerator` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: AsyncIterable(Generic[T_co]) @@ -2307,8 +2311,8 @@ Asynchronous programming .. versionadded:: 3.5.2 .. deprecated:: 3.9 - :class:`collections.abc.AsyncIterable` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.AsyncIterable` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: AsyncIterator(AsyncIterable[T_co]) @@ -2317,8 +2321,8 @@ Asynchronous programming .. versionadded:: 3.5.2 .. deprecated:: 3.9 - :class:`collections.abc.AsyncIterator` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.AsyncIterator` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: Awaitable(Generic[T_co]) @@ -2327,8 +2331,8 @@ Asynchronous programming .. versionadded:: 3.5.2 .. deprecated:: 3.9 - :class:`collections.abc.Awaitable` now supports ``[]``. See :pep:`585` - and :ref:`types-genericalias`. + :class:`collections.abc.Awaitable` now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. Context manager types @@ -2342,8 +2346,9 @@ Context manager types .. versionadded:: 3.6.0 .. deprecated:: 3.9 - :class:`contextlib.AbstractContextManager` now supports ``[]``. See - :pep:`585` and :ref:`types-genericalias`. + :class:`contextlib.AbstractContextManager` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. .. class:: AsyncContextManager(Generic[T_co]) @@ -2353,8 +2358,9 @@ Context manager types .. versionadded:: 3.6.2 .. deprecated:: 3.9 - :class:`contextlib.AbstractAsyncContextManager` now supports ``[]``. See - :pep:`585` and :ref:`types-genericalias`. + :class:`contextlib.AbstractAsyncContextManager` + now supports subscripting (``[]``). + See :pep:`585` and :ref:`types-genericalias`. Protocols --------- @@ -2856,7 +2862,7 @@ convenience. This is subject to change, and not all deprecations are listed. +----------------------------------+---------------+-------------------+----------------+ | Feature | Deprecated in | Projected removal | PEP/issue | +==================================+===============+===================+================+ -| ``typing.io`` and ``typing.re`` | 3.8 | 3.12 | :issue:`38291` | +| ``typing.io`` and ``typing.re`` | 3.8 | 3.13 | :issue:`38291` | | submodules | | | | +----------------------------------+---------------+-------------------+----------------+ | ``typing`` versions of standard | 3.9 | Undecided | :pep:`585` | diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 3bed25645a9271..adc6cd339ac157 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -15,14 +15,22 @@ -------------- -The :mod:`venv` module provides support for creating lightweight "virtual -environments" with their own site directories, optionally isolated from system -site directories. Each virtual environment has its own Python binary (which -matches the version of the binary that was used to create this environment) and -can have its own independent set of installed Python packages in its site -directories. +.. _venv-def: +.. _venv-intro: + +The :mod:`!venv` module supports creating lightweight "virtual environments", +each with their own independent set of Python packages installed in +their :mod:`site` directories. +A virtual environment is created on top of an existing +Python installation, known as the virtual environment's "base" Python, and may +optionally be isolated from the packages in the base environment, +so only those explicitly installed in the virtual environment are available. + +When used from within a virtual environment, common installation tools such as +`pip`_ will install Python packages into a virtual environment +without needing to be told to do so explicitly. -See :pep:`405` for more information about Python virtual environments. +See :pep:`405` for more background on Python virtual environments. .. seealso:: @@ -36,54 +44,72 @@ Creating virtual environments .. include:: /using/venv-create.inc +.. _venv-explanation: -.. _venv-def: +How venvs work +-------------- -.. note:: A virtual environment is a Python environment such that the Python - interpreter, libraries and scripts installed into it are isolated from those - installed in other virtual environments, and (by default) any libraries - installed in a "system" Python, i.e., one which is installed as part of your - operating system. - - A virtual environment is a directory tree which contains Python executable - files and other files which indicate that it is a virtual environment. - - Common installation tools such as setuptools_ and pip_ work as - expected with virtual environments. In other words, when a virtual - environment is active, they install Python packages into the virtual - environment without needing to be told to do so explicitly. - - When a virtual environment is active (i.e., the virtual environment's Python - interpreter is running), the attributes :attr:`sys.prefix` and - :attr:`sys.exec_prefix` point to the base directory of the virtual - environment, whereas :attr:`sys.base_prefix` and - :attr:`sys.base_exec_prefix` point to the non-virtual environment Python - installation which was used to create the virtual environment. If a virtual - environment is not active, then :attr:`sys.prefix` is the same as - :attr:`sys.base_prefix` and :attr:`sys.exec_prefix` is the same as - :attr:`sys.base_exec_prefix` (they all point to a non-virtual environment - Python installation). - - When a virtual environment is active, any options that change the - installation path will be ignored from all :mod:`distutils` configuration - files to prevent projects being inadvertently installed outside of the - virtual environment. - - When working in a command shell, users can make a virtual environment active - by running an ``activate`` script in the virtual environment's executables - directory (the precise filename and command to use the file is - shell-dependent), which prepends the virtual environment's directory for - executables to the ``PATH`` environment variable for the running shell. There - should be no need in other circumstances to activate a virtual - environment; scripts installed into virtual environments have a "shebang" - line which points to the virtual environment's Python interpreter. This means - that the script will run with that interpreter regardless of the value of - ``PATH``. On Windows, "shebang" line processing is supported if you have the - Python Launcher for Windows installed (this was added to Python in 3.3 - see - :pep:`397` for more details). Thus, double-clicking an installed script in a - Windows Explorer window should run the script with the correct interpreter - without there needing to be any reference to its virtual environment in - ``PATH``. +When a Python interpreter is running from a virtual environment, +:data:`sys.prefix` and :data:`sys.exec_prefix` +point to the directories of the virtual environment, +whereas :data:`sys.base_prefix` and :data:`sys.base_exec_prefix` +point to those of the base Python used to create the environment. +It is sufficient to check +``sys.prefix == sys.base_prefix`` to determine if the current interpreter is +running from a virtual environment. + +A virtual environment may be "activated" using a script in its binary directory +(``bin`` on POSIX; ``Scripts`` on Windows). +This will prepend that directory to your :envvar:`!PATH`, so that running +:program:`!python` will invoke the environment's Python interpreter +and you can run installed scripts without having to use their full path. +The invocation of the activation script is platform-specific +(:samp:`{}` must be replaced by the path to the directory +containing the virtual environment): + ++-------------+------------+--------------------------------------------------+ +| Platform | Shell | Command to activate virtual environment | ++=============+============+==================================================+ +| POSIX | bash/zsh | :samp:`$ source {}/bin/activate` | +| +------------+--------------------------------------------------+ +| | fish | :samp:`$ source {}/bin/activate.fish` | +| +------------+--------------------------------------------------+ +| | csh/tcsh | :samp:`$ source {}/bin/activate.csh` | +| +------------+--------------------------------------------------+ +| | PowerShell | :samp:`$ {}/bin/Activate.ps1` | ++-------------+------------+--------------------------------------------------+ +| Windows | cmd.exe | :samp:`C:\\> {}\\Scripts\\activate.bat` | +| +------------+--------------------------------------------------+ +| | PowerShell | :samp:`PS C:\\> {}\\Scripts\\Activate.ps1` | ++-------------+------------+--------------------------------------------------+ + +.. versionadded:: 3.4 + :program:`!fish` and :program:`!csh` activation scripts. + +.. versionadded:: 3.8 + PowerShell activation scripts installed under POSIX for PowerShell Core + support. + +You don't specifically *need* to activate a virtual environment, +as you can just specify the full path to that environment's +Python interpreter when invoking Python. +Furthermore, all scripts installed in the environment +should be runnable without activating it. + +In order to achieve this, scripts installed into virtual environments have +a "shebang" line which points to the environment's Python interpreter, +i.e. :samp:`#!/{}/bin/python`. +This means that the script will run with that interpreter regardless of the +value of :envvar:`!PATH`. On Windows, "shebang" line processing is supported if +you have the :ref:`launcher` installed. Thus, double-clicking an installed +script in a Windows Explorer window should run it with the correct interpreter +without the environment needing to be activated or on the :envvar:`!PATH`. + +When a virtual environment has been activated, the :envvar:`!VIRTUAL_ENV` +environment variable is set to the path of the environment. +Since explicitly activating a virtual environment is not required to use it, +:envvar:`!VIRTUAL_ENV` cannot be relied upon to determine +whether a virtual environment is being used. .. warning:: Because scripts installed in environments should not expect the environment to be activated, their shebang lines contain the absolute paths @@ -99,6 +125,11 @@ Creating virtual environments environment in its new location. Otherwise, software installed into the environment may not work as expected. +You can deactivate a virtual environment by typing ``deactivate`` in your shell. +The exact mechanism is platform-specific and is an internal implementation +detail (typically, a script or shell function will be used). + + .. _venv-api: API @@ -191,6 +222,45 @@ creation according to their needs, the :class:`EnvBuilder` class. ``clear=True``, contents of the environment directory will be cleared and then all necessary subdirectories will be recreated. + The returned context object is a :class:`types.SimpleNamespace` with the + following attributes: + + * ``env_dir`` - The location of the virtual environment. Used for + ``__VENV_DIR__`` in activation scripts (see :meth:`install_scripts`). + + * ``env_name`` - The name of the virtual environment. Used for + ``__VENV_NAME__`` in activation scripts (see :meth:`install_scripts`). + + * ``prompt`` - The prompt to be used by the activation scripts. Used for + ``__VENV_PROMPT__`` in activation scripts (see :meth:`install_scripts`). + + * ``executable`` - The underlying Python executable used by the virtual + environment. This takes into account the case where a virtual environment + is created from another virtual environment. + + * ``inc_path`` - The include path for the virtual environment. + + * ``lib_path`` - The purelib path for the virtual environment. + + * ``bin_path`` - The script path for the virtual environment. + + * ``bin_name`` - The name of the script path relative to the virtual + environment location. Used for ``__VENV_BIN_NAME__`` in activation + scripts (see :meth:`install_scripts`). + + * ``env_exe`` - The name of the Python interpreter in the virtual + environment. Used for ``__VENV_PYTHON__`` in activation scripts + (see :meth:`install_scripts`). + + * ``env_exec_cmd`` - The name of the Python interpreter, taking into + account filesystem redirections. This can be used to run Python in + the virtual environment. + + + .. versionchanged:: 3.12 + The attribute ``lib_path`` was added to the context, and the context + object was documented. + .. versionchanged:: 3.11 The *venv* :ref:`sysconfig installation scheme ` diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 8397de4fb488f1..73e7b21ae405d2 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -143,9 +143,12 @@ See :ref:`__slots__ documentation ` for details. ``ProxyType`` or ``CallableProxyType``, depending on whether *object* is callable. Proxy objects are not :term:`hashable` regardless of the referent; this avoids a number of problems related to their fundamentally mutable nature, and - prevent their use as dictionary keys. *callback* is the same as the parameter + prevents their use as dictionary keys. *callback* is the same as the parameter of the same name to the :func:`ref` function. + Accessing an attribute of the proxy object after the referent is + garbage collected raises :exc:`ReferenceError`. + .. versionchanged:: 3.8 Extended the operator support on proxy objects to include the matrix multiplication operators ``@`` and ``@=``. @@ -209,7 +212,7 @@ objects. discarded when no strong reference to it exists any more. -.. class:: WeakMethod(method) +.. class:: WeakMethod(method[, callback]) A custom :class:`ref` subclass which simulates a weak reference to a bound method (i.e., a method defined on a class and looked up on an instance). @@ -235,6 +238,8 @@ objects. >>> r() >>> + *callback* is the same as the parameter of the same name to the :func:`ref` function. + .. versionadded:: 3.4 .. class:: finalize(obj, func, /, *args, **kwargs) diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index 06223e667a450a..75dea466335163 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -7,6 +7,8 @@ .. moduleauthor:: Phillip J. Eby .. sectionauthor:: Phillip J. Eby +**Source code:** :source:`Lib/wsgiref` + -------------- The Web Server Gateway Interface (WSGI) is a standard interface between web diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst index 2f1879dc056a88..d2e5619e7e47c2 100644 --- a/Doc/library/zoneinfo.rst +++ b/Doc/library/zoneinfo.rst @@ -9,6 +9,8 @@ .. moduleauthor:: Paul Ganssle .. sectionauthor:: Paul Ganssle +**Source code:** :source:`Lib/zoneinfo` + -------------- The :mod:`zoneinfo` module provides a concrete time zone implementation to diff --git a/Doc/license.rst b/Doc/license.rst index 00691b30ba6d3e..4caecdce7777f4 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -984,3 +984,31 @@ https://www.w3.org/TR/xml-c14n2-testcases/ and is distributed under the THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Audioop +------- + +The audioop module uses the code base in g771.c file of the SoX project:: + + Programming the AdLib/Sound Blaster + FM Music Chips + Version 2.0 (24 Feb 1992) + Copyright (c) 1991, 1992 by Jeffrey S. Lee + jlee@smylex.uucp + Warranty and Copyright Policy + This document is provided on an "as-is" basis, and its author makes + no warranty or representation, express or implied, with respect to + its quality performance or fitness for a particular purpose. In no + event will the author of this document be liable for direct, indirect, + special, incidental, or consequential damages arising out of the use + or inability to use the information contained within. Use of this + document is at your own risk. + This file may be used and copied freely so long as the applicable + copyright notices are retained, and no modifications are made to the + text of the document. No money shall be charged for its distribution + beyond reasonable shipping, handling and duplication costs, nor shall + proprietary changes be made to this document so that it cannot be + distributed freely. This document may not be included in published + material or commercial packages without the written consent of its + author. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 9acad500a40919..9e09515f50d12f 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -343,7 +343,7 @@ the case of :keyword:`except`, but in the case of exception groups we can have partial matches when the type matches some of the exceptions in the group. This means that multiple :keyword:`!except*` clauses can execute, each handling part of the exception group. -Each clause executes once and handles an exception group +Each clause executes at most once and handles an exception group of all matching exceptions. Each exception in the group is handled by at most one :keyword:`!except*` clause, the first that matches it. :: @@ -364,16 +364,28 @@ one :keyword:`!except*` clause, the first that matches it. :: | ValueError: 1 +------------------------------------ - Any remaining exceptions that were not handled by any :keyword:`!except*` - clause are re-raised at the end, combined into an exception group along with - all exceptions that were raised from within :keyword:`!except*` clauses. - An :keyword:`!except*` clause must have a matching type, - and this type cannot be a subclass of :exc:`BaseExceptionGroup`. - It is not possible to mix :keyword:`except` and :keyword:`!except*` - in the same :keyword:`try`. - :keyword:`break`, :keyword:`continue` and :keyword:`return` - cannot appear in an :keyword:`!except*` clause. +Any remaining exceptions that were not handled by any :keyword:`!except*` +clause are re-raised at the end, combined into an exception group along with +all exceptions that were raised from within :keyword:`!except*` clauses. + +If the raised exception is not an exception group and its type matches +one of the :keyword:`!except*` clauses, it is caught and wrapped by an +exception group with an empty message string. :: + + >>> try: + ... raise BlockingIOError + ... except* BlockingIOError as e: + ... print(repr(e)) + ... + ExceptionGroup('', (BlockingIOError())) + +An :keyword:`!except*` clause must have a matching type, +and this type cannot be a subclass of :exc:`BaseExceptionGroup`. +It is not possible to mix :keyword:`except` and :keyword:`!except*` +in the same :keyword:`try`. +:keyword:`break`, :keyword:`continue` and :keyword:`return` +cannot appear in an :keyword:`!except*` clause. .. index:: @@ -581,6 +593,7 @@ The :keyword:`!match` statement keyword: if keyword: as pair: match; case + single: as; match statement single: : (colon); compound statement .. versionadded:: 3.10 diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index f2465cdf40f6d5..92cd2f8f8074b1 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1904,6 +1904,8 @@ Attribute lookup speed can be significantly improved as well. and *__weakref__* for each instance. +.. _datamodel-note-slots: + Notes on using *__slots__* """""""""""""""""""""""""" @@ -2821,7 +2823,7 @@ Customizing positional arguments in class pattern matching When using a class name in a pattern, positional arguments in the pattern are not allowed by default, i.e. ``case MyClass(x, y)`` is typically invalid without special -support in ``MyClass``. To be able to use that kind of patterns, the class needs to +support in ``MyClass``. To be able to use that kind of pattern, the class needs to define a *__match_args__* attribute. .. data:: object.__match_args__ diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index cc969752d5d7b7..0cdf91e75b3444 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -154,7 +154,7 @@ tuple may or may not yield the same object). single: , (comma) Note that tuples are not formed by the parentheses, but rather by use of the -comma operator. The exception is the empty tuple, for which parentheses *are* +comma. The exception is the empty tuple, for which parentheses *are* required --- allowing unparenthesized "nothing" in expressions would cause ambiguities and allow common typos to pass uncaught. diff --git a/Doc/reference/grammar.rst b/Doc/reference/grammar.rst index 59b45005836a76..bc1db7b039cd5a 100644 --- a/Doc/reference/grammar.rst +++ b/Doc/reference/grammar.rst @@ -12,7 +12,7 @@ and `PEG `_. In particular, ``&`` followed by a symbol, token or parenthesized group indicates a positive lookahead (i.e., is required to match but not consumed), while ``!`` indicates a negative lookahead (i.e., is -required _not_ to match). We use the ``|`` separator to mean PEG's +required *not* to match). We use the ``|`` separator to mean PEG's "ordered choice" (written as ``/`` in traditional PEG grammars). See :pep:`617` for more details on the grammar's syntax. diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 5c9937fb5b6d72..c98ac81e415b72 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -330,7 +330,7 @@ statement, of a variable or attribute annotation and an optional assignment stat annotated_assignment_stmt: `augtarget` ":" `expression` : ["=" (`starred_expression` | `yield_expression`)] -The difference from normal :ref:`assignment` is that only single target is allowed. +The difference from normal :ref:`assignment` is that only a single target is allowed. For simple names as assignment targets, if in class or module scope, the annotations are evaluated and stored in a special class or module @@ -365,8 +365,8 @@ target, then the interpreter evaluates the target except for the last IDEs. .. versionchanged:: 3.8 - Now annotated assignments allow same expressions in the right hand side as - the regular assignments. Previously, some expressions (like un-parenthesized + Now annotated assignments allow the same expressions in the right hand side as + regular assignments. Previously, some expressions (like un-parenthesized tuple expressions) caused a syntax error. @@ -756,7 +756,7 @@ commas) the two steps are carried out separately for each clause, just as though the clauses had been separated out into individual import statements. -The details of the first step, finding and loading modules are described in +The details of the first step, finding and loading modules, are described in greater detail in the section on the :ref:`import system `, which also describes the various types of packages and modules that can be imported, as well as all the hooks that can be used to customize diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 7f82dc32113a9d..958665db69e227 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -7,10 +7,7 @@ sphinx==4.5.0 blurb -# sphinx-lint 0.6.2 yields many default role errors due to the new regular -# expression used for default role detection, so we don't use the version -# until the errors are fixed. -sphinx-lint==0.6.4 +sphinx-lint==0.6.7 # The theme used by the documentation is stored separately, so we need # to install that as well. diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 647554d057d917..6c383ea57c8a96 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -26,7 +26,7 @@ from sphinx.errors import NoUri except ImportError: from sphinx.environment import NoUri -from sphinx.locale import translators +from sphinx.locale import _ as sphinx_gettext from sphinx.util import status_iterator, logging from sphinx.util.nodes import split_explicit_title from sphinx.writers.text import TextWriter, TextTranslator @@ -109,7 +109,7 @@ class ImplementationDetail(Directive): def run(self): self.assert_has_content() pnode = nodes.compound(classes=['impl-detail']) - label = translators['sphinx'].gettext(self.label_text) + label = sphinx_gettext(self.label_text) content = self.content add_text = nodes.strong(label, label) self.state.nested_parse(content, self.content_offset, pnode) @@ -257,7 +257,7 @@ def run(self): else: args = [] - label = translators['sphinx'].gettext(self._label[min(2, len(args))]) + label = sphinx_gettext(self._label[min(2, len(args))]) text = label.format(name="``{}``".format(name), args=", ".join("``{}``".format(a) for a in args if a)) @@ -436,7 +436,7 @@ def run(self): else: label = self._removed_label - label = translators['sphinx'].gettext(label) + label = sphinx_gettext(label) text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) if len(self.arguments) == 3: inodes, messages = self.state.inline_text(self.arguments[2], diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 99a77e7addd774..52db51e84cd5fc 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -840,8 +840,9 @@ will always bind to the first parameter. For example:: But using ``/`` (positional only arguments), it is possible since it allows ``name`` as a positional argument and ``'name'`` as a key in the keyword arguments:: - def foo(name, /, **kwds): - return 'name' in kwds + >>> def foo(name, /, **kwds): + ... return 'name' in kwds + ... >>> foo(1, **{'name': 2}) True diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 12b00be3793e14..c8e89d9b79bddd 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -122,7 +122,7 @@ An example that uses most of the list methods:: You might have noticed that methods like ``insert``, ``remove`` or ``sort`` that only modify the list have no return value printed -- they return the default -``None``. [1]_ This is a design principle for all mutable data structures in +``None``. [#]_ This is a design principle for all mutable data structures in Python. Another thing you might notice is that not all data can be sorted or @@ -731,5 +731,5 @@ interpreter will raise a :exc:`TypeError` exception. .. rubric:: Footnotes -.. [1] Other languages may return the mutated object, which allows method +.. [#] Other languages may return the mutated object, which allows method chaining, such as ``d->insert("a")->remove("b")->sort();``. diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 67bb19556681c0..e09c829b8e9721 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -496,7 +496,7 @@ Raising and Handling Multiple Unrelated Exceptions ================================================== There are situations where it is necessary to report several exceptions that -have occurred. This it often the case in concurrency frameworks, when several +have occurred. This is often the case in concurrency frameworks, when several tasks may have failed in parallel, but there are also other use cases where it is desirable to continue execution and collect multiple errors rather than raise the first exception. diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index de84ab7fac8faa..3581b3727a53ea 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -478,7 +478,7 @@ becomes complicated. Rather than having users constantly writing and debugging code to save complicated data types to files, Python allows you to use the popular data interchange format called `JSON (JavaScript Object Notation) -`_. The standard module called :mod:`json` can take Python +`_. The standard module called :mod:`json` can take Python data hierarchies, and convert them to string representations; this process is called :dfn:`serializing`. Reconstructing the data from the string representation is called :dfn:`deserializing`. Between serializing and deserializing, the diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index b9785832864e22..0422cd2e4e7650 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -16,8 +16,8 @@ re-used. .. deprecated:: 3.6 ``pyvenv`` was the recommended tool for creating virtual environments for - Python 3.3 and 3.4, and is `deprecated in Python 3.6 - `_. + Python 3.3 and 3.4, and is + :ref:`deprecated in Python 3.6 `. .. versionchanged:: 3.5 The use of ``venv`` is now recommended for creating virtual environments. @@ -105,45 +105,3 @@ Multiple paths can be given to ``venv``, in which case an identical virtual environment will be created, according to the given options, at each provided path. -Once a virtual environment has been created, it can be "activated" using a -script in the virtual environment's binary directory. The invocation of the -script is platform-specific (`` must be replaced by the path of the -directory containing the virtual environment): - -+-------------+-----------------+-----------------------------------------+ -| Platform | Shell | Command to activate virtual environment | -+=============+=================+=========================================+ -| POSIX | bash/zsh | $ source /bin/activate | -+-------------+-----------------+-----------------------------------------+ -| | fish | $ source /bin/activate.fish | -+-------------+-----------------+-----------------------------------------+ -| | csh/tcsh | $ source /bin/activate.csh | -+-------------+-----------------+-----------------------------------------+ -| | PowerShell Core | $ /bin/Activate.ps1 | -+-------------+-----------------+-----------------------------------------+ -| Windows | cmd.exe | C:\\> \\Scripts\\activate.bat | -+-------------+-----------------+-----------------------------------------+ -| | PowerShell | PS C:\\> \\Scripts\\Activate.ps1 | -+-------------+-----------------+-----------------------------------------+ - -When a virtual environment is active, the :envvar:`VIRTUAL_ENV` environment -variable is set to the path of the virtual environment. This can be used to -check if one is running inside a virtual environment. - -You don't specifically *need* to activate an environment; activation just -prepends the virtual environment's binary directory to your path, so that -"python" invokes the virtual environment's Python interpreter and you can run -installed scripts without having to use their full path. However, all scripts -installed in a virtual environment should be runnable without activating it, -and run with the virtual environment's Python automatically. - -You can deactivate a virtual environment by typing "deactivate" in your shell. -The exact mechanism is platform-specific and is an internal implementation -detail (typically a script or shell function will be used). - -.. versionadded:: 3.4 - ``fish`` and ``csh`` activation scripts. - -.. versionadded:: 3.8 - PowerShell activation scripts installed under POSIX for PowerShell Core - support. diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 4ab68e140b9e31..4526dc34872d9f 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -853,7 +853,6 @@ minor version. I.e. ``/usr/bin/python3.7-32`` will request usage of the not provably i386/32-bit". To request a specific environment, use the new ``-V:`` argument with the complete tag. - The ``/usr/bin/env`` form of shebang line has one further special property. Before looking for installed Python interpreters, this form will search the executable :envvar:`PATH` for a Python executable. This corresponds to the @@ -863,6 +862,13 @@ be found, it will be handled as described below. Additionally, the environment variable :envvar:`PYLAUNCHER_NO_SEARCH_PATH` may be set (to any value) to skip this additional search. +Shebang lines that do not match any of these patterns are treated as **Windows** +paths that are absolute or relative to the directory containing the script file. +This is a convenience for Windows-only scripts, such as those generated by an +installer, since the behavior is not compatible with Unix-style shells. +These paths may be quoted, and may include multiple arguments, after which the +path to the script and any additional arguments will be appended. + Arguments in shebang lines -------------------------- diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 39997661bb96c4..0c3bfda1933957 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -395,7 +395,7 @@ This section has just been a quick overview of the new features, giving enough of an explanation to start you programming, but many details have been simplified or ignored. Where should you go to get a more complete picture? -https://docs.python.org/dev/howto/descriptor.html is a lengthy tutorial introduction to +The :ref:`descriptorhowto` is a lengthy tutorial introduction to the descriptor features, written by Guido van Rossum. If my description has whetted your appetite, go read this tutorial next, because it goes into much more detail about the new features while still remaining quite easy to read. diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index b96dfe9159da1a..34f2656f765c7d 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -217,7 +217,7 @@ the time required to finish the job. During the 2.6 development cycle, Georg Brandl put a lot of effort into building a new toolchain for processing the documentation. The resulting package is called Sphinx, and is available from -http://sphinx-doc.org/. +https://www.sphinx-doc.org/. Sphinx concentrates on HTML output, producing attractively styled and modern HTML; printed output is still supported through conversion to @@ -235,7 +235,7 @@ have adopted Sphinx as their documentation tool. `Documenting Python `__ Describes how to write for Python's documentation. - `Sphinx `__ + `Sphinx `__ Documentation and code for the Sphinx toolchain. `Docutils `__ @@ -1926,7 +1926,7 @@ changes, or look through the Subversion logs for all the details. the left to six places. (Contributed by Skip Montanaro; :issue:`1158`.) * The :mod:`decimal` module was updated to version 1.66 of - `the General Decimal Specification `__. New features + `the General Decimal Specification `__. New features include some methods for some basic mathematical functions such as :meth:`exp` and :meth:`log10`:: diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index 6ce6358d49fbca..fba8816bb243a3 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -451,7 +451,7 @@ Major performance enhancements have been added: * The :mod:`json` module now has a C extension to substantially improve its performance. In addition, the API was modified so that json works only with :class:`str`, not with :class:`bytes`. That change makes the - module closely match the `JSON specification `_ + module closely match the `JSON specification `_ which is defined in terms of Unicode. (Contributed by Bob Ippolito and converted to Py3.1 by Antoine Pitrou diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 24d5bba66e3300..d0b436664ac33f 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -77,8 +77,9 @@ Interpreter improvements: New typing features: * :pep:`604`, Allow writing union types as X | Y -* :pep:`613`, Explicit Type Aliases * :pep:`612`, Parameter Specification Variables +* :pep:`613`, Explicit Type Aliases +* :pep:`647`, User-Defined Type Guards Important deprecations, removals or restrictions: @@ -2151,8 +2152,7 @@ Porting to Python 3.10 * The ``PY_SSIZE_T_CLEAN`` macro must now be defined to use :c:func:`PyArg_ParseTuple` and :c:func:`Py_BuildValue` formats which use ``#``: ``es#``, ``et#``, ``s#``, ``u#``, ``y#``, ``z#``, ``U#`` and ``Z#``. - See :ref:`Parsing arguments and building values - ` and the :pep:`353`. + See :ref:`arg-parsing` and :pep:`353`. (Contributed by Victor Stinner in :issue:`40943`.) * Since :c:func:`Py_REFCNT()` is changed to the inline static function, @@ -2183,8 +2183,7 @@ Porting to Python 3.10 :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome` and :c:func:`Py_GetProgramName` functions now return ``NULL`` if called before :c:func:`Py_Initialize` (before Python is initialized). Use the new - :ref:`Python Initialization Configuration API ` to get the - :ref:`Python Path Configuration. `. + :ref:`init-config` API to get the :ref:`init-path-config`. (Contributed by Victor Stinner in :issue:`42260`.) * :c:func:`PyList_SET_ITEM`, :c:func:`PyTuple_SET_ITEM` and @@ -2198,7 +2197,7 @@ Porting to Python 3.10 ``picklebufobject.h``, ``pyarena.h``, ``pyctype.h``, ``pydebug.h``, ``pyfpe.h``, and ``pytime.h`` have been moved to the ``Include/cpython`` directory. These files must not be included directly, as they are already - included in ``Python.h``: :ref:`Include Files `. If they have + included in ``Python.h``; see :ref:`api-includes`. If they have been included directly, consider including ``Python.h`` instead. (Contributed by Nicholas Sim in :issue:`35134`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 8f3ef3ffc1353e..f09ccb133fbc88 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -4,6 +4,7 @@ :Release: |release| :Date: |today| +:Editor: Pablo Galindo Salgado .. Rules for maintenance: @@ -614,12 +615,18 @@ asyncio These are primarily intended for internal use, notably by :class:`~asyncio.TaskGroup`. + +.. _whatsnew311-contextlib: + contextlib ---------- -Added non parallel-safe :func:`~contextlib.chdir` context manager to change -the current working directory and then restore it on exit. Simple wrapper -around :func:`~os.chdir`. (Contributed by Filipe Laíns in :issue:`25625`) +* Added non parallel-safe :func:`~contextlib.chdir` context manager to change + the current working directory and then restore it on exit. Simple wrapper + around :func:`~os.chdir`. (Contributed by Filipe Laíns in :issue:`25625`) + + +.. _whatsnew311-dataclasses: dataclasses ----------- @@ -629,11 +636,15 @@ dataclasses :class:`dict`, :class:`list` or :class:`set`. (Contributed by Eric V. Smith in :issue:`44674`.) + +.. _whatsnew311-datetime: + datetime -------- * Add :attr:`datetime.UTC`, a convenience alias for :attr:`datetime.timezone.utc`. (Contributed by Kabir Kwatra in :gh:`91973`.) + * :meth:`datetime.date.fromisoformat`, :meth:`datetime.time.fromisoformat` and :meth:`datetime.datetime.fromisoformat` can now be used to parse most ISO 8601 formats (barring only those that support fractional hours and minutes). @@ -658,7 +669,7 @@ enum (used by :func:`str`, :func:`format` and :term:`f-string`\s). * Changed :class:`~enum.IntEnum`, :class:`~enum.IntFlag` and :class:`~enum.StrEnum` - to now inherit from :class:`ReprEnum`, + to now inherit from :class:`~enum.ReprEnum`, so their :func:`str` output now matches :func:`format` (both ``str(AnIntEnum.ONE)`` and ``format(AnIntEnum.ONE)`` return ``'1'``, whereas before ``str(AnIntEnum.ONE)`` returned ``'AnIntEnum.ONE'``. @@ -708,6 +719,18 @@ enum inverted flags are coerced to their positive equivalent. +.. _whatsnew311-fcntl: + +fcntl +----- + +* On FreeBSD, the :data:`!F_DUP2FD` and :data:`!F_DUP2FD_CLOEXEC` flags respectively + are supported, the former equals to ``dup2`` usage while the latter set + the ``FD_CLOEXEC`` flag in addition. + + +.. _whatsnew311-fractions: + fractions --------- @@ -718,6 +741,9 @@ fractions that an ``isinstance(some_fraction, typing.SupportsInt)`` check passes. (Contributed by Mark Dickinson in :issue:`44547`.) + +.. _whatsnew311-functools: + functools --------- @@ -748,6 +774,9 @@ functools (Contributed by Yurii Karabas in :issue:`46014`.) + +.. _whatsnew311-hashlib: + hashlib ------- @@ -766,6 +795,9 @@ hashlib of files or file-like objects. (Contributed by Christian Heimes in :gh:`89313`.) + +.. _whatsnew311-idle: + IDLE and idlelib ---------------- @@ -803,6 +835,9 @@ inspect (Contributed by Pablo Galindo in :gh:`88116`.) + +.. _whatsnew311-locale: + locale ------ @@ -830,6 +865,8 @@ logging (Contributed by Kirill Pinchuk in :gh:`88457`.) +.. _whatsnew311-math: + math ---- @@ -849,6 +886,8 @@ math (Contributed by Victor Stinner in :issue:`46917`.) +.. _whatsnew311-operator: + operator -------- @@ -857,6 +896,8 @@ operator (Contributed by Antony Lee in :issue:`44019`.) +.. _whatsnew311-os: + os -- @@ -865,6 +906,8 @@ os (Contributed by Dong-hee Na in :issue:`44611`.) +.. _whatsnew311-pathlib: + pathlib ------- @@ -873,6 +916,9 @@ pathlib :data:`~os.sep` or :data:`~os.altsep`. (Contributed by Eisuke Kawasima in :issue:`22276` and :issue:`33392`.) + +.. _whatsnew311-re: + re -- @@ -880,6 +926,9 @@ re ``?+``, ``{m,n}+``) are now supported in regular expressions. (Contributed by Jeffrey C. Jacobs and Serhiy Storchaka in :issue:`433030`.) + +.. _whatsnew311-shutil: + shutil ------ @@ -887,6 +936,8 @@ shutil (Contributed by Serhiy Storchaka in :issue:`46245`.) +.. _whatsnew311-socket: + socket ------ @@ -898,6 +949,9 @@ socket instead of only raising the last error. (Contributed by Irit Katriel in :issue:`29980`.) + +.. _whatsnew311-sqlite3: + sqlite3 ------- @@ -961,13 +1015,15 @@ string (Contributed by Ben Kehoe in :gh:`90465`.) +.. _whatsnew311-sys: + sys --- * :func:`sys.exc_info` now derives the ``type`` and ``traceback`` fields from the ``value`` (the exception instance), so when an exception is modified while it is being handled, the changes are reflected in - the results of subsequent calls to :func:`exc_info`. + the results of subsequent calls to :func:`!exc_info`. (Contributed by Irit Katriel in :issue:`45711`.) * Add :func:`sys.exception` which returns the active exception instance @@ -978,6 +1034,8 @@ sys (Contributed by Victor Stinner in :gh:`57684`.) +.. _whatsnew311-sysconfig: + sysconfig --------- @@ -995,6 +1053,21 @@ sysconfig (Contributed by Miro Hrončok in :issue:`45413`.) +.. _whatsnew311-tempfile: + +tempfile +-------- + +* :class:`~tempfile.SpooledTemporaryFile` objects now fully implement the methods + of :class:`io.BufferedIOBase` or :class:`io.TextIOBase` + (depending on file mode). + This lets them work correctly with APIs that expect file-like objects, + such as compression modules. + (Contributed by Carey Metcalfe in :gh:`70363`.) + + +.. _whatsnew311-threading: + threading --------- @@ -1006,6 +1079,8 @@ threading (Contributed by Victor Stinner in :issue:`41710`.) +.. _whatsnew311-time: + time ---- @@ -1023,6 +1098,18 @@ time (Contributed by Benjamin Szőke, Dong-hee Na, Eryk Sun and Victor Stinner in :issue:`21302` and :issue:`45429`.) +.. _whatsnew311-tkinter: + +tkinter +------- + +* Added method ``info_patchlevel()`` which returns the exact version of + the Tcl library as a named tuple similar to :data:`sys.version_info`. + (Contributed by Serhiy Storchaka in :gh:`91827`.) + + +.. _whatsnew311-traceback: + traceback --------- @@ -1036,6 +1123,8 @@ traceback (Contributed by Irit Katriel in :issue:`33809`.) +.. _whatsnew311-typing: + typing ------ @@ -1076,7 +1165,7 @@ For major changes, see :ref:`new-feat-related-type-hints-311`. to clear all registered overloads of a function. (Contributed by Jelle Zijlstra in :gh:`89263`.) -* The :meth:`__init__` method of :class:`~typing.Protocol` subclasses +* The :meth:`~object.__init__` method of :class:`~typing.Protocol` subclasses is now preserved. (Contributed by Adrian Garcia Badarasco in :gh:`88970`.) * The representation of empty tuple types (``Tuple[()]``) is simplified. @@ -1105,19 +1194,16 @@ For major changes, see :ref:`new-feat-related-type-hints-311`. by Nikita Sobolev in :gh:`90729`.) -tkinter -------- - -* Added method ``info_patchlevel()`` which returns the exact version of - the Tcl library as a named tuple similar to :data:`sys.version_info`. - (Contributed by Serhiy Storchaka in :gh:`91827`.) - +.. _whatsnew311-unicodedata: unicodedata ----------- -* The Unicode database has been updated to version 14.0.0. (Contributed by Benjamin Peterson in :issue:`45190`). +* The Unicode database has been updated to version 14.0.0. + (Contributed by Benjamin Peterson in :issue:`45190`). + +.. _whatsnew311-unittest: unittest -------- @@ -1131,6 +1217,8 @@ unittest (Contributed by Serhiy Storchaka in :issue:`45046`.) +.. _whatsnew311-venv: + venv ---- @@ -1144,6 +1232,9 @@ venv Third party code that also creates new virtual environments should do the same. (Contributed by Miro Hrončok in :issue:`45413`.) + +.. _whatsnew311-warnings: + warnings -------- @@ -1170,14 +1261,6 @@ zipfile (Contributed by Miguel Brito in :gh:`88261`.) -fcntl ------ - -* On FreeBSD, the :attr:`F_DUP2FD` and :attr:`F_DUP2FD_CLOEXEC` flags respectively - are supported, the former equals to ``dup2`` usage while the latter set - the ``FD_CLOEXEC`` flag in addition. - - .. _whatsnew311-optimizations: Optimizations @@ -1481,58 +1564,100 @@ contributors are volunteers from the community. CPython bytecode changes ======================== -* The bytecode now contains inline cache entries, which take the form of - :opcode:`CACHE` instructions. Many opcodes expect to be followed by an exact - number of caches, and instruct the interpreter to skip over them at runtime. - Populated caches can look like arbitrary instructions, so great care should be - taken when reading or modifying raw, adaptive bytecode containing quickened - data. +The bytecode now contains inline cache entries, +which take the form of the newly-added :opcode:`CACHE` instructions. +Many opcodes expect to be followed by an exact number of caches, +and instruct the interpreter to skip over them at runtime. +Populated caches can look like arbitrary instructions, +so great care should be taken when reading or modifying +raw, adaptive bytecode containing quickened data. -* Replaced all numeric ``BINARY_*`` and ``INPLACE_*`` instructions with a single - :opcode:`BINARY_OP` implementation. -* Replaced the three call instructions: :opcode:`CALL_FUNCTION`, - :opcode:`CALL_FUNCTION_KW` and :opcode:`CALL_METHOD` with - :opcode:`PUSH_NULL`, :opcode:`PRECALL`, :opcode:`CALL`, - and :opcode:`KW_NAMES`. - This decouples the argument shifting for methods from the handling of - keyword arguments and allows better specialization of calls. +.. _whatsnew311-added-opcodes: -* Removed ``COPY_DICT_WITHOUT_KEYS`` and ``GEN_START``. +New opcodes +----------- -* :opcode:`MATCH_CLASS` and :opcode:`MATCH_KEYS` no longer push an additional - boolean value indicating whether the match succeeded or failed. Instead, they - indicate failure with :const:`None` (where a tuple of extracted values would - otherwise be). +* :opcode:`ASYNC_GEN_WRAP`, :opcode:`RETURN_GENERATOR` and :opcode:`SEND`, + used in generators and co-routines. -* Replace several stack manipulation instructions (``DUP_TOP``, ``DUP_TOP_TWO``, - ``ROT_TWO``, ``ROT_THREE``, ``ROT_FOUR``, and ``ROT_N``) with new - :opcode:`COPY` and :opcode:`SWAP` instructions. +* :opcode:`COPY_FREE_VARS`, + which avoids needing special caller-side code for closures. -* Replaced :opcode:`JUMP_IF_NOT_EXC_MATCH` by :opcode:`CHECK_EXC_MATCH` which - performs the check but does not jump. +* :opcode:`JUMP_BACKWARD_NO_INTERRUPT`, + for use in certain loops where handling interrupts is undesirable. -* Replaced :opcode:`JUMP_IF_NOT_EG_MATCH` by :opcode:`CHECK_EG_MATCH` which - performs the check but does not jump. +* :opcode:`MAKE_CELL`, to create :ref:`cell-objects`. -* Replaced :opcode:`JUMP_ABSOLUTE` by the relative :opcode:`JUMP_BACKWARD`. +* :opcode:`CHECK_EG_MATCH` and :opcode:`PREP_RERAISE_STAR`, + to handle the :ref:`new exception groups and except* ` + added in :pep:`654`. -* Added :opcode:`JUMP_BACKWARD_NO_INTERRUPT`, which is used in certain loops where it - is undesirable to handle interrupts. +* :opcode:`PUSH_EXC_INFO`, for use in exception handlers. -* Replaced :opcode:`POP_JUMP_IF_TRUE` and :opcode:`POP_JUMP_IF_FALSE` by - the relative :opcode:`POP_JUMP_FORWARD_IF_TRUE`, :opcode:`POP_JUMP_BACKWARD_IF_TRUE`, - :opcode:`POP_JUMP_FORWARD_IF_FALSE` and :opcode:`POP_JUMP_BACKWARD_IF_FALSE`. +* :opcode:`RESUME`, a no-op, + for internal tracing, debugging and optimization checks. -* Added :opcode:`POP_JUMP_FORWARD_IF_NOT_NONE`, :opcode:`POP_JUMP_BACKWARD_IF_NOT_NONE`, - :opcode:`POP_JUMP_FORWARD_IF_NONE` and :opcode:`POP_JUMP_BACKWARD_IF_NONE` - opcodes to speed up conditional jumps. -* :opcode:`JUMP_IF_TRUE_OR_POP` and :opcode:`JUMP_IF_FALSE_OR_POP` are now - relative rather than absolute. +.. _whatsnew311-replaced-opcodes: -* :opcode:`RESUME` has been added. It is a no-op. Performs internal tracing, - debugging and optimization checks. +Replaced opcodes +---------------- + ++------------------------------------+-----------------------------------+-----------------------------------------+ +| Replaced Opcode(s) | New Opcode(s) | Notes | ++====================================+===================================+=========================================+ +| | :opcode:`!BINARY_*` | :opcode:`BINARY_OP` | Replaced all numeric binary/in-place | +| | :opcode:`!INPLACE_*` | | opcodes with a single opcode | ++------------------------------------+-----------------------------------+-----------------------------------------+ +| | :opcode:`!CALL_FUNCTION` | | :opcode:`CALL` | Decouples argument shifting for methods | +| | :opcode:`!CALL_FUNCTION_KW` | | :opcode:`KW_NAMES` | from handling of keyword arguments; | +| | :opcode:`!CALL_METHOD` | | :opcode:`PRECALL` | allows better specialization of calls | +| | | :opcode:`PUSH_NULL` | | ++------------------------------------+-----------------------------------+-----------------------------------------+ +| | :opcode:`!DUP_TOP` | | :opcode:`COPY` | Stack manipulation instructions | +| | :opcode:`!DUP_TOP_TWO` | | :opcode:`SWAP` | | +| | :opcode:`!ROT_TWO` | | | +| | :opcode:`!ROT_THREE` | | | +| | :opcode:`!ROT_FOUR` | | | +| | :opcode:`!ROT_N` | | | ++------------------------------------+-----------------------------------+-----------------------------------------+ +| | :opcode:`!JUMP_IF_NOT_EXC_MATCH` | | :opcode:`CHECK_EXC_MATCH` | Now performs check but doesn't jump | ++------------------------------------+-----------------------------------+-----------------------------------------+ +| | :opcode:`!JUMP_ABSOLUTE` | | :opcode:`JUMP_BACKWARD` | See [#bytecode-jump]_; | +| | :opcode:`!POP_JUMP_IF_FALSE` | | :opcode:`POP_JUMP_BACKWARD_IF_* | ``TRUE``, ``FALSE``, | +| | :opcode:`!POP_JUMP_IF_TRUE` | ` | ``NONE`` and ``NOT_NONE`` variants | +| | | :opcode:`POP_JUMP_FORWARD_IF_* | for each direction | +| | ` | | ++------------------------------------+-----------------------------------+-----------------------------------------+ +| | :opcode:`!SETUP_WITH` | :opcode:`BEFORE_WITH` | :keyword:`with` block setup | +| | :opcode:`!SETUP_ASYNC_WITH` | | | ++------------------------------------+-----------------------------------+-----------------------------------------+ + +.. [#bytecode-jump] All jump opcodes are now relative, including the + existing :opcode:`JUMP_IF_TRUE_OR_POP` and :opcode:`JUMP_IF_FALSE_OR_POP`. + The argument is now an offset from the current instruction + rather than an absolute location. + + +.. _whatsnew311-changed-opcodes: +.. _whatsnew311-removed-opcodes: +.. _whatsnew311-changed-removed-opcodes: + +Changed/removed opcodes +----------------------- + +* Changed :opcode:`MATCH_CLASS` and :opcode:`MATCH_KEYS` + to no longer push an additional boolean value to indicate success/failure. + Instead, ``None`` is pushed on failure + in place of the tuple of extracted values. + +* Changed opcodes that work with exceptions to reflect them + now being represented as one item on the stack instead of three + (see :gh:`89874`). + +* Removed :opcode:`!COPY_DICT_WITHOUT_KEYS`, :opcode:`!GEN_START`, + :opcode:`!POP_BLOCK`, :opcode:`!SETUP_FINALLY` and :opcode:`!YIELD_FROM`. .. _whatsnew311-deprecated: @@ -1545,78 +1670,107 @@ This section lists Python APIs that have been deprecated in Python 3.11. Deprecated C APIs are :ref:`listed separately `. + +.. _whatsnew311-deprecated-language: +.. _whatsnew311-deprecated-builtins: + +Language/Builtins +----------------- + * Chaining :class:`classmethod` descriptors (introduced in :issue:`19072`) is now deprecated. It can no longer be used to wrap other descriptors such as :class:`property`. The core design of this feature was flawed and caused a number of downstream problems. To "pass-through" a - :class:`classmethod`, consider using the ``__wrapped__`` attribute + :class:`classmethod`, consider using the :attr:`!__wrapped__` attribute that was added in Python 3.10. (Contributed by Raymond Hettinger in :gh:`89519`.) -* Octal escapes in string and bytes literals with value larger than ``0o377`` now - produce :exc:`DeprecationWarning`. - In a future Python version they will be a :exc:`SyntaxWarning` and +* Octal escapes in string and bytes literals with values larger than ``0o377`` + (255 in decimal) now produce a :exc:`DeprecationWarning`. + In a future Python version, they will raise a :exc:`SyntaxWarning` and eventually a :exc:`SyntaxError`. (Contributed by Serhiy Storchaka in :gh:`81548`.) -* The :mod:`lib2to3` package and ``2to3`` tool are now deprecated and may not - be able to parse Python 3.10 or newer. See the :pep:`617` (New PEG parser for - CPython). (Contributed by Victor Stinner in :issue:`40360`.) +* The delegation of :func:`int` to :meth:`~object.__trunc__` is now deprecated. + Calling ``int(a)`` when ``type(a)`` implements :meth:`!__trunc__` but not + :meth:`~object.__int__` or :meth:`~object.__index__` now raises + a :exc:`DeprecationWarning`. + (Contributed by Zackery Spytz in :issue:`44977`.) -* Undocumented modules ``sre_compile``, ``sre_constants`` and ``sre_parse`` - are now deprecated. - (Contributed by Serhiy Storchaka in :issue:`47152`.) -* :class:`webbrowser.MacOSX` is deprecated and will be removed in Python 3.13. - It is untested and undocumented and also not used by webbrowser itself. - (Contributed by Dong-hee Na in :issue:`42255`.) +.. _whatsnew311-deprecated-modules: -* The behavior of returning a value from a :class:`~unittest.TestCase` and - :class:`~unittest.IsolatedAsyncioTestCase` test methods (other than the - default ``None`` value), is now deprecated. +Modules +------- -* Deprecated the following :mod:`unittest` functions, scheduled for removal in - Python 3.13: +.. _whatsnew311-pep594: - * :func:`unittest.findTestCases` - * :func:`unittest.makeSuite` - * :func:`unittest.getTestCaseNames` +* :pep:`594` led to the deprecations of the following modules + slated for removal in Python 3.13: - Use :class:`~unittest.TestLoader` method instead: + +---------------------+---------------------+---------------------+---------------------+---------------------+ + | :mod:`aifc` | :mod:`chunk` | :mod:`msilib` | :mod:`pipes` | :mod:`telnetlib` | + +---------------------+---------------------+---------------------+---------------------+---------------------+ + | :mod:`audioop` | :mod:`crypt` | :mod:`nis` | :mod:`sndhdr` | :mod:`uu` | + +---------------------+---------------------+---------------------+---------------------+---------------------+ + | :mod:`cgi` | :mod:`imghdr` | :mod:`nntplib` | :mod:`spwd` | :mod:`xdrlib` | + +---------------------+---------------------+---------------------+---------------------+---------------------+ + | :mod:`cgitb` | :mod:`mailcap` | :mod:`ossaudiodev` | :mod:`sunau` | | + +---------------------+---------------------+---------------------+---------------------+---------------------+ - * :meth:`unittest.TestLoader.loadTestsFromModule` - * :meth:`unittest.TestLoader.loadTestsFromTestCase` - * :meth:`unittest.TestLoader.getTestCaseNames` + (Contributed by Brett Cannon in :issue:`47061` and Victor Stinner in + :gh:`68966`.) - (Contributed by Erlend E. Aasland in :issue:`5846`.) +* The :mod:`asynchat`, :mod:`asyncore` and :mod:`smtpd` modules have been + deprecated since at least Python 3.6. Their documentation and deprecation + warnings have now been updated to note they will be removed in Python 3.12. + (Contributed by Hugo van Kemenade in :issue:`47022`.) -* The :meth:`turtle.RawTurtle.settiltangle` is deprecated since Python 3.1, - it now emits a deprecation warning and will be removed in Python 3.13. Use - :meth:`turtle.RawTurtle.tiltangle` instead (it was earlier incorrectly marked - as deprecated, its docstring is now corrected). - (Contributed by Hugo van Kemenade in :issue:`45837`.) +* The :mod:`lib2to3` package and :ref:`2to3 <2to3-reference>` tool + are now deprecated and may not be able to parse Python 3.10 or newer. + See :pep:`617`, introducing the new PEG parser, for details. + (Contributed by Victor Stinner in :issue:`40360`.) -* The delegation of :func:`int` to :meth:`__trunc__` is now deprecated. Calling - ``int(a)`` when ``type(a)`` implements :meth:`__trunc__` but not - :meth:`__int__` or :meth:`__index__` now raises a :exc:`DeprecationWarning`. - (Contributed by Zackery Spytz in :issue:`44977`.) +* Undocumented modules :mod:`!sre_compile`, :mod:`!sre_constants` + and :mod:`!sre_parse` are now deprecated. + (Contributed by Serhiy Storchaka in :issue:`47152`.) + + +.. _whatsnew311-deprecated-stdlib: + +Standard Library +---------------- * The following have been deprecated in :mod:`configparser` since Python 3.2. - Their deprecation warnings have now been updated to note they will removed in - Python 3.12: + Their deprecation warnings have now been updated to note they will be removed + in Python 3.12: - * the :class:`configparser.SafeConfigParser` class - * the :attr:`configparser.ParsingError.filename` property + * the :class:`!configparser.SafeConfigParser` class + * the :attr:`!configparser.ParsingError.filename` property * the :meth:`configparser.RawConfigParser.readfp` method (Contributed by Hugo van Kemenade in :issue:`45173`.) -* :class:`configparser.LegacyInterpolation` has been deprecated in the docstring - since Python 3.2. It now emits a :exc:`DeprecationWarning` and will be removed +* :class:`!configparser.LegacyInterpolation` has been deprecated in the docstring + since Python 3.2, and is not listed in the :mod:`configparser` documentation. + It now emits a :exc:`DeprecationWarning` and will be removed in Python 3.13. Use :class:`configparser.BasicInterpolation` or :class:`configparser.ExtendedInterpolation` instead. (Contributed by Hugo van Kemenade in :issue:`46607`.) +* The older set of :mod:`importlib.resources` functions were deprecated + in favor of the replacements added in Python 3.9 + and will be removed in a future Python version, + due to not supporting resources located within package subdirectories: + + * :func:`importlib.resources.contents` + * :func:`importlib.resources.is_resource` + * :func:`importlib.resources.open_binary` + * :func:`importlib.resources.open_text` + * :func:`importlib.resources.read_binary` + * :func:`importlib.resources.read_text` + * :func:`importlib.resources.path` + * The :func:`locale.getdefaultlocale` function is deprecated and will be removed in Python 3.13. Use :func:`locale.setlocale`, :func:`locale.getpreferredencoding(False) ` and @@ -1627,46 +1781,25 @@ Deprecated C APIs are :ref:`listed separately `. removed in Python 3.13. Use ``locale.setlocale(locale.LC_ALL, "")`` instead. (Contributed by Victor Stinner in :gh:`90817`.) -.. _whatsnew311-pep594: - -* :pep:`594` led to the deprecations of the following modules which are - slated for removal in Python 3.13: - - * :mod:`aifc` - * :mod:`audioop` - * :mod:`cgi` - * :mod:`cgitb` - * :mod:`chunk` - * :mod:`crypt` - * :mod:`imghdr` - * :mod:`mailcap` - * :mod:`msilib` - * :mod:`nis` - * :mod:`nntplib` - * :mod:`ossaudiodev` - * :mod:`pipes` - * :mod:`sndhdr` - * :mod:`spwd` - * :mod:`sunau` - * :mod:`telnetlib` - * :mod:`uu` - * :mod:`xdrlib` - - (Contributed by Brett Cannon in :issue:`47061` and Victor Stinner in - :gh:`68966`.) +* Stricter rules will now be applied for numerical group references + and group names in :ref:`regular expressions `. + Only sequences of ASCII digits will now be accepted as a numerical reference, + and the group name in :class:`bytes` patterns and replacement strings + can only contain ASCII letters, digits and underscores. + For now, a deprecation warning is raised for syntax violating these rules. + (Contributed by Serhiy Storchaka in :gh:`91760`.) -* The :mod:`asynchat`, :mod:`asyncore` and :mod:`smtpd` modules have been - deprecated since at least Python 3.6. Their documentation and deprecation - warnings have now been updated to note they will removed in Python 3.12. - (Contributed by Hugo van Kemenade in :issue:`47022`.) +* In the :mod:`re` module, the :func:`!re.template` function + and the corresponding :data:`!re.TEMPLATE` and :data:`!re.T` flags + are deprecated, as they were undocumented and lacked an obvious purpose. + They will be removed in Python 3.13. + (Contributed by Serhiy Storchaka and Miro Hrončok in :gh:`92728`.) -* More strict rules will be applied now applied for numerical group references - and group names in regular expressions in future Python versions. - Only sequence of ASCII digits will be now accepted as a numerical reference. - The group name in bytes patterns and replacement strings could only - contain ASCII letters and digits and underscore. - For now, a deprecation warning is raised for such syntax. - (Contributed by Serhiy Storchaka in :gh:`91760`.) +* :func:`turtle.settiltangle` has been deprecated since Python 3.1; + it now emits a deprecation warning and will be removed in Python 3.13. Use + :func:`turtle.tiltangle` instead (it was earlier incorrectly marked + as deprecated, and its docstring is now corrected). + (Contributed by Hugo van Kemenade in :issue:`45837`.) * :class:`typing.Text`, which exists solely to provide compatibility support between Python 2 and Python 3 code, is now deprecated. Its removal is @@ -1674,14 +1807,32 @@ Deprecated C APIs are :ref:`listed separately `. wherever possible. (Contributed by Alex Waygood in :gh:`92332`.) -* The keyword argument syntax for constructing :data:`~typing.TypedDict` types +* The keyword argument syntax for constructing :data:`typing.TypedDict` types is now deprecated. Support will be removed in Python 3.13. (Contributed by Jingchen Ye in :gh:`90224`.) -* The :func:`re.template` function and the corresponding :const:`re.TEMPLATE` - and :const:`re.T` flags are deprecated, as they were undocumented and - lacked an obvious purpose. They will be removed in Python 3.13. - (Contributed by Serhiy Storchaka and Miro Hrončok in :gh:`92728`.) +* :class:`!webbrowser.MacOSX` is deprecated and will be removed in Python 3.13. + It is untested, undocumented, and not used by :mod:`webbrowser` itself. + (Contributed by Dong-hee Na in :issue:`42255`.) + +* The behavior of returning a value from a :class:`~unittest.TestCase` and + :class:`~unittest.IsolatedAsyncioTestCase` test methods (other than the + default ``None`` value) is now deprecated. + +* Deprecated the following not-formally-documented :mod:`unittest` functions, + scheduled for removal in Python 3.13: + + * :func:`!unittest.findTestCases` + * :func:`!unittest.makeSuite` + * :func:`!unittest.getTestCaseNames` + + Use :class:`~unittest.TestLoader` methods instead: + + * :meth:`unittest.TestLoader.loadTestsFromModule` + * :meth:`unittest.TestLoader.loadTestsFromTestCase` + * :meth:`unittest.TestLoader.getTestCaseNames` + + (Contributed by Erlend E. Aasland in :issue:`5846`.) .. _whatsnew311-pending-removal: @@ -1696,33 +1847,56 @@ and will be removed in Python 3.12. C APIs pending removal are :ref:`listed separately `. -* :class:`pkgutil.ImpImporter` -* :class:`pkgutil.ImpLoader` -* :envvar:`PYTHONTHREADDEBUG` +* The :mod:`asynchat` module +* The :mod:`asyncore` module +* The :ref:`entire distutils package ` +* The :mod:`imp` module +* The :class:`typing.io ` namespace +* The :class:`typing.re ` namespace +* :func:`!cgi.log` * :func:`importlib.find_loader` -* :func:`importlib.util.module_for_loader` -* :func:`importlib.util.set_loader_wrapper` -* :func:`importlib.util.set_package_wrapper` * :meth:`importlib.abc.Loader.module_repr` -* :meth:`importlib.abc.Loadermodule_repr` -* :meth:`importlib.abc.MetaPathFinder.find_module` * :meth:`importlib.abc.MetaPathFinder.find_module` * :meth:`importlib.abc.PathEntryFinder.find_loader` * :meth:`importlib.abc.PathEntryFinder.find_module` -* :meth:`importlib.machinery.BuiltinImporter.find_module` -* :meth:`importlib.machinery.BuiltinLoader.module_repr` -* :meth:`importlib.machinery.FileFinder.find_loader` -* :meth:`importlib.machinery.FileFinder.find_module` -* :meth:`importlib.machinery.FrozenImporter.find_module` -* :meth:`importlib.machinery.FrozenLoader.module_repr` +* :meth:`!importlib.machinery.BuiltinImporter.find_module` +* :meth:`!importlib.machinery.BuiltinLoader.module_repr` +* :meth:`!importlib.machinery.FileFinder.find_loader` +* :meth:`!importlib.machinery.FileFinder.find_module` +* :meth:`!importlib.machinery.FrozenImporter.find_module` +* :meth:`!importlib.machinery.FrozenLoader.module_repr` * :meth:`importlib.machinery.PathFinder.find_module` -* :meth:`importlib.machinery.WindowsRegistryFinder.find_module` +* :meth:`!importlib.machinery.WindowsRegistryFinder.find_module` +* :func:`importlib.util.module_for_loader` +* :func:`!importlib.util.set_loader_wrapper` +* :func:`!importlib.util.set_package_wrapper` +* :class:`pkgutil.ImpImporter` +* :class:`pkgutil.ImpLoader` * :meth:`pathlib.Path.link_to` -* The entire :ref:`distutils namespace ` -* :func:`cgi.log` -* :func:`sqlite3.OptimizedUnicode` -* :func:`sqlite3.enable_shared_cache` - +* :func:`!sqlite3.enable_shared_cache` +* :func:`!sqlite3.OptimizedUnicode` +* :envvar:`PYTHONTHREADDEBUG` environment variable +* The following deprecated aliases in :mod:`unittest`: + + ============================ =============================== =============== + Deprecated alias Method Name Deprecated in + ============================ =============================== =============== + ``failUnless`` :meth:`.assertTrue` 3.1 + ``failIf`` :meth:`.assertFalse` 3.1 + ``failUnlessEqual`` :meth:`.assertEqual` 3.1 + ``failIfEqual`` :meth:`.assertNotEqual` 3.1 + ``failUnlessAlmostEqual`` :meth:`.assertAlmostEqual` 3.1 + ``failIfAlmostEqual`` :meth:`.assertNotAlmostEqual` 3.1 + ``failUnlessRaises`` :meth:`.assertRaises` 3.1 + ``assert_`` :meth:`.assertTrue` 3.2 + ``assertEquals`` :meth:`.assertEqual` 3.2 + ``assertNotEquals`` :meth:`.assertNotEqual` 3.2 + ``assertAlmostEquals`` :meth:`.assertAlmostEqual` 3.2 + ``assertNotAlmostEquals`` :meth:`.assertNotAlmostEqual` 3.2 + ``assertRegexpMatches`` :meth:`.assertRegex` 3.2 + ``assertRaisesRegexp`` :meth:`.assertRaisesRegex` 3.2 + ``assertNotRegexpMatches`` :meth:`.assertNotRegex` 3.5 + ============================ =============================== =============== .. _whatsnew311-removed: .. _whatsnew311-python-api-removed: @@ -1730,7 +1904,7 @@ C APIs pending removal are Removed ======= -This section lists Python APIs that have been removed in Python 3.12. +This section lists Python APIs that have been removed in Python 3.11. Removed C APIs are :ref:`listed separately `. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 70e45258654f41..f138fa5c0e9fb6 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2052,6 +2052,8 @@ tkinter The :mod:`tkinter.tix` module is now deprecated. :mod:`tkinter` users should use :mod:`tkinter.ttk` instead. +.. _whatsnew36-venv: + venv ~~~~ diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 7f85ff3ffa9120..4e2dbe3b539fa6 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -122,8 +122,8 @@ Positional-only parameters There is a new function parameter syntax ``/`` to indicate that some function parameters must be specified positionally and cannot be used as keyword arguments. This is the same notation shown by ``help()`` for C -functions annotated with Larry Hastings' `Argument Clinic -`_ tool. +functions annotated with Larry Hastings' +:ref:`Argument Clinic ` tool. In the following example, parameters *a* and *b* are positional-only, while *c* or *d* can be positional or keyword, and *e* or *f* are diff --git a/Grammar/python.gram b/Grammar/python.gram index 51f846a57f404b..bae8bc32242f9a 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -1247,8 +1247,10 @@ invalid_try_stmt: | a='try' ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'try' statement on line %d", a->lineno) } | 'try' ':' block !('except' | 'finally') { RAISE_SYNTAX_ERROR("expected 'except' or 'finally' block") } - | 'try' ':' block* ((except_block+ except_star_block) | (except_star_block+ except_block)) block* { - RAISE_SYNTAX_ERROR("cannot have both 'except' and 'except*' on the same 'try'") } + | 'try' ':' block* except_block+ a='except' b='*' expression ['as' NAME] ':' { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot have both 'except' and 'except*' on the same 'try'") } + | 'try' ':' block* except_star_block+ a='except' [expression ['as' NAME]] ':' { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot have both 'except' and 'except*' on the same 'try'") } invalid_except_stmt: | 'except' '*'? a=expression ',' expressions ['as' NAME ] ':' { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "multiple exception types must be parenthesized") } diff --git a/Include/dynamic_annotations.h b/Include/dynamic_annotations.h index 0bd1a833c2e5a5..4d4def9bf8983e 100644 --- a/Include/dynamic_annotations.h +++ b/Include/dynamic_annotations.h @@ -44,7 +44,7 @@ Actual implementation of these macros may differ depending on the dynamic analysis tool being used. - See http://code.google.com/p/data-race-test/ for more information. + See https://code.google.com/p/data-race-test/ for more information. This file supports the following dynamic analysis tools: - None (DYNAMIC_ANNOTATIONS_ENABLED is not defined or zero). @@ -140,7 +140,7 @@ of the mutex's critical sections individually using the annotations above. This annotation makes sense only for hybrid race detectors. For pure happens-before detectors this is a no-op. For more details see - http://code.google.com/p/data-race-test/wiki/PureHappensBeforeVsHybrid . */ + https://code.google.com/p/data-race-test/wiki/PureHappensBeforeVsHybrid . */ #define _Py_ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX(mu) \ AnnotateMutexIsUsedAsCondVar(__FILE__, __LINE__, mu) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 627401409a1f68..7f217c1db4fe61 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 11 -#define PY_MICRO_VERSION 0 +#define PY_MICRO_VERSION 1 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.11.0" +#define PY_VERSION "3.11.1" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/LICENSE b/LICENSE index 02a5145f0e3852..9838d443f9fc02 100644 --- a/LICENSE +++ b/LICENSE @@ -2,12 +2,12 @@ A. HISTORY OF THE SOFTWARE ========================== Python was created in the early 1990s by Guido van Rossum at Stichting -Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands +Mathematisch Centrum (CWI, see https://www.cwi.nl) in the Netherlands as a successor of a language called ABC. Guido remains Python's principal author, although it includes many contributions from others. In 1995, Guido continued his work on Python at the Corporation for -National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) +National Research Initiatives (CNRI, see https://www.cnri.reston.va.us) in Reston, Virginia where he released several versions of the software. @@ -19,7 +19,7 @@ https://www.python.org/psf/) was formed, a non-profit organization created specifically to own Python-related Intellectual Property. Zope Corporation was a sponsoring member of the PSF. -All Python releases are Open Source (see http://www.opensource.org for +All Python releases are Open Source (see https://opensource.org for the Open Source Definition). Historically, most, but not all, Python releases have also been GPL-compatible; the table below summarizes the various releases. diff --git a/Lib/argparse.py b/Lib/argparse.py index 1c5520c4b41bd1..77619088614e02 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1997,7 +1997,11 @@ def consume_optional(start_index): # arguments, try to parse more single-dash options out # of the tail of the option string chars = self.prefix_chars - if arg_count == 0 and option_string[1] not in chars: + if ( + arg_count == 0 + and option_string[1] not in chars + and explicit_arg != '' + ): action_tuples.append((action, [], option_string)) char = option_string[0] option_string = char + explicit_arg[0] diff --git a/Lib/ast.py b/Lib/ast.py index b0e1c41709fad8..623b9a1b805d0e 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -53,10 +53,12 @@ def parse(source, filename='', mode='exec', *, def literal_eval(node_or_string): """ - Safely evaluate an expression node or a string containing a Python + Evaluate an expression node or a string containing only a Python expression. The string or node provided may only consist of the following Python literal structures: strings, bytes, numbers, tuples, lists, dicts, sets, booleans, and None. + + Caution: A complex expression can overflow the C stack and cause a crash. """ if isinstance(node_or_string, str): node_or_string = parse(node_or_string.lstrip(" \t"), mode='eval') @@ -234,6 +236,12 @@ def increment_lineno(node, n=1): location in a file. """ for child in walk(node): + # TypeIgnore is a special case where lineno is not an attribute + # but rather a field of the node itself. + if isinstance(child, TypeIgnore): + child.lineno = getattr(child, 'lineno', 0) + n + continue + if 'lineno' in child._attributes: child.lineno = getattr(child, 'lineno', 0) + n if ( diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index fa00bf9a2ca090..8f3e1b3df2a389 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -577,9 +577,11 @@ async def shutdown_default_executor(self): def _do_shutdown(self, future): try: self._default_executor.shutdown(wait=True) - self.call_soon_threadsafe(future.set_result, None) + if not self.is_closed(): + self.call_soon_threadsafe(future.set_result, None) except Exception as ex: - self.call_soon_threadsafe(future.set_exception, ex) + if not self.is_closed(): + self.call_soon_threadsafe(future.set_exception, ex) def _check_running(self): if self.is_running(): @@ -593,12 +595,13 @@ def run_forever(self): self._check_closed() self._check_running() self._set_coroutine_origin_tracking(self._debug) - self._thread_id = threading.get_ident() old_agen_hooks = sys.get_asyncgen_hooks() - sys.set_asyncgen_hooks(firstiter=self._asyncgen_firstiter_hook, - finalizer=self._asyncgen_finalizer_hook) try: + self._thread_id = threading.get_ident() + sys.set_asyncgen_hooks(firstiter=self._asyncgen_firstiter_hook, + finalizer=self._asyncgen_finalizer_hook) + events._set_running_loop(self) while True: self._run_once() @@ -972,6 +975,8 @@ async def _connect_sock(self, exceptions, addr_info, local_addr_infos=None): if sock is not None: sock.close() raise + finally: + exceptions = my_exceptions = None async def create_connection( self, protocol_factory, host=None, port=None, @@ -1069,17 +1074,20 @@ async def create_connection( if sock is None: exceptions = [exc for sub in exceptions for exc in sub] - if len(exceptions) == 1: - raise exceptions[0] - else: - # If they all have the same str(), raise one. - model = str(exceptions[0]) - if all(str(exc) == model for exc in exceptions): + try: + if len(exceptions) == 1: raise exceptions[0] - # Raise a combined exception so the user can see all - # the various error messages. - raise OSError('Multiple exceptions: {}'.format( - ', '.join(str(exc) for exc in exceptions))) + else: + # If they all have the same str(), raise one. + model = str(exceptions[0]) + if all(str(exc) == model for exc in exceptions): + raise exceptions[0] + # Raise a combined exception so the user can see all + # the various error messages. + raise OSError('Multiple exceptions: {}'.format( + ', '.join(str(exc) for exc in exceptions))) + finally: + exceptions = None else: if sock is None: @@ -1872,6 +1880,8 @@ def _run_once(self): event_list = self._selector.select(timeout) self._process_events(event_list) + # Needed to break cycles when an exception occurs. + event_list = None # Handle 'later' callbacks that are ready. end_time = self.time() + self._clock_resolution diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 14d50519228814..e15bb4141fc02a 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -215,13 +215,11 @@ def _process_exited(self, returncode): # object. On Python 3.6, it is required to avoid a ResourceWarning. self._proc.returncode = returncode self._call(self._protocol.process_exited) - self._try_finish() + for p in self._pipes.values(): + if p is not None: + p.pipe.close() - # wake up futures waiting for wait() - for waiter in self._exit_waiters: - if not waiter.cancelled(): - waiter.set_result(returncode) - self._exit_waiters = None + self._try_finish() async def _wait(self): """Wait until the process exit and return the process return code. @@ -247,6 +245,11 @@ def _call_connection_lost(self, exc): try: self._protocol.connection_lost(exc) finally: + # wake up futures waiting for wait() + for waiter in self._exit_waiters: + if not waiter.cancelled(): + waiter.set_result(self._returncode) + self._exit_waiters = None self._loop = None self._proc = None self._protocol = None diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 0d26ea545baa5d..af3f9e970b5106 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -671,6 +671,21 @@ def get_event_loop(self): if (self._local._loop is None and not self._local._set_called and threading.current_thread() is threading.main_thread()): + stacklevel = 2 + try: + f = sys._getframe(1) + except AttributeError: + pass + else: + while f: + module = f.f_globals.get('__name__') + if not (module == 'asyncio' or module.startswith('asyncio.')): + break + f = f.f_back + stacklevel += 1 + import warnings + warnings.warn('There is no current event loop', + DeprecationWarning, stacklevel=stacklevel) self.set_event_loop(self.new_event_loop()) if self._local._loop is None: @@ -786,12 +801,13 @@ def get_event_loop(): def _get_event_loop(stacklevel=3): + # This internal method is going away in Python 3.12, left here only for + # backwards compatibility with 3.10.0 - 3.10.8 and 3.11.0. + # Similarly, this method's C equivalent in _asyncio is going away as well. + # See GH-99949 for more details. current_loop = _get_running_loop() if current_loop is not None: return current_loop - import warnings - warnings.warn('There is no current event loop', - DeprecationWarning, stacklevel=stacklevel) return get_event_loop_policy().get_event_loop() diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index be2458acb955ec..3a6b44a0910869 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -398,6 +398,8 @@ def _call_set_state(source): if dest_loop is None or dest_loop is source_loop: _set_state(destination, source) else: + if dest_loop.is_closed(): + return dest_loop.call_soon_threadsafe(_set_state, destination, source) destination.add_done_callback(_call_check_cancel) diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index ddb9daca026936..c6aab408fc7410 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -60,6 +60,7 @@ def __init__(self, loop, sock, protocol, waiter=None, self._pending_write = 0 self._conn_lost = 0 self._closing = False # Set when close() called. + self._called_connection_lost = False self._eof_written = False if self._server is not None: self._server._attach() @@ -136,7 +137,7 @@ def _force_close(self, exc): self._empty_waiter.set_result(None) else: self._empty_waiter.set_exception(exc) - if self._closing: + if self._closing and self._called_connection_lost: return self._closing = True self._conn_lost += 1 @@ -151,6 +152,8 @@ def _force_close(self, exc): self._loop.call_soon(self._call_connection_lost, exc) def _call_connection_lost(self, exc): + if self._called_connection_lost: + return try: self._protocol.connection_lost(exc) finally: @@ -166,6 +169,7 @@ def _call_connection_lost(self, exc): if server is not None: server._detach() self._server = None + self._called_connection_lost = True def get_write_buffer_size(self): size = self._pending_write diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index c9bbe2ac014351..8ab420d5bd719d 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -630,7 +630,11 @@ async def sock_connect(self, sock, address): fut = self.create_future() self._sock_connect(fut, sock, address) - return await fut + try: + return await fut + finally: + # Needed to break cycles when an exception occurs. + fut = None def _sock_connect(self, fut, sock, address): fd = sock.fileno() @@ -652,6 +656,8 @@ def _sock_connect(self, fut, sock, address): fut.set_exception(exc) else: fut.set_result(None) + finally: + fut = None def _sock_write_done(self, fd, fut, handle=None): if handle is None or not handle.cancelled(): @@ -675,6 +681,8 @@ def _sock_connect_cb(self, fut, sock, address): fut.set_exception(exc) else: fut.set_result(None) + finally: + fut = None async def sock_accept(self, sock): """Accept a connection. diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index de00953cc1d0f7..bbf9cad6bc7f86 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -107,8 +107,11 @@ def close(self): protocol's connection_lost() method will (eventually) called with None as its argument. """ - self._closed = True - self._ssl_protocol._start_shutdown() + if not self._closed: + self._closed = True + self._ssl_protocol._start_shutdown() + else: + self._ssl_protocol = None def __del__(self, _warnings=warnings): if not self._closed: @@ -196,12 +199,6 @@ def get_read_buffer_size(self): """Return the current size of the read buffer.""" return self._ssl_protocol._get_read_buffer_size() - def get_write_buffer_limits(self): - """Get the high and low watermarks for write flow control. - Return a tuple (low, high) where low and high are - positive number of bytes.""" - return self._ssl_protocol._transport.get_write_buffer_limits() - @property def _protocol_paused(self): # Required for sendfile fallback pause_writing/resume_writing logic diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 5d5e2a8a85dd48..911419e1769c17 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -128,11 +128,11 @@ async def __aexit__(self, et, exc, tb): # Exceptions are heavy objects that can have object # cycles (bad for GC); let's not keep a reference to # a bunch of them. - errors = self._errors - self._errors = None - - me = BaseExceptionGroup('unhandled errors in a TaskGroup', errors) - raise me from None + try: + me = BaseExceptionGroup('unhandled errors in a TaskGroup', self._errors) + raise me from None + finally: + self._errors = None def create_task(self, coro, *, name=None, context=None): if not self._entered: diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index cf7683fee64621..0495f332f3127a 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -223,7 +223,8 @@ async def _make_subprocess_transport(self, protocol, args, shell, return transp def _child_watcher_callback(self, pid, returncode, transp): - self.call_soon_threadsafe(transp._process_exited, returncode) + # Skip one iteration for callbacks to be executed + self.call_soon_threadsafe(self.call_soon, transp._process_exited, returncode) async def create_unix_connection( self, protocol_factory, path=None, *, @@ -799,12 +800,11 @@ class _UnixSubprocessTransport(base_subprocess.BaseSubprocessTransport): def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): stdin_w = None - if stdin == subprocess.PIPE: - # Use a socket pair for stdin, since not all platforms + if stdin == subprocess.PIPE and sys.platform.startswith('aix'): + # Use a socket pair for stdin on AIX, since it does not # support selecting read events on the write end of a # socket (which we use in order to detect closing of the - # other end). Notably this is needed on AIX, and works - # just fine on other platforms. + # other end). stdin, stdin_w = socket.socketpair() try: self._proc = subprocess.Popen( diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index 90b259cbafead2..8ee7e1ebd94ac6 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -439,7 +439,11 @@ def select(self, timeout=None): self._poll(timeout) tmp = self._results self._results = [] - return tmp + try: + return tmp + finally: + # Needed to break cycles when an exception occurs. + tmp = None def _result(self, value): fut = self._loop.create_future() @@ -841,6 +845,8 @@ def _poll(self, timeout=None): else: f.set_result(value) self._results.append(f) + finally: + f = None # Remove unregistered futures for ov in self._unregistered: diff --git a/Lib/codecs.py b/Lib/codecs.py index e6ad6e3a052364..3b173b612101e7 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -878,7 +878,8 @@ def open(filename, mode='r', encoding=None, errors='strict', buffering=-1): codecs. Output is also codec dependent and will usually be Unicode as well. - Underlying encoded files are always opened in binary mode. + If encoding is not None, then the + underlying encoded files are always opened in binary mode. The default file mode is 'r', meaning to open the file in read mode. encoding specifies the encoding which is to be used for the diff --git a/Lib/codeop.py b/Lib/codeop.py index 45a378baba4337..2213b69f231f92 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -56,22 +56,22 @@ def _maybe_compile(compiler, source, filename, symbol): if symbol != "eval": source = "pass" # Replace it with a 'pass' statement - try: - return compiler(source, filename, symbol) - except SyntaxError: # Let other compile() errors propagate. - pass - - # Catch syntax warnings after the first compile - # to emit warnings (SyntaxWarning, DeprecationWarning) at most once. + # Disable compiler warnings when checking for incomplete input. with warnings.catch_warnings(): - warnings.simplefilter("error") - + warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning)) try: - compiler(source + "\n", filename, symbol) - except SyntaxError as e: - if "incomplete input" in str(e): + compiler(source, filename, symbol) + except SyntaxError: # Let other compile() errors propagate. + try: + compiler(source + "\n", filename, symbol) return None - raise + except SyntaxError as e: + if "incomplete input" in str(e): + return None + # fallthrough + + return compiler(source, filename, symbol) + def _is_syntax_error(err1, err2): rep1 = repr(err1) diff --git a/Lib/ctypes/test/test_struct_fields.py b/Lib/ctypes/test/test_struct_fields.py index ee8415f3e630c2..fefeaea1496a3e 100644 --- a/Lib/ctypes/test/test_struct_fields.py +++ b/Lib/ctypes/test/test_struct_fields.py @@ -54,6 +54,15 @@ class X(Structure): x.char = b'a\0b\0' self.assertEqual(bytes(x), b'a\x00###') + def test_gh99275(self): + class BrokenStructure(Structure): + def __init_subclass__(cls, **kwargs): + cls._fields_ = [] # This line will fail, `stgdict` is not ready + + with self.assertRaisesRegex(TypeError, + 'ctypes state is not initialized'): + class Subclass(BrokenStructure): ... + # __set__ and __get__ should raise a TypeError in case their self # argument is not a ctype instance. def test___set__(self): diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 97ad2b8ed8a50d..f95d5a99a3a153 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -332,13 +332,13 @@ class Person(Structure): cls, msg = self.get_except(Person, b"Someone", (1, 2)) self.assertEqual(cls, RuntimeError) self.assertEqual(msg, - "(Phone) : " + "(Phone) TypeError: " "expected bytes, int found") cls, msg = self.get_except(Person, b"Someone", (b"a", b"b", b"c")) self.assertEqual(cls, RuntimeError) self.assertEqual(msg, - "(Phone) : too many initializers") + "(Phone) TypeError: too many initializers") def test_huge_field_name(self): # issue12881: segfault with large structure field names diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index a567a33d646f44..37e4ff702d61f9 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -412,13 +412,11 @@ def wrapper(self): def _create_fn(name, args, body, *, globals=None, locals=None, return_type=MISSING): - # Note that we mutate locals when exec() is called. Caller - # beware! The only callers are internal to this module, so no + # Note that we may mutate locals. Callers beware! + # The only callers are internal to this module, so no # worries about external callers. if locals is None: locals = {} - if 'BUILTINS' not in locals: - locals['BUILTINS'] = builtins return_annotation = '' if return_type is not MISSING: locals['_return_type'] = return_type @@ -444,7 +442,7 @@ def _field_assign(frozen, name, value, self_name): # self_name is what "self" is called in this function: don't # hard-code "self", since that might be a field name. if frozen: - return f'BUILTINS.object.__setattr__({self_name},{name!r},{value})' + return f'__dataclass_builtins_object__.__setattr__({self_name},{name!r},{value})' return f'{self_name}.{name}={value}' @@ -551,6 +549,7 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, locals.update({ 'MISSING': MISSING, '_HAS_DEFAULT_FACTORY': _HAS_DEFAULT_FACTORY, + '__dataclass_builtins_object__': object, }) body_lines = [] diff --git a/Lib/datetime.py b/Lib/datetime.py index 00ded32cc3e3cb..c3c2568f986594 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1030,7 +1030,11 @@ def ctime(self): self._day, self._year) def strftime(self, fmt): - "Format using strftime()." + """ + Format using strftime(). + + Example: "%d/%m/%Y, %H:%M:%S" + """ return _wrap_strftime(self, fmt, self.timetuple()) def __format__(self, fmt): diff --git a/Lib/distutils/tests/test_sysconfig.py b/Lib/distutils/tests/test_sysconfig.py index d1c472794c5ae5..6833d22af5fe2a 100644 --- a/Lib/distutils/tests/test_sysconfig.py +++ b/Lib/distutils/tests/test_sysconfig.py @@ -49,6 +49,7 @@ def test_get_config_vars(self): self.assertIsInstance(cvars, dict) self.assertTrue(cvars) + @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") def test_srcdir(self): # See Issues #15322, #15364. srcdir = sysconfig.get_config_var('srcdir') diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py index ea4058512fe366..bf98f513366b31 100644 --- a/Lib/encodings/idna.py +++ b/Lib/encodings/idna.py @@ -39,23 +39,21 @@ def nameprep(label): # Check bidi RandAL = [stringprep.in_table_d1(x) for x in label] - for c in RandAL: - if c: - # There is a RandAL char in the string. Must perform further - # tests: - # 1) The characters in section 5.8 MUST be prohibited. - # This is table C.8, which was already checked - # 2) If a string contains any RandALCat character, the string - # MUST NOT contain any LCat character. - if any(stringprep.in_table_d2(x) for x in label): - raise UnicodeError("Violation of BIDI requirement 2") - - # 3) If a string contains any RandALCat character, a - # RandALCat character MUST be the first character of the - # string, and a RandALCat character MUST be the last - # character of the string. - if not RandAL[0] or not RandAL[-1]: - raise UnicodeError("Violation of BIDI requirement 3") + if any(RandAL): + # There is a RandAL char in the string. Must perform further + # tests: + # 1) The characters in section 5.8 MUST be prohibited. + # This is table C.8, which was already checked + # 2) If a string contains any RandALCat character, the string + # MUST NOT contain any LCat character. + if any(stringprep.in_table_d2(x) for x in label): + raise UnicodeError("Violation of BIDI requirement 2") + # 3) If a string contains any RandALCat character, a + # RandALCat character MUST be the first character of the + # string, and a RandALCat character MUST be the last + # character of the string. + if not RandAL[0] or not RandAL[-1]: + raise UnicodeError("Violation of BIDI requirement 3") return label diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 4a6ba9cedf3d1c..1a2f57c07ba341 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -11,7 +11,7 @@ __all__ = ["version", "bootstrap"] _PACKAGE_NAMES = ('setuptools', 'pip') _SETUPTOOLS_VERSION = "65.5.0" -_PIP_VERSION = "22.3" +_PIP_VERSION = "22.3.1" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION, "py3"), ("pip", _PIP_VERSION, "py3"), diff --git a/Lib/ensurepip/_bundled/pip-22.3-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-22.3.1-py3-none-any.whl similarity index 94% rename from Lib/ensurepip/_bundled/pip-22.3-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-22.3.1-py3-none-any.whl index d6fccd9de92afc..c5b7753e757df2 100644 Binary files a/Lib/ensurepip/_bundled/pip-22.3-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-22.3.1-py3-none-any.whl differ diff --git a/Lib/enum.py b/Lib/enum.py index ff8f5cc453170e..1efddfa1a0acc6 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -22,14 +22,14 @@ class nonmember(object): """ - Protects item from becaming an Enum member during class creation. + Protects item from becoming an Enum member during class creation. """ def __init__(self, value): self.value = value class member(object): """ - Forces item to became an Enum member during class creation. + Forces item to become an Enum member during class creation. """ def __init__(self, value): self.value = value @@ -114,9 +114,12 @@ def _break_on_call_reduce(self, proto): setattr(obj, '__module__', '') def _iter_bits_lsb(num): - # num must be an integer + # num must be a positive integer + original = num if isinstance(num, Enum): num = num.value + if num < 0: + raise ValueError('%r is not a positive integer' % original) while num: b = num & (~num + 1) yield b @@ -171,7 +174,8 @@ class auto: """ Instances are replaced with an appropriate value in Enum class suites. """ - value = _auto_null + def __init__(self, value=_auto_null): + self.value = value def __repr__(self): return "auto(%r)" % self.value @@ -410,7 +414,7 @@ def __setitem__(self, key, value): value = value.value elif _is_descriptor(value): pass - # TODO: uncomment next three lines in 3.12 + # TODO: uncomment next three lines in 3.13 # elif _is_internal_class(self._cls_name, value): # # do nothing, name will be a normal attribute # pass @@ -421,15 +425,31 @@ def __setitem__(self, key, value): elif isinstance(value, member): # unwrap value here -- it will become a member value = value.value + non_auto_store = True + single = False if isinstance(value, auto): - if value.value == _auto_null: - value.value = self._generate_next_value( - key, 1, len(self._member_names), self._last_values[:], - ) - self._auto_called = True - value = value.value + single = True + value = (value, ) + if isinstance(value, tuple): + auto_valued = [] + for v in value: + if isinstance(v, auto): + non_auto_store = False + if v.value == _auto_null: + v.value = self._generate_next_value( + key, 1, len(self._member_names), self._last_values[:], + ) + self._auto_called = True + v = v.value + self._last_values.append(v) + auto_valued.append(v) + if single: + value = auto_valued[0] + else: + value = tuple(auto_valued) self._member_names[key] = None - self._last_values.append(value) + if non_auto_store: + self._last_values.append(value) super().__setitem__(key, value) def update(self, members, **more_members): @@ -1839,6 +1859,9 @@ def __call__(self, enumeration): if name in member_names: # not an alias continue + if alias.value < 0: + # negative numbers are not checked + continue values = list(_iter_bits_lsb(alias.value)) missed = [v for v in values if v not in member_values] if missed: diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index c514e0d382cbc7..65c45e2b17dfc0 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -1985,7 +1985,7 @@ class MozillaCookieJar(FileCookieJar): This class differs from CookieJar only in the format it uses to save and load cookies to and from a file. This class uses the Mozilla/Netscape - `cookies.txt' format. lynx uses this file format, too. + `cookies.txt' format. curl and lynx use this file format, too. Don't expect cookies saved while the browser is running to be noticed by the browser (in fact, Mozilla on unix will overwrite your saved cookies if diff --git a/Lib/http/server.py b/Lib/http/server.py index f2aeb65942020a..058ee47ba10e62 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -93,6 +93,7 @@ import html import http.client import io +import itertools import mimetypes import os import posixpath @@ -562,6 +563,11 @@ def log_error(self, format, *args): self.log_message(format, *args) + # https://en.wikipedia.org/wiki/List_of_Unicode_characters#Control_codes + _control_char_table = str.maketrans( + {c: fr'\x{c:02x}' for c in itertools.chain(range(0x20), range(0x7f,0xa0))}) + _control_char_table[ord('\\')] = r'\\' + def log_message(self, format, *args): """Log an arbitrary message. @@ -577,12 +583,16 @@ def log_message(self, format, *args): The client ip and current date/time are prefixed to every message. + Unicode control characters are replaced with escaped hex + before writing the output to stderr. + """ + message = format % args sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), - format%args)) + message.translate(self._control_char_table))) def version_string(self): """Return the server software version string.""" diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index 7fa7facf8cf780..e64e96f75e6cfc 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -4,6 +4,11 @@ Released on 2022-10-03 ========================= +gh-97527: Fix a bug in the previous bugfix that caused IDLE to not +start when run with 3.10.8, 3.12.0a1, and at least Microsoft Python +3.10.2288.0 installed without the Lib/test package. 3.11.0 was never +affected. + gh-65802: Document handling of extensions in Save As dialogs. gh-95191: Include prompts when saving Shell (interactive input/output). diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py index 9ca3a156f4b97f..bb07231cd590b6 100644 --- a/Lib/idlelib/config_key.py +++ b/Lib/idlelib/config_key.py @@ -41,32 +41,22 @@ def translate_key(key, modifiers): return f'Key-{key}' -class GetKeysDialog(Toplevel): +class GetKeysFrame(Frame): # Dialog title for invalid key sequence keyerror_title = 'Key Sequence Error' - def __init__(self, parent, title, action, current_key_sequences, - *, _htest=False, _utest=False): + def __init__(self, parent, action, current_key_sequences): """ parent - parent of this dialog - title - string which is the title of the popup dialog - action - string, the name of the virtual event these keys will be + action - the name of the virtual event these keys will be mapped to - current_key_sequences - list, a list of all key sequence lists + current_key_sequences - a list of all key sequence lists currently mapped to virtual events, for overlap checking - _htest - bool, change box location when running htest - _utest - bool, do not wait when running unittest """ - Toplevel.__init__(self, parent) - self.withdraw() # Hide while setting geometry. - self.configure(borderwidth=5) - self.resizable(height=False, width=False) - self.title(title) - self.transient(parent) - _setup_dialog(self) - self.grab_set() - self.protocol("WM_DELETE_WINDOW", self.cancel) + super().__init__(parent) + self['borderwidth'] = 2 + self['relief'] = 'sunken' self.parent = parent self.action = action self.current_key_sequences = current_key_sequences @@ -82,39 +72,14 @@ def __init__(self, parent, title, action, current_key_sequences, self.modifier_vars.append(variable) self.advanced = False self.create_widgets() - self.update_idletasks() - self.geometry( - "+%d+%d" % ( - parent.winfo_rootx() + - (parent.winfo_width()/2 - self.winfo_reqwidth()/2), - parent.winfo_rooty() + - ((parent.winfo_height()/2 - self.winfo_reqheight()/2) - if not _htest else 150) - ) ) # Center dialog over parent (or below htest box). - if not _utest: - self.deiconify() # Geometry set, unhide. - self.wait_window() def showerror(self, *args, **kwargs): # Make testing easier. Replace in #30751. messagebox.showerror(*args, **kwargs) def create_widgets(self): - self.frame = frame = Frame(self, borderwidth=2, relief='sunken') - frame.pack(side='top', expand=True, fill='both') - - frame_buttons = Frame(self) - frame_buttons.pack(side='bottom', fill='x') - - self.button_ok = Button(frame_buttons, text='OK', - width=8, command=self.ok) - self.button_ok.grid(row=0, column=0, padx=5, pady=5) - self.button_cancel = Button(frame_buttons, text='Cancel', - width=8, command=self.cancel) - self.button_cancel.grid(row=0, column=1, padx=5, pady=5) - # Basic entry key sequence. - self.frame_keyseq_basic = Frame(frame, name='keyseq_basic') + self.frame_keyseq_basic = Frame(self, name='keyseq_basic') self.frame_keyseq_basic.grid(row=0, column=0, sticky='nsew', padx=5, pady=5) basic_title = Label(self.frame_keyseq_basic, @@ -127,7 +92,7 @@ def create_widgets(self): basic_keys.pack(ipadx=5, ipady=5, fill='x') # Basic entry controls. - self.frame_controls_basic = Frame(frame) + self.frame_controls_basic = Frame(self) self.frame_controls_basic.grid(row=1, column=0, sticky='nsew', padx=5) # Basic entry modifiers. @@ -169,7 +134,7 @@ def create_widgets(self): self.button_clear.grid(row=2, column=0, columnspan=4) # Advanced entry key sequence. - self.frame_keyseq_advanced = Frame(frame, name='keyseq_advanced') + self.frame_keyseq_advanced = Frame(self, name='keyseq_advanced') self.frame_keyseq_advanced.grid(row=0, column=0, sticky='nsew', padx=5, pady=5) advanced_title = Label(self.frame_keyseq_advanced, justify='left', @@ -181,7 +146,7 @@ def create_widgets(self): self.advanced_keys.pack(fill='x') # Advanced entry help text. - self.frame_help_advanced = Frame(frame) + self.frame_help_advanced = Frame(self) self.frame_help_advanced.grid(row=1, column=0, sticky='nsew', padx=5) help_advanced = Label(self.frame_help_advanced, justify='left', text="Key bindings are specified using Tkinter keysyms as\n"+ @@ -196,7 +161,7 @@ def create_widgets(self): help_advanced.grid(row=0, column=0, sticky='nsew') # Switch between basic and advanced. - self.button_level = Button(frame, command=self.toggle_level, + self.button_level = Button(self, command=self.toggle_level, text='<< Basic Key Binding Entry') self.button_level.grid(row=2, column=0, stick='ew', padx=5, pady=5) self.toggle_level() @@ -257,7 +222,8 @@ def clear_key_seq(self): variable.set('') self.key_string.set('') - def ok(self, event=None): + def ok(self): + self.result = '' keys = self.key_string.get().strip() if not keys: self.showerror(title=self.keyerror_title, parent=self, @@ -265,13 +231,7 @@ def ok(self, event=None): return if (self.advanced or self.keys_ok(keys)) and self.bind_ok(keys): self.result = keys - self.grab_release() - self.destroy() - - def cancel(self, event=None): - self.result = '' - self.grab_release() - self.destroy() + return def keys_ok(self, keys): """Validity check on user's 'basic' keybinding selection. @@ -319,6 +279,73 @@ def bind_ok(self, keys): return True +class GetKeysWindow(Toplevel): + + def __init__(self, parent, title, action, current_key_sequences, + *, _htest=False, _utest=False): + """ + parent - parent of this dialog + title - string which is the title of the popup dialog + action - string, the name of the virtual event these keys will be + mapped to + current_key_sequences - list, a list of all key sequence lists + currently mapped to virtual events, for overlap checking + _htest - bool, change box location when running htest + _utest - bool, do not wait when running unittest + """ + super().__init__(parent) + self.withdraw() # Hide while setting geometry. + self['borderwidth'] = 5 + self.resizable(height=False, width=False) + # Needed for winfo_reqwidth(). + self.update_idletasks() + # Center dialog over parent (or below htest box). + x = (parent.winfo_rootx() + + (parent.winfo_width()//2 - self.winfo_reqwidth()//2)) + y = (parent.winfo_rooty() + + ((parent.winfo_height()//2 - self.winfo_reqheight()//2) + if not _htest else 150)) + self.geometry(f"+{x}+{y}") + + self.title(title) + self.frame = frame = GetKeysFrame(self, action, current_key_sequences) + self.protocol("WM_DELETE_WINDOW", self.cancel) + frame_buttons = Frame(self) + self.button_ok = Button(frame_buttons, text='OK', + width=8, command=self.ok) + self.button_cancel = Button(frame_buttons, text='Cancel', + width=8, command=self.cancel) + self.button_ok.grid(row=0, column=0, padx=5, pady=5) + self.button_cancel.grid(row=0, column=1, padx=5, pady=5) + frame.pack(side='top', expand=True, fill='both') + frame_buttons.pack(side='bottom', fill='x') + + self.transient(parent) + _setup_dialog(self) + self.grab_set() + if not _utest: + self.deiconify() # Geometry set, unhide. + self.wait_window() + + @property + def result(self): + return self.frame.result + + @result.setter + def result(self, value): + self.frame.result = value + + def ok(self, event=None): + self.frame.ok() + self.grab_release() + self.destroy() + + def cancel(self, event=None): + self.result = '' + self.grab_release() + self.destroy() + + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_config_key', verbosity=2, exit=False) diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 8e478d743fb767..cda7966d558a51 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -24,7 +24,7 @@ from tkinter import messagebox from idlelib.config import idleConf, ConfigChanges -from idlelib.config_key import GetKeysDialog +from idlelib.config_key import GetKeysWindow from idlelib.dynoption import DynOptionMenu from idlelib import macosx from idlelib.query import SectionName, HelpSource @@ -1397,7 +1397,7 @@ def get_new_keys(self): for event in key_set_changes: current_bindings[event] = key_set_changes[event].split() current_key_sequences = list(current_bindings.values()) - new_keys = GetKeysDialog(self, 'Get New Keys', bind_name, + new_keys = GetKeysWindow(self, 'Get New Keys', bind_name, current_key_sequences).result if new_keys: if self.keyset_source.get(): # Current key set is a built-in. diff --git a/Lib/idlelib/idle_test/test_config_key.py b/Lib/idlelib/idle_test/test_config_key.py index bf66cadf57cd3c..32f878b842b276 100644 --- a/Lib/idlelib/idle_test/test_config_key.py +++ b/Lib/idlelib/idle_test/test_config_key.py @@ -13,15 +13,13 @@ from idlelib.idle_test.mock_idle import Func from idlelib.idle_test.mock_tk import Mbox_func -gkd = config_key.GetKeysDialog - class ValidationTest(unittest.TestCase): "Test validation methods: ok, keys_ok, bind_ok." - class Validator(gkd): + class Validator(config_key.GetKeysFrame): def __init__(self, *args, **kwargs): - config_key.GetKeysDialog.__init__(self, *args, **kwargs) + super().__init__(*args, **kwargs) class list_keys_final: get = Func() self.list_keys_final = list_keys_final @@ -34,15 +32,14 @@ def setUpClass(cls): cls.root = Tk() cls.root.withdraw() keylist = [[''], ['', '']] - cls.dialog = cls.Validator( - cls.root, 'Title', '<>', keylist, _utest=True) + cls.dialog = cls.Validator(cls.root, '<>', keylist) @classmethod def tearDownClass(cls): - cls.dialog.cancel() + del cls.dialog cls.root.update_idletasks() cls.root.destroy() - del cls.dialog, cls.root + del cls.root def setUp(self): self.dialog.showerror.message = '' @@ -111,14 +108,14 @@ def setUpClass(cls): requires('gui') cls.root = Tk() cls.root.withdraw() - cls.dialog = gkd(cls.root, 'Title', '<>', [], _utest=True) + cls.dialog = config_key.GetKeysFrame(cls.root, '<>', []) @classmethod def tearDownClass(cls): - cls.dialog.cancel() + del cls.dialog cls.root.update_idletasks() cls.root.destroy() - del cls.dialog, cls.root + del cls.root def test_toggle_level(self): dialog = self.dialog @@ -130,7 +127,7 @@ def stackorder(): this can be used to check whether a frame is above or below another one. """ - for index, child in enumerate(dialog.frame.winfo_children()): + for index, child in enumerate(dialog.winfo_children()): if child._name == 'keyseq_basic': basic = index if child._name == 'keyseq_advanced': @@ -161,7 +158,7 @@ def stackorder(): class KeySelectionTest(unittest.TestCase): "Test selecting key on Basic frames." - class Basic(gkd): + class Basic(config_key.GetKeysFrame): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) class list_keys_final: @@ -179,14 +176,14 @@ def setUpClass(cls): requires('gui') cls.root = Tk() cls.root.withdraw() - cls.dialog = cls.Basic(cls.root, 'Title', '<>', [], _utest=True) + cls.dialog = cls.Basic(cls.root, '<>', []) @classmethod def tearDownClass(cls): - cls.dialog.cancel() + del cls.dialog cls.root.update_idletasks() cls.root.destroy() - del cls.dialog, cls.root + del cls.root def setUp(self): self.dialog.clear_key_seq() @@ -206,7 +203,7 @@ def test_get_modifiers(self): dialog.modifier_checkbuttons['foo'].invoke() eq(gm(), ['BAZ']) - @mock.patch.object(gkd, 'get_modifiers') + @mock.patch.object(config_key.GetKeysFrame, 'get_modifiers') def test_build_key_string(self, mock_modifiers): dialog = self.dialog key = dialog.list_keys_final @@ -227,7 +224,7 @@ def test_build_key_string(self, mock_modifiers): dialog.build_key_string() eq(string(), '') - @mock.patch.object(gkd, 'get_modifiers') + @mock.patch.object(config_key.GetKeysFrame, 'get_modifiers') def test_final_key_selected(self, mock_modifiers): dialog = self.dialog key = dialog.list_keys_final @@ -240,7 +237,7 @@ def test_final_key_selected(self, mock_modifiers): eq(string(), '') -class CancelTest(unittest.TestCase): +class CancelWindowTest(unittest.TestCase): "Simulate user clicking [Cancel] button." @classmethod @@ -248,21 +245,89 @@ def setUpClass(cls): requires('gui') cls.root = Tk() cls.root.withdraw() - cls.dialog = gkd(cls.root, 'Title', '<>', [], _utest=True) + cls.dialog = config_key.GetKeysWindow( + cls.root, 'Title', '<>', [], _utest=True) @classmethod def tearDownClass(cls): cls.dialog.cancel() + del cls.dialog cls.root.update_idletasks() cls.root.destroy() - del cls.dialog, cls.root + del cls.root - def test_cancel(self): + @mock.patch.object(config_key.GetKeysFrame, 'ok') + def test_cancel(self, mock_frame_ok): self.assertEqual(self.dialog.winfo_class(), 'Toplevel') self.dialog.button_cancel.invoke() with self.assertRaises(TclError): self.dialog.winfo_class() self.assertEqual(self.dialog.result, '') + mock_frame_ok.assert_not_called() + + +class OKWindowTest(unittest.TestCase): + "Simulate user clicking [OK] button." + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.dialog = config_key.GetKeysWindow( + cls.root, 'Title', '<>', [], _utest=True) + + @classmethod + def tearDownClass(cls): + cls.dialog.cancel() + del cls.dialog + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + @mock.patch.object(config_key.GetKeysFrame, 'ok') + def test_ok(self, mock_frame_ok): + self.assertEqual(self.dialog.winfo_class(), 'Toplevel') + self.dialog.button_ok.invoke() + with self.assertRaises(TclError): + self.dialog.winfo_class() + mock_frame_ok.assert_called() + + +class WindowResultTest(unittest.TestCase): + "Test window result get and set." + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.dialog = config_key.GetKeysWindow( + cls.root, 'Title', '<>', [], _utest=True) + + @classmethod + def tearDownClass(cls): + cls.dialog.cancel() + del cls.dialog + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + def test_result(self): + dialog = self.dialog + eq = self.assertEqual + + dialog.result = '' + eq(dialog.result, '') + eq(dialog.frame.result,'') + + dialog.result = 'bar' + eq(dialog.result,'bar') + eq(dialog.frame.result,'bar') + + dialog.frame.result = 'foo' + eq(dialog.result, 'foo') + eq(dialog.frame.result,'foo') class HelperTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index 3005ce08c9bf43..e5d5b4013fca57 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -954,8 +954,8 @@ def test_set_keys_type(self): def test_get_new_keys(self): eq = self.assertEqual d = self.page - orig_getkeysdialog = configdialog.GetKeysDialog - gkd = configdialog.GetKeysDialog = Func(return_self=True) + orig_getkeysdialog = configdialog.GetKeysWindow + gkd = configdialog.GetKeysWindow = Func(return_self=True) gnkn = d.get_new_keys_name = Func() d.button_new_keys.state(('!disabled',)) @@ -997,7 +997,7 @@ def test_get_new_keys(self): eq(d.keybinding.get(), '') del d.get_new_keys_name - configdialog.GetKeysDialog = orig_getkeysdialog + configdialog.GetKeysWindow = orig_getkeysdialog def test_get_new_keys_name(self): orig_sectionname = configdialog.SectionName diff --git a/Lib/idlelib/idle_test/test_text.py b/Lib/idlelib/idle_test/test_text.py index 0f31179e04b28f..43a9ba02c3d3c9 100644 --- a/Lib/idlelib/idle_test/test_text.py +++ b/Lib/idlelib/idle_test/test_text.py @@ -6,7 +6,7 @@ from test.support import requires from _tkinter import TclError -class TextTest(object): +class TextTest: "Define items common to both sets of tests." hw = 'hello\nworld' # Several tests insert this after initialization. diff --git a/Lib/idlelib/idle_test/test_zzdummy.py b/Lib/idlelib/idle_test/test_zzdummy.py index 1013cdc3c46f4f..209d8564da0664 100644 --- a/Lib/idlelib/idle_test/test_zzdummy.py +++ b/Lib/idlelib/idle_test/test_zzdummy.py @@ -19,7 +19,7 @@ } code_sample = """\ -class C1(): +class C1: # Class comment. def __init__(self, a, b): self.a = a diff --git a/Lib/idlelib/macosx.py b/Lib/idlelib/macosx.py index 53848fb079eab1..89b645702d334d 100644 --- a/Lib/idlelib/macosx.py +++ b/Lib/idlelib/macosx.py @@ -14,12 +14,25 @@ _tk_type = None def _init_tk_type(): - """ - Initializes OS X Tk variant values for - isAquaTk(), isCarbonTk(), isCocoaTk(), and isXQuartz(). + """ Initialize _tk_type for isXyzTk functions. + + This function is only called once, when _tk_type is still None. """ global _tk_type if platform == 'darwin': + + # When running IDLE, GUI is present, test/* may not be. + # When running tests, test/* is present, GUI may not be. + # If not, guess most common. Does not matter for testing. + from idlelib.__init__ import testing + if testing: + from test.support import requires, ResourceDenied + try: + requires('gui') + except ResourceDenied: + _tk_type = "cocoa" + return + root = tkinter.Tk() ws = root.tk.call('tk', 'windowingsystem') if 'x11' in ws: @@ -33,6 +46,7 @@ def _init_tk_type(): root.destroy() else: _tk_type = "other" + return def isAquaTk(): """ diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index 9ceae8a0e04b5a..bbdbceebe759d9 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -184,6 +184,10 @@ class EntryPoint(DeprecatedTuple): following the attr, and following any extras. """ + name: str + value: str + group: str + dist: Optional['Distribution'] = None def __init__(self, name, value, group): @@ -543,7 +547,7 @@ def locate_file(self, path): """ @classmethod - def from_name(cls, name): + def from_name(cls, name: str): """Return the Distribution for the given package name. :param name: The name of the distribution package to search for. @@ -551,13 +555,13 @@ def from_name(cls, name): package, if found. :raises PackageNotFoundError: When the named package's distribution metadata cannot be found. + :raises ValueError: When an invalid value is supplied for name. """ - for resolver in cls._discover_resolvers(): - dists = resolver(DistributionFinder.Context(name=name)) - dist = next(iter(dists), None) - if dist is not None: - return dist - else: + if not name: + raise ValueError("A distribution name is required.") + try: + return next(cls.discover(name=name)) + except StopIteration: raise PackageNotFoundError(name) @classmethod @@ -945,13 +949,26 @@ def _normalized_name(self): normalized name from the file system path. """ stem = os.path.basename(str(self._path)) - return self._name_from_stem(stem) or super()._normalized_name + return ( + pass_none(Prepared.normalize)(self._name_from_stem(stem)) + or super()._normalized_name + ) - def _name_from_stem(self, stem): - name, ext = os.path.splitext(stem) + @staticmethod + def _name_from_stem(stem): + """ + >>> PathDistribution._name_from_stem('foo-3.0.egg-info') + 'foo' + >>> PathDistribution._name_from_stem('CherryPy-3.0.dist-info') + 'CherryPy' + >>> PathDistribution._name_from_stem('face.egg-info') + 'face' + >>> PathDistribution._name_from_stem('foo.bar') + """ + filename, ext = os.path.splitext(stem) if ext not in ('.dist-info', '.egg-info'): return - name, sep, rest = stem.partition('-') + name, sep, rest = filename.partition('-') return name @@ -991,6 +1008,15 @@ def version(distribution_name): return distribution(distribution_name).version +_unique = functools.partial( + unique_everseen, + key=operator.attrgetter('_normalized_name'), +) +""" +Wrapper for ``distributions`` to return unique distributions by name. +""" + + def entry_points(**params) -> Union[EntryPoints, SelectableGroups]: """Return EntryPoint objects for all installed packages. @@ -1008,10 +1034,8 @@ def entry_points(**params) -> Union[EntryPoints, SelectableGroups]: :return: EntryPoints or SelectableGroups for all installed packages. """ - norm_name = operator.attrgetter('_normalized_name') - unique = functools.partial(unique_everseen, key=norm_name) eps = itertools.chain.from_iterable( - dist.entry_points for dist in unique(distributions()) + dist.entry_points for dist in _unique(distributions()) ) return SelectableGroups.load(eps).select(**params) diff --git a/Lib/inspect.py b/Lib/inspect.py index cbc0632484b832..d1a9daf289131a 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1448,7 +1448,10 @@ def getargvalues(frame): def formatannotation(annotation, base_module=None): if getattr(annotation, '__module__', None) == 'typing': - return repr(annotation).replace('typing.', '') + def repl(match): + text = match.group() + return text.removeprefix('typing.') + return re.sub(r'[\w\.]+', repl, repr(annotation)) if isinstance(annotation, types.GenericAlias): return str(annotation) if isinstance(annotation, type): @@ -2454,7 +2457,10 @@ def _signature_from_callable(obj, *, # Was this function wrapped by a decorator? if follow_wrapper_chains: - obj = unwrap(obj, stop=(lambda f: hasattr(f, "__signature__"))) + # Unwrap until we find an explicit signature or a MethodType (which will be + # handled explicitly below). + obj = unwrap(obj, stop=(lambda f: hasattr(f, "__signature__") + or isinstance(f, types.MethodType))) if isinstance(obj, types.MethodType): # If the unwrapped object is a *method*, we might want to # skip its first parameter (self). diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 3f15601e700d68..1cb71d8032e173 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1077,15 +1077,16 @@ def is_link_local(self): @property def is_private(self): - """Test if this address is allocated for private networks. + """Test if this network belongs to a private range. Returns: - A boolean, True if the address is reserved per + A boolean, True if the network is reserved per iana-ipv4-special-registry or iana-ipv6-special-registry. """ - return (self.network_address.is_private and - self.broadcast_address.is_private) + return any(self.network_address in priv_network and + self.broadcast_address in priv_network + for priv_network in self._constants._private_networks) @property def is_global(self): @@ -1122,6 +1123,15 @@ def is_loopback(self): return (self.network_address.is_loopback and self.broadcast_address.is_loopback) + +class _BaseConstants: + + _private_networks = [] + + +_BaseNetwork._constants = _BaseConstants + + class _BaseV4: """Base IPv4 object. @@ -1561,6 +1571,7 @@ class _IPv4Constants: IPv4Address._constants = _IPv4Constants +IPv4Network._constants = _IPv4Constants class _BaseV6: @@ -2285,3 +2296,4 @@ class _IPv6Constants: IPv6Address._constants = _IPv6Constants +IPv6Network._constants = _IPv6Constants diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index e4c21daaf3e47f..ed2c74771ea87d 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -1,4 +1,4 @@ -r"""JSON (JavaScript Object Notation) is a subset of +r"""JSON (JavaScript Object Notation) is a subset of JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data interchange format. diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index d7d824454e1ba3..c5d9ae2d0d5d04 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -252,7 +252,7 @@ def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): class JSONDecoder(object): - """Simple JSON decoder + """Simple JSON decoder Performs the following translations in decoding by default: diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 864f46d9dbb726..45f547741885a8 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -72,7 +72,7 @@ def replace(match): c_encode_basestring_ascii or py_encode_basestring_ascii) class JSONEncoder(object): - """Extensible JSON encoder for Python data structures. + """Extensible JSON encoder for Python data structures. Supports the following objects and types by default: diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 458c5fb7d0dcca..bcee2bab0b81aa 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -496,7 +496,7 @@ def __init__(self, *args, **kwargs): def usesTime(self): fmt = self._fmt - return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_format) >= 0 + return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_search) >= 0 def validate(self): pattern = Template.pattern diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index c6853e0513e080..f5a9760fd496b6 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -891,6 +891,13 @@ def _connect_unixsocket(self, address): raise def createSocket(self): + """ + Try to create a socket and, if it's not a datagram socket, connect it + to the other end. This method is called during handler initialization, + but it's not regarded as an error if the other end isn't listening yet + --- the method will be called again when emitting an event, + if there is no socket at that point. + """ address = self.address socktype = self.socktype @@ -898,7 +905,7 @@ def createSocket(self): self.unixsocket = True # Syslog server may be unavailable during handler initialisation. # C's openlog() function also ignores connection errors. - # Moreover, we ignore these errors while logging, so it not worse + # Moreover, we ignore these errors while logging, so it's not worse # to ignore it also here. try: self._connect_unixsocket(address) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index cc42dbdda05b91..ea369507297f86 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -161,10 +161,10 @@ def unregister(self, name, rtype): def _send(self, cmd, name, rtype): self.ensure_running() msg = '{0}:{1}:{2}\n'.format(cmd, name, rtype).encode('ascii') - if len(name) > 512: + if len(msg) > 512: # posix guarantees that writes to a pipe of less than PIPE_BUF # bytes are atomic, and that PIPE_BUF >= 512 - raise ValueError('name too long') + raise ValueError('msg too long') nbytes = os.write(self._fd, msg) assert nbytes == len(msg), "nbytes {0:n} but len(msg) {1:n}".format( nbytes, len(msg)) diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 881f2001dd5980..9a1e5aa17b87a2 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -173,7 +173,10 @@ def __init__(self, name=None, create=False, size=0): ) finally: _winapi.CloseHandle(h_map) - size = _winapi.VirtualQuerySize(p_buf) + try: + size = _winapi.VirtualQuerySize(p_buf) + finally: + _winapi.UnmapViewOfFile(p_buf) self._mmap = mmap.mmap(-1, size, tagname=name) self._size = size diff --git a/Lib/os.py b/Lib/os.py index 648188e0f13490..fd1e774fdcbcfa 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -288,7 +288,8 @@ def walk(top, topdown=True, onerror=None, followlinks=False): dirpath, dirnames, filenames dirpath is a string, the path to the directory. dirnames is a list of - the names of the subdirectories in dirpath (excluding '.' and '..'). + the names of the subdirectories in dirpath (including symlinks to directories, + and excluding '.' and '..'). filenames is a list of the names of the non-directory files in dirpath. Note that the names in the lists are just names, with no path components. To get a full path (which begins with top) to a file or directory in diff --git a/Lib/pdb.py b/Lib/pdb.py index fe8ddd1abb1085..411ce53115e7dd 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1332,6 +1332,12 @@ def do_list(self, arg): if last is None: last = first + 10 filename = self.curframe.f_code.co_filename + # gh-93696: stdlib frozen modules provide a useful __file__ + # this workaround can be removed with the closure of gh-89815 + if filename.startswith(">> try:\n' ' ... raise ExceptionGroup("eg",\n' @@ -2579,21 +2581,32 @@ ' | ValueError: 1\n' ' +------------------------------------\n' '\n' - ' Any remaining exceptions that were not handled by any ' - ':keyword:`!except*`\n' - ' clause are re-raised at the end, combined into an exception ' - 'group along with\n' - ' all exceptions that were raised from within ' - ':keyword:`!except*` clauses.\n' + 'Any remaining exceptions that were not handled by any "except*" ' + 'clause\n' + 'are re-raised at the end, combined into an exception group along ' + 'with\n' + 'all exceptions that were raised from within "except*" clauses.\n' + '\n' + 'If the raised exception is not an exception group and its type ' + 'matches\n' + 'one of the "except*" clauses, it is caught and wrapped by an ' + 'exception\n' + 'group with an empty message string.\n' + '\n' + ' >>> try:\n' + ' ... raise BlockingIOError\n' + ' ... except* BlockingIOError as e:\n' + ' ... print(repr(e))\n' + ' ...\n' + " ExceptionGroup('', (BlockingIOError()))\n" '\n' - ' An :keyword:`!except*` clause must have a matching type,\n' - ' and this type cannot be a subclass of ' - ':exc:`BaseExceptionGroup`.\n' - ' It is not possible to mix :keyword:`except` and ' - ':keyword:`!except*`\n' - ' in the same :keyword:`try`.\n' - ' :keyword:`break`, :keyword:`continue` and :keyword:`return`\n' - ' cannot appear in an :keyword:`!except*` clause.\n' + 'An "except*" clause must have a matching type, and this type ' + 'cannot be\n' + 'a subclass of "BaseExceptionGroup". It is not possible to mix ' + '"except"\n' + 'and "except*" in the same "try". "break", "continue" and ' + '"return"\n' + 'cannot appear in an "except*" clause.\n' '\n' '\n' '"else" clause\n' @@ -7308,7 +7321,7 @@ 'the clauses had been separated out into individual import ' 'statements.\n' '\n' - 'The details of the first step, finding and loading modules are\n' + 'The details of the first step, finding and loading modules, are\n' 'described in greater detail in the section on the import system, ' 'which\n' 'also describes the various types of packages and modules that can ' @@ -11096,8 +11109,9 @@ 'y)" is\n' 'typically invalid without special support in "MyClass". To ' 'be able to\n' - 'use that kind of patterns, the class needs to define a\n' - '*__match_args__* attribute.\n' + 'use that kind of pattern, the class needs to define a ' + '*__match_args__*\n' + 'attribute.\n' '\n' 'object.__match_args__\n' '\n' @@ -11302,6 +11316,10 @@ '*start* and\n' ' *end* are interpreted as in slice notation.\n' '\n' + ' If *sub* is empty, returns the number of empty strings ' + 'between\n' + ' characters which is the length of the string plus one.\n' + '\n' "str.encode(encoding='utf-8', errors='strict')\n" '\n' ' Return an encoded version of the string as a bytes ' @@ -11808,7 +11826,7 @@ 'followed by\n' ' the string itself.\n' '\n' - 'str.rsplit(sep=None, maxsplit=- 1)\n' + 'str.rsplit(sep=None, maxsplit=-1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -11849,7 +11867,7 @@ " >>> 'Monty Python'.removesuffix(' Python')\n" " 'Monty'\n" '\n' - 'str.split(sep=None, maxsplit=- 1)\n' + 'str.split(sep=None, maxsplit=-1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -12650,9 +12668,10 @@ 'the type matches some of the exceptions in the group. This means ' 'that\n' 'multiple "except*" clauses can execute, each handling part of the\n' - 'exception group. Each clause executes once and handles an exception\n' - 'group of all matching exceptions. Each exception in the group is\n' - 'handled by at most one "except*" clause, the first that matches it.\n' + 'exception group. Each clause executes at most once and handles an\n' + 'exception group of all matching exceptions. Each exception in the\n' + 'group is handled by at most one "except*" clause, the first that\n' + 'matches it.\n' '\n' ' >>> try:\n' ' ... raise ExceptionGroup("eg",\n' @@ -12673,20 +12692,31 @@ ' | ValueError: 1\n' ' +------------------------------------\n' '\n' - ' Any remaining exceptions that were not handled by any ' - ':keyword:`!except*`\n' - ' clause are re-raised at the end, combined into an exception group ' - 'along with\n' - ' all exceptions that were raised from within :keyword:`!except*` ' - 'clauses.\n' + 'Any remaining exceptions that were not handled by any "except*" ' + 'clause\n' + 'are re-raised at the end, combined into an exception group along ' + 'with\n' + 'all exceptions that were raised from within "except*" clauses.\n' '\n' - ' An :keyword:`!except*` clause must have a matching type,\n' - ' and this type cannot be a subclass of :exc:`BaseExceptionGroup`.\n' - ' It is not possible to mix :keyword:`except` and ' - ':keyword:`!except*`\n' - ' in the same :keyword:`try`.\n' - ' :keyword:`break`, :keyword:`continue` and :keyword:`return`\n' - ' cannot appear in an :keyword:`!except*` clause.\n' + 'If the raised exception is not an exception group and its type ' + 'matches\n' + 'one of the "except*" clauses, it is caught and wrapped by an ' + 'exception\n' + 'group with an empty message string.\n' + '\n' + ' >>> try:\n' + ' ... raise BlockingIOError\n' + ' ... except* BlockingIOError as e:\n' + ' ... print(repr(e))\n' + ' ...\n' + " ExceptionGroup('', (BlockingIOError()))\n" + '\n' + 'An "except*" clause must have a matching type, and this type cannot ' + 'be\n' + 'a subclass of "BaseExceptionGroup". It is not possible to mix ' + '"except"\n' + 'and "except*" in the same "try". "break", "continue" and "return"\n' + 'cannot appear in an "except*" clause.\n' '\n' '\n' '"else" clause\n' @@ -13947,17 +13977,11 @@ 'dictionaries or\n' 'other mutable types (that are compared by value rather than ' 'by object\n' - 'identity) may not be used as keys. Numeric types used for ' - 'keys obey\n' - 'the normal rules for numeric comparison: if two numbers ' - 'compare equal\n' - '(such as "1" and "1.0") then they can be used ' - 'interchangeably to index\n' - 'the same dictionary entry. (Note however, that since ' - 'computers store\n' - 'floating-point numbers as approximations it is usually ' - 'unwise to use\n' - 'them as dictionary keys.)\n' + 'identity) may not be used as keys. Values that compare equal ' + '(such as\n' + '"1", "1.0", and "True") can be used interchangeably to index ' + 'the same\n' + 'dictionary entry.\n' '\n' 'class dict(**kwargs)\n' 'class dict(mapping, **kwargs)\n' diff --git a/Lib/random.py b/Lib/random.py index 1f3530e880fce2..f94616e048c554 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -282,10 +282,10 @@ def randbytes(self, n): ## -------------------- integer methods ------------------- def randrange(self, start, stop=None, step=_ONE): - """Choose a random item from range(start, stop[, step]). + """Choose a random item from range(stop) or range(start, stop[, step]). - This fixes the problem with randint() which includes the - endpoint; in Python this is usually not what you want. + Roughly equivalent to ``choice(range(start, stop, step))`` but + supports arbitrarily large ranges and is optimized for common cases. """ diff --git a/Lib/shutil.py b/Lib/shutil.py index 6093e11374294a..bfed796d51b1d0 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -489,12 +489,13 @@ def _copytree(entries, src, dst, symlinks, ignore, copy_function, # otherwise let the copy occur. copy2 will raise an error if srcentry.is_dir(): copytree(srcobj, dstname, symlinks, ignore, - copy_function, dirs_exist_ok=dirs_exist_ok) + copy_function, ignore_dangling_symlinks, + dirs_exist_ok) else: copy_function(srcobj, dstname) elif srcentry.is_dir(): copytree(srcobj, dstname, symlinks, ignore, copy_function, - dirs_exist_ok=dirs_exist_ok) + ignore_dangling_symlinks, dirs_exist_ok) else: # Will raise a SpecialFileError for unsupported file types copy_function(srcobj, dstname) diff --git a/Lib/sndhdr.py b/Lib/sndhdr.py index 98a783448239a8..45def9ad16d32c 100644 --- a/Lib/sndhdr.py +++ b/Lib/sndhdr.py @@ -77,6 +77,7 @@ def whathdr(filename): tests = [] def test_aifc(h, f): + """AIFC and AIFF files""" with warnings.catch_warnings(): warnings.simplefilter('ignore', category=DeprecationWarning) import aifc @@ -100,6 +101,7 @@ def test_aifc(h, f): def test_au(h, f): + """AU and SND files""" if h.startswith(b'.snd'): func = get_long_be elif h[:4] in (b'\0ds.', b'dns.'): @@ -133,6 +135,7 @@ def test_au(h, f): def test_hcom(h, f): + """HCOM file""" if h[65:69] != b'FSSD' or h[128:132] != b'HCOM': return None divisor = get_long_be(h[144:148]) @@ -146,6 +149,7 @@ def test_hcom(h, f): def test_voc(h, f): + """VOC file""" if not h.startswith(b'Creative Voice File\032'): return None sbseek = get_short_le(h[20:22]) @@ -160,6 +164,7 @@ def test_voc(h, f): def test_wav(h, f): + """WAV file""" import wave # 'RIFF' 'WAVE' 'fmt ' if not h.startswith(b'RIFF') or h[8:12] != b'WAVE' or h[12:16] != b'fmt ': @@ -176,6 +181,7 @@ def test_wav(h, f): def test_8svx(h, f): + """8SVX file""" if not h.startswith(b'FORM') or h[8:12] != b'8SVX': return None # Should decode it to get #channels -- assume always 1 @@ -185,6 +191,7 @@ def test_8svx(h, f): def test_sndt(h, f): + """SNDT file""" if h.startswith(b'SOUND'): nsamples = get_long_le(h[8:12]) rate = get_short_le(h[20:22]) @@ -194,6 +201,7 @@ def test_sndt(h, f): def test_sndr(h, f): + """SNDR file""" if h.startswith(b'\0\0'): rate = get_short_le(h[2:4]) if 4000 <= rate <= 25000: diff --git a/Lib/socket.py b/Lib/socket.py index a19652247a5293..0717c696b12e03 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -254,17 +254,18 @@ def __repr__(self): self.type, self.proto) if not closed: + # getsockname and getpeername may not be available on WASI. try: laddr = self.getsockname() if laddr: s += ", laddr=%s" % str(laddr) - except error: + except (error, AttributeError): pass try: raddr = self.getpeername() if raddr: s += ", raddr=%s" % str(raddr) - except error: + except (error, AttributeError): pass s += '>' return s diff --git a/Lib/statistics.py b/Lib/statistics.py index 9598ab6d090d0d..3b3b43babb42e2 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -1382,3 +1382,9 @@ def __hash__(self): def __repr__(self): return f'{type(self).__name__}(mu={self._mu!r}, sigma={self._sigma!r})' + + def __getstate__(self): + return self._mu, self._sigma + + def __setstate__(self, state): + self._mu, self._sigma = state diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 7ae8df154b481f..9cadd1bf8e622c 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -456,7 +456,8 @@ def check_output(*popenargs, timeout=None, **kwargs): if 'input' in kwargs and kwargs['input'] is None: # Explicitly passing input=None was previously equivalent to passing an # empty string. That is maintained here for backwards compatibility. - if kwargs.get('universal_newlines') or kwargs.get('text'): + if kwargs.get('universal_newlines') or kwargs.get('text') or kwargs.get('encoding') \ + or kwargs.get('errors'): empty = '' else: empty = b'' @@ -508,7 +509,8 @@ def run(*popenargs, The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes - will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. + will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them, + or pass capture_output=True to capture both. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py index 7973f26f98b8b2..a47f5a96b89722 100755 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -23,8 +23,6 @@ import os import sys import tokenize -if not hasattr(tokenize, 'NL'): - raise ValueError("tokenize.NL doesn't exist -- tokenize module too old") __all__ = ["check", "NannyNag", "process_tokens"] diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 169c88d63f781b..87f44d99f59da5 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2339,6 +2339,8 @@ def next(self): # Advance the file pointer. if self.offset != self.fileobj.tell(): + if self.offset == 0: + return None self.fileobj.seek(self.offset - 1) if not self.fileobj.read(1): raise ReadError("unexpected end of data") diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 599c3f2abf33c7..b50a1543205cec 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5438,6 +5438,14 @@ def test_resource_tracker_reused(self): self.assertTrue(is_resource_tracker_reused) + def test_too_long_name_resource(self): + # gh-96819: Resource names that will make the length of a write to a pipe + # greater than PIPE_BUF are not allowed + rtype = "shared_memory" + too_long_name_resource = "a" * (512 - len(rtype)) + with self.assertRaises(ValueError): + resource_tracker.register(too_long_name_resource, rtype) + class TestSimpleQueue(unittest.TestCase): @@ -6036,5 +6044,5 @@ def test_semlock_subclass(self): class SemLock(_multiprocessing.SemLock): pass name = f'test_semlock_subclass-{os.getpid()}' - s = SemLock(1, 0, 10, name, 0) + s = SemLock(1, 0, 10, name, False) _multiprocessing.sem_unlink(name) diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index 00333cc9036a3c..e7f8a945c1a9af 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -419,6 +419,38 @@ def hook(event, args): sys._getframe() +def test_syslog(): + import syslog + + def hook(event, args): + if event.startswith("syslog."): + print(event, *args) + + sys.addaudithook(hook) + syslog.openlog('python') + syslog.syslog('test') + syslog.setlogmask(syslog.LOG_DEBUG) + syslog.closelog() + # implicit open + syslog.syslog('test2') + # open with default ident + syslog.openlog(logoption=syslog.LOG_NDELAY, facility=syslog.LOG_LOCAL0) + sys.argv = None + syslog.openlog() + syslog.closelog() + + +def test_not_in_gc(): + import gc + + hook = lambda *a: None + sys.addaudithook(hook) + + for o in gc.get_objects(): + if isinstance(o, list): + assert hook not in o + + if __name__ == "__main__": from test.support import suppress_msvcrt_asserts diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 9016cffba95ca4..0228d6be03264c 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -1803,12 +1803,12 @@ static PyObject * test_Py_UNICODE_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - const Py_UNICODE *a; - const Py_UNICODE *b; - const Py_UNICODE *c; - const Py_UNICODE *d; + const Py_UNICODE *a = NULL; + const Py_UNICODE *b = NULL; + const Py_UNICODE *c = NULL; + const Py_UNICODE *d = NULL; Py_ssize_t d_length; - const Py_UNICODE *e; + const Py_UNICODE *e = NULL; Py_ssize_t e_length; if (!_PyArg_ParseStack(args, nargs, "O&O&O&u#Z#:test_Py_UNICODE_converter", @@ -1839,7 +1839,7 @@ test_Py_UNICODE_converter_impl(PyObject *module, const Py_UNICODE *a, const Py_UNICODE *b, const Py_UNICODE *c, const Py_UNICODE *d, Py_ssize_t d_length, const Py_UNICODE *e, Py_ssize_t e_length) -/*[clinic end generated code: output=45e92604de227552 input=064a3b68ad7f04b0]*/ +/*[clinic end generated code: output=9d41b3a38a0f6f2f input=064a3b68ad7f04b0]*/ /*[clinic input] diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 21419e11c87497..499f80a15f3422 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -2776,6 +2776,15 @@ def pie(self): unpickled = self.loads(self.dumps(method, proto)) self.assertEqual(method(obj), unpickled(obj)) + descriptors = ( + PyMethodsTest.__dict__['cheese'], # static method descriptor + PyMethodsTest.__dict__['wine'], # class method descriptor + ) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + for descr in descriptors: + with self.subTest(proto=proto, descr=descr): + self.assertRaises(TypeError, self.dumps, descr, proto) + def test_c_methods(self): global Subclass class Subclass(tuple): @@ -2811,6 +2820,15 @@ class Nested(str): unpickled = self.loads(self.dumps(method, proto)) self.assertEqual(method(*args), unpickled(*args)) + descriptors = ( + bytearray.__dict__['maketrans'], # built-in static method descriptor + dict.__dict__['fromkeys'], # built-in class method descriptor + ) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + for descr in descriptors: + with self.subTest(proto=proto, descr=descr): + self.assertRaises(TypeError, self.dumps, descr, proto) + def test_compat_pickle(self): tests = [ (range(1, 7), '__builtin__', 'xrange'), diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index 0d4c7ecf4a04f2..d69edd7bf458da 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -469,6 +469,11 @@ def test_split(self): self.checkraises(ValueError, 'hello', 'split', '', 0) def test_rsplit(self): + # without arg + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit') + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit') + self.checkequal([], '', 'rsplit') + # by a char self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|') self.checkequal(['a|b|c', 'd'], 'a|b|c|d', 'rsplit', '|', 1) @@ -522,6 +527,9 @@ def test_rsplit(self): # with keyword args self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', sep='|') + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit', sep=None) + self.checkequal(['a b c', 'd'], + 'a b c d', 'rsplit', sep=None, maxsplit=1) self.checkequal(['a|b|c', 'd'], 'a|b|c|d', 'rsplit', '|', maxsplit=1) self.checkequal(['a|b|c', 'd'], diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 46087a98a2e1f1..c33f90d8071df9 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1498,7 +1498,7 @@ def _platform_specific(self): self._env = {k.upper(): os.getenv(k) for k in os.environ} self._env["PYTHONHOME"] = os.path.dirname(self.real) - if sysconfig.is_python_build(True): + if sysconfig.is_python_build(): self._env["PYTHONPATH"] = STDLIB_DIR else: def _platform_specific(self): @@ -2078,7 +2078,7 @@ def wait_process(pid, *, exitcode, timeout=None): Raise an AssertionError if the process exit code is not equal to exitcode. - If the process runs longer than timeout seconds (SHORT_TIMEOUT by default), + If the process runs longer than timeout seconds (LONG_TIMEOUT by default), kill the process (if signal.SIGKILL is available) and raise an AssertionError. The timeout feature is not available on Windows. """ @@ -2086,7 +2086,7 @@ def wait_process(pid, *, exitcode, timeout=None): import signal if timeout is None: - timeout = SHORT_TIMEOUT + timeout = LONG_TIMEOUT t0 = time.monotonic() sleep = 0.001 max_sleep = 0.1 @@ -2097,7 +2097,7 @@ def wait_process(pid, *, exitcode, timeout=None): # process is still running dt = time.monotonic() - t0 - if dt > SHORT_TIMEOUT: + if dt > timeout: try: os.kill(pid, signal.SIGKILL) os.waitpid(pid, 0) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 25ac03cf14810f..1acecbb8abaab8 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -296,7 +296,7 @@ class TestOptionalsSingleDashCombined(ParserTestCase): Sig('-z'), ] failures = ['a', '--foo', '-xa', '-x --foo', '-x -z', '-z -x', - '-yx', '-yz a', '-yyyx', '-yyyza', '-xyza'] + '-yx', '-yz a', '-yyyx', '-yyyza', '-xyza', '-x='] successes = [ ('', NS(x=False, yyy=None, z=None)), ('-x', NS(x=True, yyy=None, z=None)), diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 90ad6af56fdb20..7581adc8fc272b 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1026,6 +1026,18 @@ def test_increment_lineno(self): self.assertEqual(ast.increment_lineno(src).lineno, 2) self.assertIsNone(ast.increment_lineno(src).end_lineno) + def test_increment_lineno_on_module(self): + src = ast.parse(dedent("""\ + a = 1 + b = 2 # type: ignore + c = 3 + d = 4 # type: ignore@tag + """), type_comments=True) + ast.increment_lineno(src, n=5) + self.assertEqual(src.type_ignores[0].lineno, 7) + self.assertEqual(src.type_ignores[1].lineno, 9) + self.assertEqual(src.type_ignores[1].tag, '@tag') + def test_iter_fields(self): node = ast.parse('foo()', mode='eval') d = dict(ast.iter_fields(node.body)) diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 063174adacfa37..6ba602dd619fba 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -746,7 +746,7 @@ async def coro(): def test_env_var_debug(self): code = '\n'.join(( 'import asyncio', - 'loop = asyncio.get_event_loop()', + 'loop = asyncio.new_event_loop()', 'print(loop.get_debug())')) # Test with -E to not fail if the unit test was run with diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 05d9107b28e2ae..c431fea401633e 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2547,8 +2547,9 @@ def test_event_loop_policy(self): def test_get_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() self.assertIsNone(policy._local._loop) - - loop = policy.get_event_loop() + with self.assertWarns(DeprecationWarning) as cm: + loop = policy.get_event_loop() + self.assertEqual(cm.filename, __file__) self.assertIsInstance(loop, asyncio.AbstractEventLoop) self.assertIs(policy._local._loop, loop) @@ -2562,7 +2563,10 @@ def test_get_event_loop_calls_set_event_loop(self): policy, "set_event_loop", wraps=policy.set_event_loop) as m_set_event_loop: - loop = policy.get_event_loop() + with self.assertWarns(DeprecationWarning) as cm: + loop = policy.get_event_loop() + self.addCleanup(loop.close) + self.assertEqual(cm.filename, __file__) # policy._local._loop must be set through .set_event_loop() # (the unix DefaultEventLoopPolicy needs this call to attach @@ -2596,7 +2600,8 @@ def test_new_event_loop(self): def test_set_event_loop(self): policy = asyncio.DefaultEventLoopPolicy() - old_loop = policy.get_event_loop() + old_loop = policy.new_event_loop() + policy.set_event_loop(old_loop) self.assertRaises(TypeError, policy.set_event_loop, object()) @@ -2709,15 +2714,11 @@ def get_event_loop(self): asyncio.set_event_loop_policy(Policy()) loop = asyncio.new_event_loop() - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() with self.assertRaisesRegex(RuntimeError, 'no running'): asyncio.get_running_loop() @@ -2731,16 +2732,11 @@ async def func(): loop.run_until_complete(func()) asyncio.set_event_loop(loop) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) - + with self.assertRaises(TestError): + asyncio.get_event_loop() asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(TestError): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaises(TestError): + asyncio.get_event_loop() finally: asyncio.set_event_loop_policy(old_policy) @@ -2764,10 +2760,8 @@ def test_get_event_loop_returns_running_loop2(self): self.addCleanup(loop2.close) self.assertEqual(cm.filename, __file__) asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'no current'): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() with self.assertRaisesRegex(RuntimeError, 'no running'): asyncio.get_running_loop() @@ -2781,15 +2775,11 @@ async def func(): loop.run_until_complete(func()) asyncio.set_event_loop(loop) - with self.assertWarns(DeprecationWarning) as cm: - self.assertIs(asyncio.get_event_loop(), loop) - self.assertEqual(cm.filename, __file__) + self.assertIs(asyncio.get_event_loop(), loop) asyncio.set_event_loop(None) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'no current'): - asyncio.get_event_loop() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() finally: asyncio.set_event_loop_policy(old_policy) diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index d71af8c13b14a1..987772e0b1964b 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -145,10 +145,8 @@ def test_initial_state(self): self.assertTrue(f.cancelled()) def test_constructor_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - self._new_future() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + self._new_future() def test_constructor_use_running_loop(self): async def test(): @@ -158,12 +156,10 @@ async def test(): self.assertIs(f.get_loop(), self.loop) def test_constructor_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - f = self._new_future() - self.assertEqual(cm.filename, __file__) + f = self._new_future() self.assertIs(f._loop, self.loop) self.assertIs(f.get_loop(), self.loop) @@ -499,10 +495,8 @@ def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) f1 = ex.submit(run, 'oi') - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.wrap_future(f1) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.wrap_future(f1) ex.shutdown(wait=True) def test_wrap_future_use_running_loop(self): @@ -517,16 +511,14 @@ async def test(): ex.shutdown(wait=True) def test_wrap_future_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) f1 = ex.submit(run, 'oi') - with self.assertWarns(DeprecationWarning) as cm: - f2 = asyncio.wrap_future(f1) - self.assertEqual(cm.filename, __file__) + f2 = asyncio.wrap_future(f1) self.assertIs(self.loop, f2._loop) ex.shutdown(wait=True) @@ -824,6 +816,21 @@ def __eq__(self, other): fut.remove_done_callback(evil()) + def test_remove_done_callbacks_list_clear(self): + # see https://github.com/python/cpython/issues/97592 for details + + fut = self._new_future() + fut.add_done_callback(str) + + for _ in range(63): + fut.add_done_callback(id) + + class evil: + def __eq__(self, other): + fut.remove_done_callback(other) + + fut.remove_done_callback(evil()) + def test_schedule_callbacks_list_mutation_1(self): # see http://bugs.python.org/issue28963 for details diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index 7fca0541ee75a2..ae30185cef776a 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -290,7 +290,33 @@ def test_force_close_idempotent(self): tr._closing = True tr._force_close(None) test_utils.run_briefly(self.loop) + # See https://github.com/python/cpython/issues/89237 + # `protocol.connection_lost` should be called even if + # the transport was closed forcefully otherwise + # the resources held by protocol will never be freed + # and waiters will never be notified leading to hang. + self.assertTrue(self.protocol.connection_lost.called) + + def test_force_close_protocol_connection_lost_once(self): + tr = self.socket_transport() self.assertFalse(self.protocol.connection_lost.called) + tr._closing = True + # Calling _force_close twice should not call + # protocol.connection_lost twice + tr._force_close(None) + tr._force_close(None) + test_utils.run_briefly(self.loop) + self.assertEqual(1, self.protocol.connection_lost.call_count) + + def test_close_protocol_connection_lost_once(self): + tr = self.socket_transport() + self.assertFalse(self.protocol.connection_lost.called) + # Calling close twice should not call + # protocol.connection_lost twice + tr.close() + tr.close() + test_utils.run_briefly(self.loop) + self.assertEqual(1, self.protocol.connection_lost.call_count) def test_fatal_error_2(self): tr = self.socket_transport() diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index a10504b1c4130e..0198da21d77028 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -1,6 +1,7 @@ """Tests for sendfile functionality.""" import asyncio +import errno import os import socket import sys @@ -484,8 +485,17 @@ def sendfile_native(transp, file, offset, count): srv_proto, cli_proto = self.prepare_sendfile(close_after=1024) with self.assertRaises(ConnectionError): - self.run_loop( - self.loop.sendfile(cli_proto.transport, self.file)) + try: + self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + except OSError as e: + # macOS may raise OSError of EPROTOTYPE when writing to a + # socket that is in the process of closing down. + if e.errno == errno.EPROTOTYPE and sys.platform == "darwin": + raise ConnectionError + else: + raise + self.run_loop(srv_proto.done) self.assertTrue(1024 <= srv_proto.nbytes < len(self.DATA), diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py index 5de9b7a14e87da..aaf3c37101f52a 100644 --- a/Lib/test/test_asyncio/test_ssl.py +++ b/Lib/test/test_asyncio/test_ssl.py @@ -1689,7 +1689,7 @@ def stop(self): def run(self): try: with self._sock: - self._sock.setblocking(0) + self._sock.setblocking(False) self._run() finally: self._s1.close() diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 0c49099bc499a5..95fc7a159baee8 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -810,10 +810,8 @@ def test_read_all_from_pipe_reader(self): self.assertEqual(data, b'data') def test_streamreader_constructor_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.StreamReader() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReader() def test_streamreader_constructor_use_running_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor @@ -827,21 +825,17 @@ async def test(): def test_streamreader_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 self.addCleanup(asyncio.set_event_loop, None) asyncio.set_event_loop(self.loop) - with self.assertWarns(DeprecationWarning) as cm: - reader = asyncio.StreamReader() - self.assertEqual(cm.filename, __file__) + reader = asyncio.StreamReader() self.assertIs(reader._loop, self.loop) def test_streamreaderprotocol_constructor_without_loop(self): reader = mock.Mock() - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.StreamReaderProtocol(reader) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReaderProtocol(reader) def test_streamreaderprotocol_constructor_use_running_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor @@ -855,13 +849,11 @@ async def test(): def test_streamreaderprotocol_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 self.addCleanup(asyncio.set_event_loop, None) asyncio.set_event_loop(self.loop) reader = mock.Mock() - with self.assertWarns(DeprecationWarning) as cm: - protocol = asyncio.StreamReaderProtocol(reader) - self.assertEqual(cm.filename, __file__) + protocol = asyncio.StreamReaderProtocol(reader) self.assertIs(protocol._loop, self.loop) def test_multiple_drain(self): diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 961c463f8dc11b..f71ad72f999ee9 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -1,4 +1,5 @@ import os +import shutil import signal import sys import unittest @@ -182,6 +183,33 @@ def test_kill(self): else: self.assertEqual(-signal.SIGKILL, returncode) + def test_kill_issue43884(self): + if sys.platform == 'win32': + blocking_shell_command = f'{sys.executable} -c "import time; time.sleep(2)"' + else: + blocking_shell_command = 'sleep 1; sleep 1' + creationflags = 0 + if sys.platform == 'win32': + from subprocess import CREATE_NEW_PROCESS_GROUP + # On windows create a new process group so that killing process + # kills the process and all its children. + creationflags = CREATE_NEW_PROCESS_GROUP + proc = self.loop.run_until_complete( + asyncio.create_subprocess_shell(blocking_shell_command, stdout=asyncio.subprocess.PIPE, + creationflags=creationflags) + ) + self.loop.run_until_complete(asyncio.sleep(1)) + if sys.platform == 'win32': + proc.send_signal(signal.CTRL_BREAK_EVENT) + # On windows it is an alias of terminate which sets the return code + proc.kill() + returncode = self.loop.run_until_complete(proc.wait()) + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + # expect 1 but sometimes get 0 + else: + self.assertEqual(-signal.SIGKILL, returncode) + def test_terminate(self): args = PROGRAM_BLOCKED proc = self.loop.run_until_complete( @@ -402,6 +430,26 @@ async def empty_error(): self.assertEqual(output, None) self.assertEqual(exitcode, 0) + @unittest.skipIf(sys.platform != 'linux', "Don't have /dev/stdin") + def test_devstdin_input(self): + + async def devstdin_input(message): + code = 'file = open("/dev/stdin"); data = file.read(); print(len(data))' + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + close_fds=False, + ) + stdout, stderr = await proc.communicate(message) + exitcode = await proc.wait() + return (stdout, exitcode) + + output, exitcode = self.loop.run_until_complete(devstdin_input(b'abc')) + self.assertEqual(output.rstrip(), b'3') + self.assertEqual(exitcode, 0) + def test_cancel_process_wait(self): # Issue #23140: cancel Process.wait() diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 7585768073e6ab..9900e307a477bd 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -200,10 +200,8 @@ async def notmuch(): a = notmuch() self.addCleanup(a.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.ensure_future(a) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.ensure_future(a) async def test(): return asyncio.ensure_future(notmuch()) @@ -213,12 +211,10 @@ async def test(): self.assertTrue(t.done()) self.assertEqual(t.result(), 'ok') - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - t = asyncio.ensure_future(notmuch()) - self.assertEqual(cm.filename, __file__) + t = asyncio.ensure_future(notmuch()) self.assertIs(t._loop, self.loop) self.loop.run_until_complete(t) self.assertTrue(t.done()) @@ -639,27 +635,30 @@ def on_timeout(): await asyncio.sleep(0) return timed_out, structured_block_finished, outer_code_reached - # Test which timed out. - t1 = self.new_task(loop, make_request_with_timeout(sleep=10.0, timeout=0.1)) - timed_out, structured_block_finished, outer_code_reached = ( - loop.run_until_complete(t1) - ) - self.assertTrue(timed_out) - self.assertFalse(structured_block_finished) # it was cancelled - self.assertTrue(outer_code_reached) # task got uncancelled after leaving - # the structured block and continued until - # completion - self.assertEqual(t1.cancelling(), 0) # no pending cancellation of the outer task - - # Test which did not time out. - t2 = self.new_task(loop, make_request_with_timeout(sleep=0, timeout=10.0)) - timed_out, structured_block_finished, outer_code_reached = ( - loop.run_until_complete(t2) - ) - self.assertFalse(timed_out) - self.assertTrue(structured_block_finished) - self.assertTrue(outer_code_reached) - self.assertEqual(t2.cancelling(), 0) + try: + # Test which timed out. + t1 = self.new_task(loop, make_request_with_timeout(sleep=10.0, timeout=0.1)) + timed_out, structured_block_finished, outer_code_reached = ( + loop.run_until_complete(t1) + ) + self.assertTrue(timed_out) + self.assertFalse(structured_block_finished) # it was cancelled + self.assertTrue(outer_code_reached) # task got uncancelled after leaving + # the structured block and continued until + # completion + self.assertEqual(t1.cancelling(), 0) # no pending cancellation of the outer task + + # Test which did not time out. + t2 = self.new_task(loop, make_request_with_timeout(sleep=0, timeout=10.0)) + timed_out, structured_block_finished, outer_code_reached = ( + loop.run_until_complete(t2) + ) + self.assertFalse(timed_out) + self.assertTrue(structured_block_finished) + self.assertTrue(outer_code_reached) + self.assertEqual(t2.cancelling(), 0) + finally: + loop.close() def test_cancel(self): @@ -1533,10 +1532,8 @@ async def coro(): self.addCleanup(a.close) futs = asyncio.as_completed([a]) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - list(futs) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + list(futs) def test_as_completed_coroutine_use_running_loop(self): loop = self.new_test_loop() @@ -1966,10 +1963,8 @@ async def coro(): inner = coro() self.addCleanup(inner.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.shield(inner) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.shield(inner) def test_shield_coroutine_use_running_loop(self): async def coro(): @@ -1983,15 +1978,13 @@ async def test(): self.assertEqual(res, 42) def test_shield_coroutine_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 async def coro(): return 42 asyncio.set_event_loop(self.loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - outer = asyncio.shield(coro()) - self.assertEqual(cm.filename, __file__) + outer = asyncio.shield(coro()) self.assertEqual(outer._loop, self.loop) res = self.loop.run_until_complete(outer) self.assertEqual(res, 42) @@ -2817,7 +2810,7 @@ def test_current_task_no_running_loop(self): self.assertIsNone(asyncio.current_task(loop=self.loop)) def test_current_task_no_running_loop_implicit(self): - with self.assertRaises(RuntimeError): + with self.assertRaisesRegex(RuntimeError, 'no running event loop'): asyncio.current_task() def test_current_task_with_implicit_loop(self): @@ -2981,10 +2974,8 @@ def _gather(self, *args, **kwargs): return asyncio.gather(*args, **kwargs) def test_constructor_empty_sequence_without_loop(self): - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.gather() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather() def test_constructor_empty_sequence_use_running_loop(self): async def gather(): @@ -2997,12 +2988,10 @@ async def gather(): self.assertEqual(fut.result(), []) def test_constructor_empty_sequence_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 asyncio.set_event_loop(self.one_loop) self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - fut = asyncio.gather() - self.assertEqual(cm.filename, __file__) + fut = asyncio.gather() self.assertIsInstance(fut, asyncio.Future) self.assertIs(fut._loop, self.one_loop) self._run_loop(self.one_loop) @@ -3090,10 +3079,8 @@ async def coro(): self.addCleanup(gen1.close) gen2 = coro() self.addCleanup(gen2.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaises(RuntimeError): - asyncio.gather(gen1, gen2) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather(gen1, gen2) def test_constructor_use_running_loop(self): async def coro(): @@ -3107,16 +3094,14 @@ async def gather(): self.one_loop.run_until_complete(fut) def test_constructor_use_global_loop(self): - # Deprecated in 3.10 + # Deprecated in 3.10, undeprecated in 3.11.1 async def coro(): return 'abc' asyncio.set_event_loop(self.other_loop) self.addCleanup(asyncio.set_event_loop, None) gen1 = coro() gen2 = coro() - with self.assertWarns(DeprecationWarning) as cm: - fut = asyncio.gather(gen1, gen2) - self.assertEqual(cm.filename, __file__) + fut = asyncio.gather(gen1, gen2) self.assertIs(fut._loop, self.other_loop) self.other_loop.run_until_complete(fut) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 1d922783ce3b36..01c1214c7f7cc2 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1740,7 +1740,8 @@ def f(): def test_child_watcher_replace_mainloop_existing(self): policy = self.create_policy() - loop = policy.get_event_loop() + loop = policy.new_event_loop() + policy.set_event_loop(loop) # Explicitly setup SafeChildWatcher, # default ThreadedChildWatcher has no _loop property diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index 6b4f65c3376f5d..5033acc052486b 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -239,6 +239,17 @@ def test_read_self_pipe_restart(self): self.close_loop(self.loop) self.assertFalse(self.loop.call_exception_handler.called) + def test_address_argument_type_error(self): + # Regression test for https://github.com/python/cpython/issues/98793 + proactor = self.loop._proactor + sock = socket.socket(type=socket.SOCK_DGRAM) + bad_address = None + with self.assertRaises(TypeError): + proactor.connect(sock, bad_address) + with self.assertRaises(TypeError): + proactor.sendto(sock, b'abc', addr=bad_address) + sock.close() + class WinPolicyTests(test_utils.TestCase): diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 18426f27a2e32f..75e96f069d8c27 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -16,6 +16,7 @@ class AuditTest(unittest.TestCase): + maxDiff = None @support.requires_subprocess() def do_test(self, *args): @@ -185,5 +186,34 @@ def test_sys_getframe(self): self.assertEqual(actual, expected) + def test_syslog(self): + syslog = import_helper.import_module("syslog") + + returncode, events, stderr = self.run_python("test_syslog") + if returncode: + self.fail(stderr) + + if support.verbose: + print('Events:', *events, sep='\n ') + + self.assertSequenceEqual( + events, + [('syslog.openlog', ' ', f'python 0 {syslog.LOG_USER}'), + ('syslog.syslog', ' ', f'{syslog.LOG_INFO} test'), + ('syslog.setlogmask', ' ', f'{syslog.LOG_DEBUG}'), + ('syslog.closelog', '', ''), + ('syslog.syslog', ' ', f'{syslog.LOG_INFO} test2'), + ('syslog.openlog', ' ', f'audit-tests.py 0 {syslog.LOG_USER}'), + ('syslog.openlog', ' ', f'audit-tests.py {syslog.LOG_NDELAY} {syslog.LOG_LOCAL0}'), + ('syslog.openlog', ' ', f'None 0 {syslog.LOG_USER}'), + ('syslog.closelog', '', '')] + ) + + def test_not_in_gc(self): + returncode, _, stderr = self.run_python("test_not_in_gc") + if returncode: + self.fail(stderr) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py index 0061b3fa8e6555..4c3cf0b964ae56 100644 --- a/Lib/test/test_baseexception.py +++ b/Lib/test/test_baseexception.py @@ -114,6 +114,31 @@ def test_interface_no_arg(self): [repr(exc), exc.__class__.__name__ + '()']) self.interface_test_driver(results) + def test_setstate_refcount_no_crash(self): + # gh-97591: Acquire strong reference before calling tp_hash slot + # in PyObject_SetAttr. + import gc + d = {} + class HashThisKeyWillClearTheDict(str): + def __hash__(self) -> int: + d.clear() + return super().__hash__() + class Value(str): + pass + exc = Exception() + + d[HashThisKeyWillClearTheDict()] = Value() # refcount of Value() is 1 now + + # Exception.__setstate__ should aquire a strong reference of key and + # value in the dict. Otherwise, Value()'s refcount would go below + # zero in the tp_hash call in PyObject_SetAttr(), and it would cause + # crash in GC. + exc.__setstate__(d) # __hash__() is called again here, clearing the dict. + + # This GC would crash if the refcount of Value() goes below zero. + gc.collect() + + class UsageTests(unittest.TestCase): """Test usage of exceptions""" diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 64c74ec2c59ae0..6cd72eda774c2a 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -159,7 +159,7 @@ def test_import(self): __import__('string') __import__(name='sys') __import__(name='time', level=0) - self.assertRaises(ImportError, __import__, 'spamspam') + self.assertRaises(ModuleNotFoundError, __import__, 'spamspam') self.assertRaises(TypeError, __import__, 1, 2, 3, 4) self.assertRaises(ValueError, __import__, '') self.assertRaises(TypeError, __import__, 'sys', name='sys') @@ -737,6 +737,7 @@ def test_exec_globals(self): self.assertRaises(TypeError, exec, code, {'__builtins__': 123}) + def test_exec_globals_frozen(self): class frozendict_error(Exception): pass @@ -768,6 +769,36 @@ def __setitem__(self, key, value): self.assertRaises(frozendict_error, exec, code, namespace) + def test_exec_globals_error_on_get(self): + # custom `globals` or `builtins` can raise errors on item access + class setonlyerror(Exception): + pass + + class setonlydict(dict): + def __getitem__(self, key): + raise setonlyerror + + # globals' `__getitem__` raises + code = compile("globalname", "test", "exec") + self.assertRaises(setonlyerror, + exec, code, setonlydict({'globalname': 1})) + + # builtins' `__getitem__` raises + code = compile("superglobal", "test", "exec") + self.assertRaises(setonlyerror, exec, code, + {'__builtins__': setonlydict({'superglobal': 1})}) + + def test_exec_globals_dict_subclass(self): + class customdict(dict): # this one should not do anything fancy + pass + + code = compile("superglobal", "test", "exec") + # works correctly + exec(code, {'__builtins__': customdict({'superglobal': 1})}) + # custom builtins dict subclass is missing key + self.assertRaisesRegex(NameError, "name 'superglobal' is not defined", + exec, code, {'__builtins__': customdict()}) + def test_exec_redirected(self): savestdout = sys.stdout sys.stdout = None # Whatever that cannot flush() @@ -2345,7 +2376,7 @@ def test_type_name(self): self.assertEqual(A.__module__, __name__) with self.assertRaises(ValueError): type('A\x00B', (), {}) - with self.assertRaises(ValueError): + with self.assertRaises(UnicodeEncodeError): type('A\udcdcB', (), {}) with self.assertRaises(TypeError): type(b'A', (), {}) @@ -2362,7 +2393,7 @@ def test_type_name(self): with self.assertRaises(ValueError): A.__name__ = 'A\x00B' self.assertEqual(A.__name__, 'C') - with self.assertRaises(ValueError): + with self.assertRaises(UnicodeEncodeError): A.__name__ = 'A\udcdcB' self.assertEqual(A.__name__, 'C') with self.assertRaises(TypeError): diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 4c971bc5ed0586..0974002a93b6e1 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -558,7 +558,7 @@ def __index__(self): self.kwargs.clear() gc.collect() return 0 - x = IntWithDict(dont_inherit=IntWithDict()) + x = IntWithDict(optimize=IntWithDict()) # We test the argument handling of "compile" here, the compilation # itself is not relevant. When we pass flags=x below, x.__index__() is # called, which changes the keywords dict. diff --git a/Lib/test/test_capi/__init__.py b/Lib/test/test_capi/__init__.py new file mode 100644 index 00000000000000..4b16ecc31156a5 --- /dev/null +++ b/Lib/test/test_capi/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_capi/__main__.py b/Lib/test/test_capi/__main__.py new file mode 100644 index 00000000000000..05d01775ddf43c --- /dev/null +++ b/Lib/test/test_capi/__main__.py @@ -0,0 +1,3 @@ +import unittest + +unittest.main('test.test_capi') diff --git a/Lib/test/test_getargs2.py b/Lib/test/test_capi/test_getargs.py similarity index 100% rename from Lib/test/test_getargs2.py rename to Lib/test/test_capi/test_getargs.py diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi/test_misc.py similarity index 89% rename from Lib/test/test_capi.py rename to Lib/test/test_capi/test_misc.py index e157d9fdc850b6..6cda91677054d7 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi/test_misc.py @@ -139,8 +139,9 @@ def test_seq_bytes_to_charp_array(self): class Z(object): def __len__(self): return 1 - self.assertRaises(TypeError, _posixsubprocess.fork_exec, - 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23) + with self.assertRaisesRegex(TypeError, 'indexing'): + _posixsubprocess.fork_exec( + 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) # Issue #15736: overflow in _PySequence_BytesToCharpArray() class Z(object): def __len__(self): @@ -148,7 +149,7 @@ def __len__(self): def __getitem__(self, i): return b'x' self.assertRaises(MemoryError, _posixsubprocess.fork_exec, - 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23) + 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') def test_subprocess_fork_exec(self): @@ -158,7 +159,7 @@ def __len__(self): # Issue #15738: crash in subprocess_fork_exec() self.assertRaises(TypeError, _posixsubprocess.fork_exec, - Z(),[b'1'],3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23) + Z(),[b'1'],True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") @@ -403,6 +404,98 @@ def items(self): self.assertRaises(TypeError, _testcapi.get_mapping_values, bad_mapping) self.assertRaises(TypeError, _testcapi.get_mapping_items, bad_mapping) + def test_mapping_has_key(self): + dct = {'a': 1} + self.assertTrue(_testcapi.mapping_has_key(dct, 'a')) + self.assertFalse(_testcapi.mapping_has_key(dct, 'b')) + + class SubDict(dict): + pass + + dct2 = SubDict({'a': 1}) + self.assertTrue(_testcapi.mapping_has_key(dct2, 'a')) + self.assertFalse(_testcapi.mapping_has_key(dct2, 'b')) + + def test_sequence_set_slice(self): + # Correct case: + data = [1, 2, 3, 4, 5] + data_copy = data.copy() + + _testcapi.sequence_set_slice(data, 1, 3, [8, 9]) + data_copy[1:3] = [8, 9] + self.assertEqual(data, data_copy) + self.assertEqual(data, [1, 8, 9, 4, 5]) + + # Custom class: + class Custom: + def __setitem__(self, index, value): + self.index = index + self.value = value + + c = Custom() + _testcapi.sequence_set_slice(c, 0, 5, 'abc') + self.assertEqual(c.index, slice(0, 5)) + self.assertEqual(c.value, 'abc') + + # Immutable sequences must raise: + bad_seq1 = (1, 2, 3, 4) + with self.assertRaises(TypeError): + _testcapi.sequence_set_slice(bad_seq1, 1, 3, (8, 9)) + self.assertEqual(bad_seq1, (1, 2, 3, 4)) + + bad_seq2 = 'abcd' + with self.assertRaises(TypeError): + _testcapi.sequence_set_slice(bad_seq2, 1, 3, 'xy') + self.assertEqual(bad_seq2, 'abcd') + + # Not a sequence: + with self.assertRaises(TypeError): + _testcapi.sequence_set_slice(None, 1, 3, 'xy') + + mapping = {1: 'a', 2: 'b', 3: 'c'} + with self.assertRaises(TypeError): + _testcapi.sequence_set_slice(mapping, 1, 3, 'xy') + self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) + + def test_sequence_del_slice(self): + # Correct case: + data = [1, 2, 3, 4, 5] + data_copy = data.copy() + + _testcapi.sequence_del_slice(data, 1, 3) + del data_copy[1:3] + self.assertEqual(data, data_copy) + self.assertEqual(data, [1, 4, 5]) + + # Custom class: + class Custom: + def __delitem__(self, index): + self.index = index + + c = Custom() + _testcapi.sequence_del_slice(c, 0, 5) + self.assertEqual(c.index, slice(0, 5)) + + # Immutable sequences must raise: + bad_seq1 = (1, 2, 3, 4) + with self.assertRaises(TypeError): + _testcapi.sequence_del_slice(bad_seq1, 1, 3) + self.assertEqual(bad_seq1, (1, 2, 3, 4)) + + bad_seq2 = 'abcd' + with self.assertRaises(TypeError): + _testcapi.sequence_del_slice(bad_seq2, 1, 3) + self.assertEqual(bad_seq2, 'abcd') + + # Not a sequence: + with self.assertRaises(TypeError): + _testcapi.sequence_del_slice(None, 1, 3) + + mapping = {1: 'a', 2: 'b', 3: 'c'} + with self.assertRaises(TypeError): + _testcapi.sequence_del_slice(mapping, 1, 3) + self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) + @unittest.skipUnless(hasattr(_testcapi, 'negative_refcount'), 'need _testcapi.negative_refcount') def test_negative_refcount(self): @@ -724,6 +817,71 @@ def test_export_symbols(self): with self.subTest(name=name): self.assertTrue(hasattr(ctypes.pythonapi, name)) + def test_eval_get_func_name(self): + def function_example(): ... + + class A: + def method_example(self): ... + + self.assertEqual(_testcapi.eval_get_func_name(function_example), + "function_example") + self.assertEqual(_testcapi.eval_get_func_name(A.method_example), + "method_example") + self.assertEqual(_testcapi.eval_get_func_name(A().method_example), + "method_example") + self.assertEqual(_testcapi.eval_get_func_name(sum), "sum") # c function + self.assertEqual(_testcapi.eval_get_func_name(A), "type") + + def test_eval_get_func_desc(self): + def function_example(): ... + + class A: + def method_example(self): ... + + self.assertEqual(_testcapi.eval_get_func_desc(function_example), + "()") + self.assertEqual(_testcapi.eval_get_func_desc(A.method_example), + "()") + self.assertEqual(_testcapi.eval_get_func_desc(A().method_example), + "()") + self.assertEqual(_testcapi.eval_get_func_desc(sum), "()") # c function + self.assertEqual(_testcapi.eval_get_func_desc(A), " object") + + def test_function_get_code(self): + import types + + def some(): + pass + + code = _testcapi.function_get_code(some) + self.assertIsInstance(code, types.CodeType) + self.assertEqual(code, some.__code__) + + with self.assertRaises(SystemError): + _testcapi.function_get_code(None) # not a function + + def test_function_get_globals(self): + def some(): + pass + + globals_ = _testcapi.function_get_globals(some) + self.assertIsInstance(globals_, dict) + self.assertEqual(globals_, some.__globals__) + + with self.assertRaises(SystemError): + _testcapi.function_get_globals(None) # not a function + + def test_function_get_module(self): + def some(): + pass + + module = _testcapi.function_get_module(some) + self.assertIsInstance(module, str) + self.assertEqual(module, some.__module__) + + with self.assertRaises(SystemError): + _testcapi.function_get_module(None) # not a function + class TestPendingCalls(unittest.TestCase): diff --git a/Lib/test/test_structmembers.py b/Lib/test/test_capi/test_structmembers.py similarity index 100% rename from Lib/test/test_structmembers.py rename to Lib/test/test_capi/test_structmembers.py diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py new file mode 100644 index 00000000000000..77a180a30693dc --- /dev/null +++ b/Lib/test/test_capi/test_unicode.py @@ -0,0 +1,474 @@ +import unittest +import sys +from test import support +from test.support import import_helper + +try: + import _testcapi +except ImportError: + _testcapi = None + + +class CAPITest(unittest.TestCase): + + # Test PyUnicode_FromFormat() + def test_from_format(self): + import_helper.import_module('ctypes') + from ctypes import ( + c_char_p, + pythonapi, py_object, sizeof, + c_int, c_long, c_longlong, c_ssize_t, + c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) + name = "PyUnicode_FromFormat" + _PyUnicode_FromFormat = getattr(pythonapi, name) + _PyUnicode_FromFormat.argtypes = (c_char_p,) + _PyUnicode_FromFormat.restype = py_object + + def PyUnicode_FromFormat(format, *args): + cargs = tuple( + py_object(arg) if isinstance(arg, str) else arg + for arg in args) + return _PyUnicode_FromFormat(format, *cargs) + + def check_format(expected, format, *args): + text = PyUnicode_FromFormat(format, *args) + self.assertEqual(expected, text) + + # ascii format, non-ascii argument + check_format('ascii\x7f=unicode\xe9', + b'ascii\x7f=%U', 'unicode\xe9') + + # non-ascii format, ascii argument: ensure that PyUnicode_FromFormatV() + # raises an error + self.assertRaisesRegex(ValueError, + r'^PyUnicode_FromFormatV\(\) expects an ASCII-encoded format ' + 'string, got a non-ASCII byte: 0xe9$', + PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii') + + # test "%c" + check_format('\uabcd', + b'%c', c_int(0xabcd)) + check_format('\U0010ffff', + b'%c', c_int(0x10ffff)) + with self.assertRaises(OverflowError): + PyUnicode_FromFormat(b'%c', c_int(0x110000)) + # Issue #18183 + check_format('\U00010000\U00100000', + b'%c%c', c_int(0x10000), c_int(0x100000)) + + # test "%" + check_format('%', + b'%') + check_format('%', + b'%%') + check_format('%s', + b'%%s') + check_format('[%]', + b'[%%]') + check_format('%abc', + b'%%%s', b'abc') + + # truncated string + check_format('abc', + b'%.3s', b'abcdef') + check_format('abc[\ufffd', + b'%.5s', 'abc[\u20ac]'.encode('utf8')) + check_format("'\\u20acABC'", + b'%A', '\u20acABC') + check_format("'\\u20", + b'%.5A', '\u20acABCDEF') + check_format("'\u20acABC'", + b'%R', '\u20acABC') + check_format("'\u20acA", + b'%.3R', '\u20acABCDEF') + check_format('\u20acAB', + b'%.3S', '\u20acABCDEF') + check_format('\u20acAB', + b'%.3U', '\u20acABCDEF') + check_format('\u20acAB', + b'%.3V', '\u20acABCDEF', None) + check_format('abc[\ufffd', + b'%.5V', None, 'abc[\u20ac]'.encode('utf8')) + + # following tests comes from #7330 + # test width modifier and precision modifier with %S + check_format("repr= abc", + b'repr=%5S', 'abc') + check_format("repr=ab", + b'repr=%.2S', 'abc') + check_format("repr= ab", + b'repr=%5.2S', 'abc') + + # test width modifier and precision modifier with %R + check_format("repr= 'abc'", + b'repr=%8R', 'abc') + check_format("repr='ab", + b'repr=%.3R', 'abc') + check_format("repr= 'ab", + b'repr=%5.3R', 'abc') + + # test width modifier and precision modifier with %A + check_format("repr= 'abc'", + b'repr=%8A', 'abc') + check_format("repr='ab", + b'repr=%.3A', 'abc') + check_format("repr= 'ab", + b'repr=%5.3A', 'abc') + + # test width modifier and precision modifier with %s + check_format("repr= abc", + b'repr=%5s', b'abc') + check_format("repr=ab", + b'repr=%.2s', b'abc') + check_format("repr= ab", + b'repr=%5.2s', b'abc') + + # test width modifier and precision modifier with %U + check_format("repr= abc", + b'repr=%5U', 'abc') + check_format("repr=ab", + b'repr=%.2U', 'abc') + check_format("repr= ab", + b'repr=%5.2U', 'abc') + + # test width modifier and precision modifier with %V + check_format("repr= abc", + b'repr=%5V', 'abc', b'123') + check_format("repr=ab", + b'repr=%.2V', 'abc', b'123') + check_format("repr= ab", + b'repr=%5.2V', 'abc', b'123') + check_format("repr= 123", + b'repr=%5V', None, b'123') + check_format("repr=12", + b'repr=%.2V', None, b'123') + check_format("repr= 12", + b'repr=%5.2V', None, b'123') + + # test integer formats (%i, %d, %u) + check_format('010', + b'%03i', c_int(10)) + check_format('0010', + b'%0.4i', c_int(10)) + check_format('-123', + b'%i', c_int(-123)) + check_format('-123', + b'%li', c_long(-123)) + check_format('-123', + b'%lli', c_longlong(-123)) + check_format('-123', + b'%zi', c_ssize_t(-123)) + + check_format('-123', + b'%d', c_int(-123)) + check_format('-123', + b'%ld', c_long(-123)) + check_format('-123', + b'%lld', c_longlong(-123)) + check_format('-123', + b'%zd', c_ssize_t(-123)) + + check_format('123', + b'%u', c_uint(123)) + check_format('123', + b'%lu', c_ulong(123)) + check_format('123', + b'%llu', c_ulonglong(123)) + check_format('123', + b'%zu', c_size_t(123)) + + # test long output + min_longlong = -(2 ** (8 * sizeof(c_longlong) - 1)) + max_longlong = -min_longlong - 1 + check_format(str(min_longlong), + b'%lld', c_longlong(min_longlong)) + check_format(str(max_longlong), + b'%lld', c_longlong(max_longlong)) + max_ulonglong = 2 ** (8 * sizeof(c_ulonglong)) - 1 + check_format(str(max_ulonglong), + b'%llu', c_ulonglong(max_ulonglong)) + PyUnicode_FromFormat(b'%p', c_void_p(-1)) + + # test padding (width and/or precision) + check_format('123'.rjust(10, '0'), + b'%010i', c_int(123)) + check_format('123'.rjust(100), + b'%100i', c_int(123)) + check_format('123'.rjust(100, '0'), + b'%.100i', c_int(123)) + check_format('123'.rjust(80, '0').rjust(100), + b'%100.80i', c_int(123)) + + check_format('123'.rjust(10, '0'), + b'%010u', c_uint(123)) + check_format('123'.rjust(100), + b'%100u', c_uint(123)) + check_format('123'.rjust(100, '0'), + b'%.100u', c_uint(123)) + check_format('123'.rjust(80, '0').rjust(100), + b'%100.80u', c_uint(123)) + + check_format('123'.rjust(10, '0'), + b'%010x', c_int(0x123)) + check_format('123'.rjust(100), + b'%100x', c_int(0x123)) + check_format('123'.rjust(100, '0'), + b'%.100x', c_int(0x123)) + check_format('123'.rjust(80, '0').rjust(100), + b'%100.80x', c_int(0x123)) + + # test %A + check_format(r"%A:'abc\xe9\uabcd\U0010ffff'", + b'%%A:%A', 'abc\xe9\uabcd\U0010ffff') + + # test %V + check_format('repr=abc', + b'repr=%V', 'abc', b'xyz') + + # test %p + # We cannot test the exact result, + # because it returns a hex representation of a C pointer, + # which is going to be different each time. But, we can test the format. + p_format_regex = r'^0x[a-zA-Z0-9]{3,}$' + p_format1 = PyUnicode_FromFormat(b'%p', 'abc') + self.assertIsInstance(p_format1, str) + self.assertRegex(p_format1, p_format_regex) + + p_format2 = PyUnicode_FromFormat(b'%p %p', '123456', b'xyz') + self.assertIsInstance(p_format2, str) + self.assertRegex(p_format2, + r'0x[a-zA-Z0-9]{3,} 0x[a-zA-Z0-9]{3,}') + + # Extra args are ignored: + p_format3 = PyUnicode_FromFormat(b'%p', '123456', None, b'xyz') + self.assertIsInstance(p_format3, str) + self.assertRegex(p_format3, p_format_regex) + + # Test string decode from parameter of %s using utf-8. + # b'\xe4\xba\xba\xe6\xb0\x91' is utf-8 encoded byte sequence of + # '\u4eba\u6c11' + check_format('repr=\u4eba\u6c11', + b'repr=%V', None, b'\xe4\xba\xba\xe6\xb0\x91') + + #Test replace error handler. + check_format('repr=abc\ufffd', + b'repr=%V', None, b'abc\xff') + + # not supported: copy the raw format string. these tests are just here + # to check for crashes and should not be considered as specifications + check_format('%s', + b'%1%s', b'abc') + check_format('%1abc', + b'%1abc') + check_format('%+i', + b'%+i', c_int(10)) + check_format('%.%s', + b'%.%s', b'abc') + + # Issue #33817: empty strings + check_format('', + b'') + check_format('', + b'%s', b'') + + # Test PyUnicode_AsWideChar() + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_aswidechar(self): + from _testcapi import unicode_aswidechar + import_helper.import_module('ctypes') + from ctypes import c_wchar, sizeof + + wchar, size = unicode_aswidechar('abcdef', 2) + self.assertEqual(size, 2) + self.assertEqual(wchar, 'ab') + + wchar, size = unicode_aswidechar('abc', 3) + self.assertEqual(size, 3) + self.assertEqual(wchar, 'abc') + + wchar, size = unicode_aswidechar('abc', 4) + self.assertEqual(size, 3) + self.assertEqual(wchar, 'abc\0') + + wchar, size = unicode_aswidechar('abc', 10) + self.assertEqual(size, 3) + self.assertEqual(wchar, 'abc\0') + + wchar, size = unicode_aswidechar('abc\0def', 20) + self.assertEqual(size, 7) + self.assertEqual(wchar, 'abc\0def\0') + + nonbmp = chr(0x10ffff) + if sizeof(c_wchar) == 2: + buflen = 3 + nchar = 2 + else: # sizeof(c_wchar) == 4 + buflen = 2 + nchar = 1 + wchar, size = unicode_aswidechar(nonbmp, buflen) + self.assertEqual(size, nchar) + self.assertEqual(wchar, nonbmp + '\0') + + # Test PyUnicode_AsWideCharString() + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_aswidecharstring(self): + from _testcapi import unicode_aswidecharstring + import_helper.import_module('ctypes') + from ctypes import c_wchar, sizeof + + wchar, size = unicode_aswidecharstring('abc') + self.assertEqual(size, 3) + self.assertEqual(wchar, 'abc\0') + + wchar, size = unicode_aswidecharstring('abc\0def') + self.assertEqual(size, 7) + self.assertEqual(wchar, 'abc\0def\0') + + nonbmp = chr(0x10ffff) + if sizeof(c_wchar) == 2: + nchar = 2 + else: # sizeof(c_wchar) == 4 + nchar = 1 + wchar, size = unicode_aswidecharstring(nonbmp) + self.assertEqual(size, nchar) + self.assertEqual(wchar, nonbmp + '\0') + + # Test PyUnicode_AsUCS4() + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_asucs4(self): + from _testcapi import unicode_asucs4 + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', + 'a\ud800b\udfffc', '\ud834\udd1e']: + l = len(s) + self.assertEqual(unicode_asucs4(s, l, True), s+'\0') + self.assertEqual(unicode_asucs4(s, l, False), s+'\uffff') + self.assertEqual(unicode_asucs4(s, l+1, True), s+'\0\uffff') + self.assertEqual(unicode_asucs4(s, l+1, False), s+'\0\uffff') + self.assertRaises(SystemError, unicode_asucs4, s, l-1, True) + self.assertRaises(SystemError, unicode_asucs4, s, l-2, False) + s = '\0'.join([s, s]) + self.assertEqual(unicode_asucs4(s, len(s), True), s+'\0') + self.assertEqual(unicode_asucs4(s, len(s), False), s+'\uffff') + + # Test PyUnicode_AsUTF8() + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_asutf8(self): + from _testcapi import unicode_asutf8 + + bmp = '\u0100' + bmp2 = '\uffff' + nonbmp = chr(0x10ffff) + + self.assertEqual(unicode_asutf8(bmp), b'\xc4\x80') + self.assertEqual(unicode_asutf8(bmp2), b'\xef\xbf\xbf') + self.assertEqual(unicode_asutf8(nonbmp), b'\xf4\x8f\xbf\xbf') + self.assertRaises(UnicodeEncodeError, unicode_asutf8, 'a\ud800b\udfffc') + + # Test PyUnicode_AsUTF8AndSize() + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_asutf8andsize(self): + from _testcapi import unicode_asutf8andsize + + bmp = '\u0100' + bmp2 = '\uffff' + nonbmp = chr(0x10ffff) + + self.assertEqual(unicode_asutf8andsize(bmp), (b'\xc4\x80', 2)) + self.assertEqual(unicode_asutf8andsize(bmp2), (b'\xef\xbf\xbf', 3)) + self.assertEqual(unicode_asutf8andsize(nonbmp), (b'\xf4\x8f\xbf\xbf', 4)) + self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, 'a\ud800b\udfffc') + + # Test PyUnicode_FindChar() + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_findchar(self): + from _testcapi import unicode_findchar + + for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": + for i, ch in enumerate(str): + self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), 1), i) + self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), -1), i) + + str = "!>_= end + self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, 1), -1) + self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, 1), -1) + # negative + self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, 1), 0) + self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, -1), 0) + + # Test PyUnicode_CopyCharacters() + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_copycharacters(self): + from _testcapi import unicode_copycharacters + + strings = [ + 'abcde', '\xa1\xa2\xa3\xa4\xa5', + '\u4f60\u597d\u4e16\u754c\uff01', + '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' + ] + + for idx, from_ in enumerate(strings): + # wide -> narrow: exceed maxchar limitation + for to in strings[:idx]: + self.assertRaises( + SystemError, + unicode_copycharacters, to, 0, from_, 0, 5 + ) + # same kind + for from_start in range(5): + self.assertEqual( + unicode_copycharacters(from_, 0, from_, from_start, 5), + (from_[from_start:from_start+5].ljust(5, '\0'), + 5-from_start) + ) + for to_start in range(5): + self.assertEqual( + unicode_copycharacters(from_, to_start, from_, to_start, 5), + (from_[to_start:to_start+5].rjust(5, '\0'), + 5-to_start) + ) + # narrow -> wide + # Tests omitted since this creates invalid strings. + + s = strings[0] + self.assertRaises(IndexError, unicode_copycharacters, s, 6, s, 0, 5) + self.assertRaises(IndexError, unicode_copycharacters, s, -1, s, 0, 5) + self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, 6, 5) + self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, -1, 5) + self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, 5) + self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, -1) + self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_pep393_utf8_caching_bug(self): + # Issue #25709: Problem with string concatenation and utf-8 cache + from _testcapi import getargs_s_hash + for k in 0x24, 0xa4, 0x20ac, 0x1f40d: + s = '' + for i in range(5): + # Due to CPython specific optimization the 's' string can be + # resized in-place. + s += chr(k) + # Parsing with the "s#" format code calls indirectly + # PyUnicode_AsUTF8AndSize() which creates the UTF-8 + # encoded string cached in the Unicode object. + self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) + # Check that the second call returns the same result + self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 9e98edf2146ca9..4dadbc0b64bdb7 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -740,6 +740,23 @@ def test_nonexisting_script(self): self.assertIn(": can't open file ", err) self.assertNotEqual(proc.returncode, 0) + @unittest.skipUnless(os.path.exists('/dev/fd/0'), 'requires /dev/fd platform') + @unittest.skipIf(sys.platform.startswith("freebsd") and + os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, + "Requires fdescfs mounted on /dev/fd on FreeBSD") + def test_script_as_dev_fd(self): + # GH-87235: On macOS passing a non-trivial script to /dev/fd/N can cause + # problems because all open /dev/fd/N file descriptors share the same + # offset. + script = 'print("12345678912345678912345")' + with os_helper.temp_dir() as work_dir: + script_name = _make_test_script(work_dir, 'script.py', script) + with open(script_name, "r") as fp: + p = spawn_python(f"/dev/fd/{fp.fileno()}", close_fds=False, pass_fds=(0,1,2,fp.fileno())) + out, err = p.communicate() + self.assertEqual(out, b"12345678912345678912345\n") + + def tearDownModule(): support.reap_children() diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 2386cf6b59f396..d3e20129eee8ef 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -132,6 +132,7 @@ import unittest import textwrap import weakref +import dis try: import ctypes @@ -671,6 +672,38 @@ def test_lines(self): self.check_lines(misshappen) self.check_lines(bug93662) + @cpython_only + def test_code_new_empty(self): + # If this test fails, it means that the construction of PyCode_NewEmpty + # needs to be modified! Please update this test *and* PyCode_NewEmpty, + # so that they both stay in sync. + def f(): + pass + PY_CODE_LOCATION_INFO_NO_COLUMNS = 13 + f.__code__ = f.__code__.replace( + co_firstlineno=42, + co_code=bytes( + [ + dis.opmap["RESUME"], 0, + dis.opmap["LOAD_ASSERTION_ERROR"], 0, + dis.opmap["RAISE_VARARGS"], 1, + ] + ), + co_linetable=bytes( + [ + (1 << 7) + | (PY_CODE_LOCATION_INFO_NO_COLUMNS << 3) + | (3 - 1), + 0, + ] + ), + ) + self.assertRaises(AssertionError, f) + self.assertEqual( + list(f.__code__.co_positions()), + 3 * [(42, 42, None, None)], + ) + if check_impl_detail(cpython=True) and ctypes is not None: py = ctypes.pythonapi diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 42c600dcb00de1..934e4bb347ef3f 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -709,7 +709,8 @@ def test_decoder_state(self): "spamspam", self.spambe) def test_bug691291(self): - # Files are always opened in binary mode, even if no binary mode was + # If encoding is not None, then + # files are always opened in binary mode, even if no binary mode was # specified. This means that no automatic conversion of '\n' is done # on reading and writing. s1 = 'Hello\r\nworld\r\n' @@ -1552,6 +1553,12 @@ def test_builtin_encode(self): self.assertEqual("pyth\xf6n.org".encode("idna"), b"xn--pythn-mua.org") self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.") + def test_builtin_decode_length_limit(self): + with self.assertRaisesRegex(UnicodeError, "too long"): + (b"xn--016c"+b"a"*1100).decode("idna") + with self.assertRaisesRegex(UnicodeError, "too long"): + (b"xn--016c"+b"a"*70).decode("idna") + def test_stream(self): r = codecs.getreader("idna")(io.BytesIO(b"abc")) r.read(3) diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index 17376c7ed7537e..133096d25a44bc 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -321,6 +321,26 @@ def test_warning(self): warnings.simplefilter('error', SyntaxWarning) compile_command('1 is 1', symbol='exec') + # Check DeprecationWarning treated as an SyntaxError + with warnings.catch_warnings(), self.assertRaises(SyntaxError): + warnings.simplefilter('error', DeprecationWarning) + compile_command(r"'\e'", symbol='exec') + + def test_incomplete_warning(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + self.assertIncomplete("'\\e' + (") + self.assertEqual(w, []) + + def test_invalid_warning(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + self.assertInvalid("'\\e' 1") + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, DeprecationWarning) + self.assertRegex(str(w[0].message), 'invalid escape sequence') + self.assertEqual(w[0].filename, '') + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index fa1d0e014dee92..5da446a13da43c 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -802,6 +802,8 @@ def throw(self, typ, val=None, tb=None): def __await__(self): yield + self.validate_abstract_methods(Awaitable, '__await__') + non_samples = [None, int(), gen(), object()] for x in non_samples: self.assertNotIsInstance(x, Awaitable) @@ -852,6 +854,8 @@ def throw(self, typ, val=None, tb=None): def __await__(self): yield + self.validate_abstract_methods(Coroutine, '__await__', 'send', 'throw') + non_samples = [None, int(), gen(), object(), Bar()] for x in non_samples: self.assertNotIsInstance(x, Coroutine) @@ -1594,6 +1598,7 @@ def __len__(self): containers = [ seq, ItemsView({1: nan, 2: obj}), + KeysView({1: nan, 2: obj}), ValuesView({1: nan, 2: obj}) ] for container in containers: @@ -1857,6 +1862,8 @@ def test_MutableMapping_subclass(self): mymap['red'] = 5 self.assertIsInstance(mymap.keys(), Set) self.assertIsInstance(mymap.keys(), KeysView) + self.assertIsInstance(mymap.values(), Collection) + self.assertIsInstance(mymap.values(), ValuesView) self.assertIsInstance(mymap.items(), Set) self.assertIsInstance(mymap.items(), ItemsView) @@ -1932,6 +1939,7 @@ def test_ByteString(self): self.assertFalse(issubclass(sample, ByteString)) self.assertNotIsInstance(memoryview(b""), ByteString) self.assertFalse(issubclass(memoryview, ByteString)) + self.validate_abstract_methods(ByteString, '__getitem__', '__len__') def test_MutableSequence(self): for sample in [tuple, str, bytes]: diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index cd4d58acd4778e..54e90663ab5100 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1039,6 +1039,32 @@ def while_not_chained(a, b, c): for instr in dis.Bytecode(while_not_chained): self.assertNotEqual(instr.opname, "EXTENDED_ARG") + def test_compare_positions(self): + for opname, op in [ + ("COMPARE_OP", "<"), + ("COMPARE_OP", "<="), + ("COMPARE_OP", ">"), + ("COMPARE_OP", ">="), + ("CONTAINS_OP", "in"), + ("CONTAINS_OP", "not in"), + ("IS_OP", "is"), + ("IS_OP", "is not"), + ]: + expr = f'a {op} b {op} c' + expected_positions = 2 * [(2, 2, 0, len(expr))] + for source in [ + f"\\\n{expr}", f'if \\\n{expr}: x', f"x if \\\n{expr} else y" + ]: + code = compile(source, "", "exec") + actual_positions = [ + instruction.positions + for instruction in dis.get_instructions(code) + if instruction.opname == opname + ] + with self.subTest(source): + self.assertEqual(actual_positions, expected_positions) + + @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): # Ensure that compiled code snippets have correct line and column numbers @@ -1368,6 +1394,13 @@ def test_stack_3050(self): # This raised on 3.10.0 to 3.10.5 compile(code, "", "single") + def test_stack_3050_2(self): + M = 3050 + args = ", ".join(f"arg{i}:type{i}" for i in range(M)) + code = f"def f({args}):\n pass" + # This raised on 3.10.0 to 3.10.5 + compile(code, "", "single") + class TestStackSizeStability(unittest.TestCase): # Check that repeating certain snippets doesn't increase the stack size diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index e046577b935e92..51ba151505fb54 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -306,15 +306,10 @@ def test_conjugate(self): self.assertClose(complex(5.3, 9.8).conjugate(), 5.3-9.8j) def test_constructor(self): - class OS: + class NS: def __init__(self, value): self.value = value def __complex__(self): return self.value - class NS(object): - def __init__(self, value): self.value = value - def __complex__(self): return self.value - self.assertEqual(complex(OS(1+10j)), 1+10j) self.assertEqual(complex(NS(1+10j)), 1+10j) - self.assertRaises(TypeError, complex, OS(None)) self.assertRaises(TypeError, complex, NS(None)) self.assertRaises(TypeError, complex, {}) self.assertRaises(TypeError, complex, NS(1.5)) diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 8fff2d47c10fd5..10f1a9efbcbdd6 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1280,7 +1280,7 @@ async def __aexit__(self, *exc): async def func(): async with CM(): - assert (1, ) == 1 + self.assertEqual((1, ), 1) with self.assertRaises(AssertionError): run_async(func()) @@ -2411,7 +2411,8 @@ class UnawaitedWarningDuringShutdownTest(unittest.TestCase): def test_unawaited_warning_during_shutdown(self): code = ("import asyncio\n" "async def f(): pass\n" - "asyncio.gather(f())\n") + "async def t(): asyncio.gather(f())\n" + "asyncio.run(t())\n") assert_python_ok("-c", code) code = ("import sys\n" diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 95a19dd46cb4ff..834217bf6030dc 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -362,6 +362,11 @@ def test_read_quoting(self): self._read_test(['1,@,3,@,5'], [['1', ',3,', '5']], quotechar='@') self._read_test(['1,\0,3,\0,5'], [['1', ',3,', '5']], quotechar='\0') + def test_read_skipinitialspace(self): + self._read_test(['no space, space, spaces,\ttab'], + [['no space', 'space', 'spaces', '\ttab']], + skipinitialspace=True) + def test_read_bigfield(self): # This exercises the buffer realloc functionality and field size # limits. @@ -448,6 +453,34 @@ def test_register_kwargs(self): self.assertEqual(csv.get_dialect(name).delimiter, ';') self.assertEqual([['X', 'Y', 'Z']], list(csv.reader(['X;Y;Z'], name))) + def test_register_kwargs_override(self): + class mydialect(csv.Dialect): + delimiter = "\t" + quotechar = '"' + doublequote = True + skipinitialspace = False + lineterminator = '\r\n' + quoting = csv.QUOTE_MINIMAL + + name = 'test_dialect' + csv.register_dialect(name, mydialect, + delimiter=';', + quotechar="'", + doublequote=False, + skipinitialspace=True, + lineterminator='\n', + quoting=csv.QUOTE_ALL) + self.addCleanup(csv.unregister_dialect, name) + + # Ensure that kwargs do override attributes of a dialect class: + dialect = csv.get_dialect(name) + self.assertEqual(dialect.delimiter, ';') + self.assertEqual(dialect.quotechar, "'") + self.assertEqual(dialect.doublequote, False) + self.assertEqual(dialect.skipinitialspace, True) + self.assertEqual(dialect.lineterminator, '\n') + self.assertEqual(dialect.quoting, csv.QUOTE_ALL) + def test_incomplete_dialect(self): class myexceltsv(csv.Dialect): delimiter = "\t" diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 63380ea0b68067..0d809bd2eb9ab0 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -231,6 +231,14 @@ class C: c = C('foo') self.assertEqual(c.object, 'foo') + def test_field_named_BUILTINS_frozen(self): + # gh-96151 + @dataclass(frozen=True) + class C: + BUILTINS: int + c = C(5) + self.assertEqual(c.BUILTINS, 5) + def test_field_named_like_builtin(self): # Attribute names can shadow built-in names # since code generation is used. diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 33d9c6def9727d..67ccaab40c5edc 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -37,7 +37,7 @@ requires_legacy_unicode_capi, check_sanitizer) from test.support import (TestFailed, run_with_locale, cpython_only, - darwin_malloc_err_warning) + darwin_malloc_err_warning, is_emscripten) from test.support.import_helper import import_fresh_module from test.support import threading_helper from test.support import warnings_helper @@ -5623,6 +5623,7 @@ def __abs__(self): # Issue 41540: @unittest.skipIf(sys.platform.startswith("aix"), "AIX: default ulimit: test is flaky because of extreme over-allocation") + @unittest.skipIf(is_emscripten, "Test is unstable on Emscripten") @unittest.skipIf(check_sanitizer(address=True, memory=True), "ASAN/MSAN sanitizer defaults to crashing " "instead of returning NULL for malloc failure.") diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index afe0f7e9c7fd42..3145dff81b4d89 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1310,6 +1310,15 @@ class X(object): with self.assertRaisesRegex(AttributeError, "'X' object has no attribute 'a'"): X().a + # Test string subclass in `__slots__`, see gh-98783 + class SubStr(str): + pass + class X(object): + __slots__ = (SubStr('x'),) + X().x = 1 + with self.assertRaisesRegex(AttributeError, "'X' object has no attribute 'a'"): + X().a + def test_slots_special(self): # Testing __dict__ and __weakref__ in __slots__... class D(object): @@ -3581,6 +3590,16 @@ def __repr__(self): self.assertEqual(o.__str__(), '41') self.assertEqual(o.__repr__(), 'A repr') + def test_repr_with_module_str_subclass(self): + # gh-98783 + class StrSub(str): + pass + class Some: + pass + Some.__module__ = StrSub('example') + self.assertIsInstance(repr(Some), str) # should not crash + self.assertIsInstance(repr(Some()), str) # should not crash + def test_keyword_arguments(self): # Testing keyword arguments to __init__, __call__... def f(a): return a diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index be271bebaaf1ff..dae93740d46c7c 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -320,6 +320,9 @@ def test_abc_registry(self): self.assertIsInstance(d.values(), collections.abc.ValuesView) self.assertIsInstance(d.values(), collections.abc.MappingView) self.assertIsInstance(d.values(), collections.abc.Sized) + self.assertIsInstance(d.values(), collections.abc.Collection) + self.assertIsInstance(d.values(), collections.abc.Iterable) + self.assertIsInstance(d.values(), collections.abc.Container) self.assertIsInstance(d.items(), collections.abc.ItemsView) self.assertIsInstance(d.items(), collections.abc.MappingView) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 6d16021a612051..6a0a2d92796703 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1076,7 +1076,7 @@ def get_disassembly(self, func, lasti=-1, wrapper=True, **kwargs): return output.getvalue() -if sys.flags.optimize: +if dis.code_info.__doc__ is None: code_info_consts = "0: None" else: code_info_consts = "0: 'Formatted details of methods, functions, or code.'" diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index b8e3c37bbda3fe..be7980eebe15df 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -341,6 +341,12 @@ def test_finalize_structseq(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, 'Tests passed\n' * INIT_LOOPS) + def test_simple_initialization_api(self): + # _testembed now uses Py_InitializeFromConfig by default + # This case specifically checks Py_Initialize(Ex) still works + out, err = self.run_embedded_interpreter("test_repeated_simple_init") + self.assertEqual(out, 'Finalized\n' * INIT_LOOPS) + def test_quickened_static_code_gets_unquickened_at_Py_FINALIZE(self): # https://github.com/python/cpython/issues/92031 @@ -1473,17 +1479,11 @@ def test_init_pyvenv_cfg(self): if not MS_WINDOWS: paths[-1] = lib_dynload else: - # Include DLLs directory as well - paths.insert(1, '.\\DLLs') - for index, path in enumerate(paths): - if index == 0: - # Because we copy the DLLs into tmpdir as well, the zip file - # entry in sys.path will be there. For a regular venv, it will - # usually be in the home directory. - paths[index] = os.path.join(tmpdir, os.path.basename(path)) - else: - paths[index] = os.path.join(pyvenv_home, os.path.basename(path)) - paths[-1] = pyvenv_home + paths = [ + os.path.join(tmpdir, os.path.basename(paths[0])), + pyvenv_home, + os.path.join(pyvenv_home, "Lib"), + ] executable = self.test_exe base_executable = os.path.join(pyvenv_home, os.path.basename(executable)) @@ -1500,12 +1500,12 @@ def test_init_pyvenv_cfg(self): config['base_prefix'] = pyvenv_home config['prefix'] = pyvenv_home config['stdlib_dir'] = os.path.join(pyvenv_home, 'Lib') - config['use_frozen_modules'] = not Py_DEBUG + config['use_frozen_modules'] = int(not Py_DEBUG) else: # cannot reliably assume stdlib_dir here because it # depends too much on our build. But it ought to be found config['stdlib_dir'] = self.IGNORE_CONFIG - config['use_frozen_modules'] = not Py_DEBUG + config['use_frozen_modules'] = int(not Py_DEBUG) env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 8cd1fe1c106c8a..a9b80ad0d3dbc3 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -14,7 +14,7 @@ from enum import Enum, IntEnum, StrEnum, EnumType, Flag, IntFlag, unique, auto from enum import STRICT, CONFORM, EJECT, KEEP, _simple_enum, _test_simple_enum from enum import verify, UNIQUE, CONTINUOUS, NAMED_FLAGS, ReprEnum -from enum import member, nonmember +from enum import member, nonmember, _iter_bits_lsb from io import StringIO from pickle import dumps, loads, PicklingError, HIGHEST_PROTOCOL from test import support @@ -174,6 +174,10 @@ def test_is_private(self): for name in self.sunder_names + self.dunder_names + self.random_names: self.assertFalse(enum._is_private('MyEnum', name), '%r is a private name?') + def test_iter_bits_lsb(self): + self.assertEqual(list(_iter_bits_lsb(7)), [1, 2, 4]) + self.assertRaisesRegex(ValueError, '-8 is not a positive integer', list, _iter_bits_lsb(-8)) + # for subclassing tests @@ -764,7 +768,7 @@ class _FlagTests: def test_default_missing_with_wrong_type_value(self): with self.assertRaisesRegex( ValueError, - "'RED' is not a valid TestFlag.Color", + "'RED' is not a valid ", ) as ctx: self.MainEnum('RED') self.assertIs(ctx.exception.__context__, None) @@ -773,7 +777,7 @@ class TestPlainEnum(_EnumTests, _PlainOutputTests, unittest.TestCase): enum_type = Enum -class TestPlainFlag(_EnumTests, _PlainOutputTests, unittest.TestCase): +class TestPlainFlag(_EnumTests, _PlainOutputTests, _FlagTests, unittest.TestCase): enum_type = Flag @@ -785,7 +789,7 @@ class TestStrEnum(_EnumTests, _MinimalOutputTests, unittest.TestCase): enum_type = StrEnum -class TestIntFlag(_EnumTests, _MinimalOutputTests, unittest.TestCase): +class TestIntFlag(_EnumTests, _MinimalOutputTests, _FlagTests, unittest.TestCase): enum_type = IntFlag @@ -797,7 +801,7 @@ class TestMixedStr(_EnumTests, _MixedOutputTests, unittest.TestCase): class enum_type(str, Enum): pass -class TestMixedIntFlag(_EnumTests, _MixedOutputTests, unittest.TestCase): +class TestMixedIntFlag(_EnumTests, _MixedOutputTests, _FlagTests, unittest.TestCase): class enum_type(int, Flag): pass @@ -3965,6 +3969,16 @@ class Sillier(IntEnum): triple = 3 value = 4 + def test_negative_alias(self): + @verify(NAMED_FLAGS) + class Color(Flag): + RED = 1 + GREEN = 2 + BLUE = 4 + WHITE = -1 + # no error means success + + class TestInternals(unittest.TestCase): sunder_names = '_bad_', '_good_', '_what_ho_' @@ -4121,6 +4135,50 @@ class Dupes(Enum): third = auto() self.assertEqual([Dupes.first, Dupes.second, Dupes.third], list(Dupes)) + def test_multiple_auto_on_line(self): + class Huh(Enum): + ONE = auto() + TWO = auto(), auto() + THREE = auto(), auto(), auto() + self.assertEqual(Huh.ONE.value, 1) + self.assertEqual(Huh.TWO.value, (2, 3)) + self.assertEqual(Huh.THREE.value, (4, 5, 6)) + # + class Hah(Enum): + def __new__(cls, value, abbr=None): + member = object.__new__(cls) + member._value_ = value + member.abbr = abbr or value[:3].lower() + return member + def _generate_next_value_(name, start, count, last): + return name + # + MONDAY = auto() + TUESDAY = auto() + WEDNESDAY = auto(), 'WED' + THURSDAY = auto(), 'Thu' + FRIDAY = auto() + self.assertEqual(Hah.MONDAY.value, 'MONDAY') + self.assertEqual(Hah.MONDAY.abbr, 'mon') + self.assertEqual(Hah.TUESDAY.value, 'TUESDAY') + self.assertEqual(Hah.TUESDAY.abbr, 'tue') + self.assertEqual(Hah.WEDNESDAY.value, 'WEDNESDAY') + self.assertEqual(Hah.WEDNESDAY.abbr, 'WED') + self.assertEqual(Hah.THURSDAY.value, 'THURSDAY') + self.assertEqual(Hah.THURSDAY.abbr, 'Thu') + self.assertEqual(Hah.FRIDAY.value, 'FRIDAY') + self.assertEqual(Hah.FRIDAY.abbr, 'fri') + # + class Huh(Enum): + def _generate_next_value_(name, start, count, last): + return count+1 + ONE = auto() + TWO = auto(), auto() + THREE = auto(), auto(), auto() + self.assertEqual(Huh.ONE.value, 1) + self.assertEqual(Huh.TWO.value, (2, 2)) + self.assertEqual(Huh.THREE.value, (3, 3, 3)) + class TestEnumTypeSubclassing(unittest.TestCase): pass diff --git a/Lib/test/test_except_star.py b/Lib/test/test_except_star.py index dbe8eff32924ed..9de72dbd5a3264 100644 --- a/Lib/test/test_except_star.py +++ b/Lib/test/test_except_star.py @@ -1000,5 +1000,204 @@ def test_exc_info_restored(self): self.assertEqual(sys.exc_info(), (None, None, None)) +class TestExceptStar_WeirdLeafExceptions(ExceptStarTest): + # Test that except* works when leaf exceptions are + # unhashable or have a bad custom __eq__ + + class UnhashableExc(ValueError): + __hash__ = None + + class AlwaysEqualExc(ValueError): + def __eq__(self, other): + return True + + class NeverEqualExc(ValueError): + def __eq__(self, other): + return False + + class BrokenEqualExc(ValueError): + def __eq__(self, other): + raise RuntimeError() + + def setUp(self): + self.bad_types = [self.UnhashableExc, + self.AlwaysEqualExc, + self.NeverEqualExc, + self.BrokenEqualExc] + + def except_type(self, eg, type): + match, rest = None, None + try: + try: + raise eg + except* type as e: + match = e + except Exception as e: + rest = e + return match, rest + + def test_catch_unhashable_leaf_exception(self): + for Bad in self.bad_types: + with self.subTest(Bad): + eg = ExceptionGroup("eg", [TypeError(1), Bad(2)]) + match, rest = self.except_type(eg, Bad) + self.assertExceptionIsLike( + match, ExceptionGroup("eg", [Bad(2)])) + self.assertExceptionIsLike( + rest, ExceptionGroup("eg", [TypeError(1)])) + + def test_propagate_unhashable_leaf(self): + for Bad in self.bad_types: + with self.subTest(Bad): + eg = ExceptionGroup("eg", [TypeError(1), Bad(2)]) + match, rest = self.except_type(eg, TypeError) + self.assertExceptionIsLike( + match, ExceptionGroup("eg", [TypeError(1)])) + self.assertExceptionIsLike( + rest, ExceptionGroup("eg", [Bad(2)])) + + def test_catch_nothing_unhashable_leaf(self): + for Bad in self.bad_types: + with self.subTest(Bad): + eg = ExceptionGroup("eg", [TypeError(1), Bad(2)]) + match, rest = self.except_type(eg, OSError) + self.assertIsNone(match) + self.assertExceptionIsLike(rest, eg) + + def test_catch_everything_unhashable_leaf(self): + for Bad in self.bad_types: + with self.subTest(Bad): + eg = ExceptionGroup("eg", [TypeError(1), Bad(2)]) + match, rest = self.except_type(eg, Exception) + self.assertExceptionIsLike(match, eg) + self.assertIsNone(rest) + + def test_reraise_unhashable_leaf(self): + for Bad in self.bad_types: + with self.subTest(Bad): + eg = ExceptionGroup( + "eg", [TypeError(1), Bad(2), ValueError(3)]) + + try: + try: + raise eg + except* TypeError: + pass + except* Bad: + raise + except Exception as e: + exc = e + + self.assertExceptionIsLike( + exc, ExceptionGroup("eg", [Bad(2), ValueError(3)])) + + +class TestExceptStar_WeirdExceptionGroupSubclass(ExceptStarTest): + # Test that except* works with exception groups that are + # unhashable or have a bad custom __eq__ + + class UnhashableEG(ExceptionGroup): + __hash__ = None + + def derive(self, excs): + return type(self)(self.message, excs) + + class AlwaysEqualEG(ExceptionGroup): + def __eq__(self, other): + return True + + def derive(self, excs): + return type(self)(self.message, excs) + + class NeverEqualEG(ExceptionGroup): + def __eq__(self, other): + return False + + def derive(self, excs): + return type(self)(self.message, excs) + + class BrokenEqualEG(ExceptionGroup): + def __eq__(self, other): + raise RuntimeError() + + def derive(self, excs): + return type(self)(self.message, excs) + + def setUp(self): + self.bad_types = [self.UnhashableEG, + self.AlwaysEqualEG, + self.NeverEqualEG, + self.BrokenEqualEG] + + def except_type(self, eg, type): + match, rest = None, None + try: + try: + raise eg + except* type as e: + match = e + except Exception as e: + rest = e + return match, rest + + def test_catch_some_unhashable_exception_group_subclass(self): + for BadEG in self.bad_types: + with self.subTest(BadEG): + eg = BadEG("eg", + [TypeError(1), + BadEG("nested", [ValueError(2)])]) + + match, rest = self.except_type(eg, TypeError) + self.assertExceptionIsLike(match, BadEG("eg", [TypeError(1)])) + self.assertExceptionIsLike(rest, + BadEG("eg", [BadEG("nested", [ValueError(2)])])) + + def test_catch_none_unhashable_exception_group_subclass(self): + for BadEG in self.bad_types: + with self.subTest(BadEG): + + eg = BadEG("eg", + [TypeError(1), + BadEG("nested", [ValueError(2)])]) + + match, rest = self.except_type(eg, OSError) + self.assertIsNone(match) + self.assertExceptionIsLike(rest, eg) + + def test_catch_all_unhashable_exception_group_subclass(self): + for BadEG in self.bad_types: + with self.subTest(BadEG): + + eg = BadEG("eg", + [TypeError(1), + BadEG("nested", [ValueError(2)])]) + + match, rest = self.except_type(eg, Exception) + self.assertExceptionIsLike(match, eg) + self.assertIsNone(rest) + + def test_reraise_unhashable_eg(self): + for BadEG in self.bad_types: + with self.subTest(BadEG): + + eg = BadEG("eg", + [TypeError(1), ValueError(2), + BadEG("nested", [ValueError(3), OSError(4)])]) + + try: + try: + raise eg + except* ValueError: + pass + except* OSError: + raise + except Exception as e: + exc = e + + self.assertExceptionIsLike( + exc, BadEG("eg", [TypeError(1), + BadEG("nested", [OSError(4)])])) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 2cfd8738304d1c..7fb45462e20f4b 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -79,16 +79,30 @@ def test_BEG_wraps_BaseException__creates_BEG(self): beg = BaseExceptionGroup("beg", [ValueError(1), KeyboardInterrupt(2)]) self.assertIs(type(beg), BaseExceptionGroup) - def test_EG_subclass_wraps_anything(self): + def test_EG_subclass_wraps_non_base_exceptions(self): class MyEG(ExceptionGroup): pass self.assertIs( type(MyEG("eg", [ValueError(12), TypeError(42)])), MyEG) - self.assertIs( - type(MyEG("eg", [ValueError(12), KeyboardInterrupt(42)])), - MyEG) + + def test_EG_subclass_does_not_wrap_base_exceptions(self): + class MyEG(ExceptionGroup): + pass + + msg = "Cannot nest BaseExceptions in 'MyEG'" + with self.assertRaisesRegex(TypeError, msg): + MyEG("eg", [ValueError(12), KeyboardInterrupt(42)]) + + def test_BEG_and_E_subclass_does_not_wrap_base_exceptions(self): + class MyEG(BaseExceptionGroup, ValueError): + pass + + msg = "Cannot nest BaseExceptions in 'MyEG'" + with self.assertRaisesRegex(TypeError, msg): + MyEG("eg", [ValueError(12), KeyboardInterrupt(42)]) + def test_BEG_subclass_wraps_anything(self): class MyBEG(BaseExceptionGroup): diff --git a/Lib/test/test_exception_hierarchy.py b/Lib/test/test_exception_hierarchy.py index 89fe9ddcefba3e..3318fa8e7746f7 100644 --- a/Lib/test/test_exception_hierarchy.py +++ b/Lib/test/test_exception_hierarchy.py @@ -63,7 +63,7 @@ def test_select_error(self): +-- InterruptedError EINTR +-- IsADirectoryError EISDIR +-- NotADirectoryError ENOTDIR - +-- PermissionError EACCES, EPERM + +-- PermissionError EACCES, EPERM, ENOTCAPABLE +-- ProcessLookupError ESRCH +-- TimeoutError ETIMEDOUT """ @@ -75,6 +75,8 @@ def _make_map(s): continue excname, _, errnames = line.partition(' ') for errname in filter(None, errnames.strip().split(', ')): + if errname == "ENOTCAPABLE" and not hasattr(errno, errname): + continue _map[getattr(errno, errname)] = getattr(builtins, excname) return _map _map = _make_map(_pep_map) @@ -91,7 +93,7 @@ def test_errno_mapping(self): othercodes = set(errno.errorcode) - set(self._map) for errcode in othercodes: e = OSError(errcode, "Some message") - self.assertIs(type(e), OSError) + self.assertIs(type(e), OSError, repr(e)) def test_try_except(self): filename = "some_hopefully_non_existing_file" diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index e153bc5c7c7350..9cb2686f17597a 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -1,11 +1,15 @@ import gc import re import sys +import textwrap +import threading import types import unittest import weakref from test import support +from test.support import threading_helper +from test.support.script_helper import assert_python_ok class ClearTest(unittest.TestCase): @@ -239,25 +243,26 @@ def inner(): class TestIncompleteFrameAreInvisible(unittest.TestCase): def test_issue95818(self): - #See GH-95818 for details - import gc - self.addCleanup(gc.set_threshold, *gc.get_threshold()) + # See GH-95818 for details + code = textwrap.dedent(f""" + import gc - gc.set_threshold(1,1,1) - class GCHello: - def __del__(self): - print("Destroyed from gc") - - def gen(): - yield + gc.set_threshold(1,1,1) + class GCHello: + def __del__(self): + print("Destroyed from gc") - fd = open(__file__) - l = [fd, GCHello()] - l.append(l) - del fd - del l - gen() + def gen(): + yield + fd = open({__file__!r}) + l = [fd, GCHello()] + l.append(l) + del fd + del l + gen() + """) + assert_python_ok("-c", code) @support.cpython_only def test_sneaky_frame_object(self): @@ -322,6 +327,46 @@ def f(): if old_enabled: gc.enable() + @support.cpython_only + @threading_helper.requires_working_threading() + def test_sneaky_frame_object_teardown(self): + + class SneakyDel: + def __del__(self): + """ + Stash a reference to the entire stack for walking later. + + It may look crazy, but you'd be surprised how common this is + when using a test runner (like pytest). The typical recipe is: + ResourceWarning + -Werror + a custom sys.unraisablehook. + """ + nonlocal sneaky_frame_object + sneaky_frame_object = sys._getframe() + + class SneakyThread(threading.Thread): + """ + A separate thread isn't needed to make this code crash, but it does + make crashes more consistent, since it means sneaky_frame_object is + backed by freed memory after the thread completes! + """ + + def run(self): + """Run SneakyDel.__del__ as this frame is popped.""" + ref = SneakyDel() + + sneaky_frame_object = None + t = SneakyThread() + t.start() + t.join() + # sneaky_frame_object can be anything, really, but it's crucial that + # SneakyThread.run's frame isn't anywhere on the stack while it's being + # torn down: + self.assertIsNotNone(sneaky_frame_object) + while sneaky_frame_object is not None: + self.assertIsNot( + sneaky_frame_object.f_code, SneakyThread.run.__code__ + ) + sneaky_frame_object = sneaky_frame_object.f_back if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 6959c2ae3c80e3..e44193a0f72da2 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -203,23 +203,11 @@ class MyList(list): self.assertEqual(repr(list[str]), 'list[str]') self.assertEqual(repr(list[()]), 'list[()]') self.assertEqual(repr(tuple[int, ...]), 'tuple[int, ...]') - x1 = tuple[ - tuple( # Effectively the same as starring; TODO - tuple[int] - ) - ] + x1 = tuple[*tuple[int]] self.assertEqual(repr(x1), 'tuple[*tuple[int]]') - x2 = tuple[ - tuple( # Ditto TODO - tuple[int, str] - ) - ] + x2 = tuple[*tuple[int, str]] self.assertEqual(repr(x2), 'tuple[*tuple[int, str]]') - x3 = tuple[ - tuple( # Ditto TODO - tuple[int, ...] - ) - ] + x3 = tuple[*tuple[int, ...]] self.assertEqual(repr(x3), 'tuple[*tuple[int, ...]]') self.assertTrue(repr(MyList[int]).endswith('.BaseTest.test_repr..MyList[int]')) self.assertEqual(repr(list[str]()), '[]') # instances should keep their normal repr @@ -273,42 +261,24 @@ def test_parameters(self): self.assertEqual(L5.__args__, (Callable[[K, V], K],)) self.assertEqual(L5.__parameters__, (K, V)) - T1 = tuple[ - tuple( # Ditto TODO - tuple[int] - ) - ] + T1 = tuple[*tuple[int]] self.assertEqual( T1.__args__, - tuple( # Ditto TODO - tuple[int] - ) + (*tuple[int],), ) self.assertEqual(T1.__parameters__, ()) - T2 = tuple[ - tuple( # Ditto TODO - tuple[T] - ) - ] + T2 = tuple[*tuple[T]] self.assertEqual( T2.__args__, - tuple( # Ditto TODO - tuple[T] - ) + (*tuple[T],), ) self.assertEqual(T2.__parameters__, (T,)) - T4 = tuple[ - tuple( # Ditto TODO - tuple[int, str] - ) - ] + T4 = tuple[*tuple[int, str]] self.assertEqual( T4.__args__, - tuple( # Ditto TODO - tuple[int, str] - ) + (*tuple[int, str],), ) self.assertEqual(T4.__parameters__, ()) @@ -343,18 +313,7 @@ def test_equality(self): self.assertEqual(list[int], list[int]) self.assertEqual(dict[str, int], dict[str, int]) self.assertEqual((*tuple[int],)[0], (*tuple[int],)[0]) - self.assertEqual( - tuple[ - tuple( # Effectively the same as starring; TODO - tuple[int] - ) - ], - tuple[ - tuple( # Ditto TODO - tuple[int] - ) - ] - ) + self.assertEqual(tuple[*tuple[int]], tuple[*tuple[int]]) self.assertNotEqual(dict[str, int], dict[str, str]) self.assertNotEqual(list, list[int]) self.assertNotEqual(list[int], list) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 5208374e20013c..87932b891f808a 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -239,6 +239,29 @@ def test_buildtree_pythonhome_win32(self): actual = getpath(ns, expected) self.assertEqual(expected, actual) + def test_no_dlls_win32(self): + "Test a layout on Windows with no DLLs directory." + ns = MockNTNamespace( + argv0=r"C:\Python\python.exe", + real_executable=r"C:\Python\python.exe", + ) + ns.add_known_xfile(r"C:\Python\python.exe") + ns.add_known_file(r"C:\Python\Lib\os.py") + expected = dict( + executable=r"C:\Python\python.exe", + base_executable=r"C:\Python\python.exe", + prefix=r"C:\Python", + exec_prefix=r"C:\Python", + module_search_paths_set=1, + module_search_paths=[ + r"C:\Python\python98.zip", + r"C:\Python\Lib", + r"C:\Python", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + def test_normal_posix(self): "Test a 'standard' install layout on *nix" ns = MockPosixNamespace( @@ -360,6 +383,70 @@ def test_venv_changed_name_posix(self): actual = getpath(ns, expected) self.assertEqual(expected, actual) + def test_venv_non_installed_zip_path_posix(self): + "Test a venv created from non-installed python has correct zip path.""" + ns = MockPosixNamespace( + argv0="/venv/bin/python", + PREFIX="/usr", + ENV_PATH="/venv/bin:/usr/bin", + ) + ns.add_known_xfile("/path/to/non-installed/bin/python") + ns.add_known_xfile("/venv/bin/python") + ns.add_known_link("/venv/bin/python", + "/path/to/non-installed/bin/python") + ns.add_known_file("/path/to/non-installed/lib/python9.8/os.py") + ns.add_known_dir("/path/to/non-installed/lib/python9.8/lib-dynload") + ns.add_known_file("/venv/pyvenv.cfg", [ + r"home = /path/to/non-installed" + ]) + expected = dict( + executable="/venv/bin/python", + prefix="/path/to/non-installed", + exec_prefix="/path/to/non-installed", + base_executable="/path/to/non-installed/bin/python", + base_prefix="/path/to/non-installed", + base_exec_prefix="/path/to/non-installed", + module_search_paths_set=1, + module_search_paths=[ + "/path/to/non-installed/lib/python98.zip", + "/path/to/non-installed/lib/python9.8", + "/path/to/non-installed/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_venv_changed_name_copy_posix(self): + "Test a venv --copies layout on *nix that lacks a distributed 'python'" + ns = MockPosixNamespace( + argv0="python", + PREFIX="/usr", + ENV_PATH="/venv/bin:/usr/bin", + ) + ns.add_known_xfile("/usr/bin/python9") + ns.add_known_xfile("/venv/bin/python") + ns.add_known_file("/usr/lib/python9.8/os.py") + ns.add_known_dir("/usr/lib/python9.8/lib-dynload") + ns.add_known_file("/venv/pyvenv.cfg", [ + r"home = /usr/bin" + ]) + expected = dict( + executable="/venv/bin/python", + prefix="/usr", + exec_prefix="/usr", + base_executable="/usr/bin/python9", + base_prefix="/usr", + base_exec_prefix="/usr", + module_search_paths_set=1, + module_search_paths=[ + "/usr/lib/python98.zip", + "/usr/lib/python9.8", + "/usr/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + def test_symlink_normal_posix(self): "Test a 'standard' install layout via symlink on *nix" ns = MockPosixNamespace( diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index a937258069ed89..ca078862cca6b9 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -26,7 +26,7 @@ import datetime import threading from unittest import mock -from io import BytesIO +from io import BytesIO, StringIO import unittest from test import support @@ -990,6 +990,27 @@ def verify_http_server_response(self, response): match = self.HTTPResponseMatch.search(response) self.assertIsNotNone(match) + def test_unprintable_not_logged(self): + # We call the method from the class directly as our Socketless + # Handler subclass overrode it... nice for everything BUT this test. + self.handler.client_address = ('127.0.0.1', 1337) + log_message = BaseHTTPRequestHandler.log_message + with mock.patch.object(sys, 'stderr', StringIO()) as fake_stderr: + log_message(self.handler, '/foo') + log_message(self.handler, '/\033bar\000\033') + log_message(self.handler, '/spam %s.', 'a') + log_message(self.handler, '/spam %s.', '\033\x7f\x9f\xa0beans') + log_message(self.handler, '"GET /foo\\b"ar\007 HTTP/1.0"') + stderr = fake_stderr.getvalue() + self.assertNotIn('\033', stderr) # non-printable chars are caught. + self.assertNotIn('\000', stderr) # non-printable chars are caught. + lines = stderr.splitlines() + self.assertIn('/foo', lines[0]) + self.assertIn(r'/\x1bbar\x00\x1b', lines[1]) + self.assertIn('/spam a.', lines[2]) + self.assertIn('/spam \\x1b\\x7f\\x9f\xa0beans.', lines[3]) + self.assertIn(r'"GET /foo\\b"ar\x07 HTTP/1.0"', lines[4]) + def test_http_1_1(self): result = self.send_typical_request(b'GET / HTTP/1.1\r\n\r\n') self.verify_http_server_response(result[0]) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index d44dc6b49f2935..4bb03908fc2bb2 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -1,3 +1,4 @@ +import gc import importlib import importlib.util import os @@ -383,6 +384,35 @@ def test_find_and_load_checked_pyc(self): self.assertEqual(mod.x, 42) + @support.cpython_only + def test_create_builtin_subinterp(self): + # gh-99578: create_builtin() behavior changes after the creation of the + # first sub-interpreter. Test both code paths, before and after the + # creation of a sub-interpreter. Previously, create_builtin() had + # a reference leak after the creation of the first sub-interpreter. + + import builtins + create_builtin = support.get_attribute(_imp, "create_builtin") + class Spec: + name = "builtins" + spec = Spec() + + def check_get_builtins(): + refcnt = sys.getrefcount(builtins) + mod = _imp.create_builtin(spec) + self.assertIs(mod, builtins) + self.assertEqual(sys.getrefcount(builtins), refcnt + 1) + # Check that a GC collection doesn't crash + gc.collect() + + check_get_builtins() + + ret = support.run_in_subinterp("import builtins") + self.assertEqual(ret, 0) + + check_get_builtins() + + class ReloadTests(unittest.TestCase): """Very basic tests to make sure that imp.reload() operates just like diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/fixtures.py index 803d3738d263f4..e7be77b3957c67 100644 --- a/Lib/test/test_importlib/fixtures.py +++ b/Lib/test/test_importlib/fixtures.py @@ -5,6 +5,7 @@ import pathlib import tempfile import textwrap +import functools import contextlib from test.support.os_helper import FS_NONASCII @@ -296,3 +297,18 @@ def setUp(self): # Add self.zip_name to the front of sys.path. self.resources = contextlib.ExitStack() self.addCleanup(self.resources.close) + + +def parameterize(*args_set): + """Run test method with a series of parameters.""" + + def wrapper(func): + @functools.wraps(func) + def _inner(self): + for args in args_set: + with self.subTest(**args): + func(self, **args) + + return _inner + + return wrapper diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/test_main.py index c80a0e4a5a9f91..d9d067c4b23d66 100644 --- a/Lib/test/test_importlib/test_main.py +++ b/Lib/test/test_importlib/test_main.py @@ -1,7 +1,6 @@ import re import json import pickle -import textwrap import unittest import warnings import importlib.metadata @@ -16,6 +15,7 @@ Distribution, EntryPoint, PackageNotFoundError, + _unique, distributions, entry_points, metadata, @@ -51,6 +51,14 @@ def test_package_not_found_mentions_metadata(self): def test_new_style_classes(self): self.assertIsInstance(Distribution, type) + @fixtures.parameterize( + dict(name=None), + dict(name=''), + ) + def test_invalid_inputs_to_from_name(self, name): + with self.assertRaises(Exception): + Distribution.from_name(name) + class ImportTests(fixtures.DistInfoPkg, unittest.TestCase): def test_import_nonexistent_module(self): @@ -78,48 +86,50 @@ def test_resolve_without_attr(self): class NameNormalizationTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): @staticmethod - def pkg_with_dashes(site_dir): + def make_pkg(name): """ - Create minimal metadata for a package with dashes - in the name (and thus underscores in the filename). + Create minimal metadata for a dist-info package with + the indicated name on the file system. """ - metadata_dir = site_dir / 'my_pkg.dist-info' - metadata_dir.mkdir() - metadata = metadata_dir / 'METADATA' - with metadata.open('w', encoding='utf-8') as strm: - strm.write('Version: 1.0\n') - return 'my-pkg' + return { + f'{name}.dist-info': { + 'METADATA': 'VERSION: 1.0\n', + }, + } def test_dashes_in_dist_name_found_as_underscores(self): """ For a package with a dash in the name, the dist-info metadata uses underscores in the name. Ensure the metadata loads. """ - pkg_name = self.pkg_with_dashes(self.site_dir) - assert version(pkg_name) == '1.0' - - @staticmethod - def pkg_with_mixed_case(site_dir): - """ - Create minimal metadata for a package with mixed case - in the name. - """ - metadata_dir = site_dir / 'CherryPy.dist-info' - metadata_dir.mkdir() - metadata = metadata_dir / 'METADATA' - with metadata.open('w', encoding='utf-8') as strm: - strm.write('Version: 1.0\n') - return 'CherryPy' + fixtures.build_files(self.make_pkg('my_pkg'), self.site_dir) + assert version('my-pkg') == '1.0' def test_dist_name_found_as_any_case(self): """ Ensure the metadata loads when queried with any case. """ - pkg_name = self.pkg_with_mixed_case(self.site_dir) + pkg_name = 'CherryPy' + fixtures.build_files(self.make_pkg(pkg_name), self.site_dir) assert version(pkg_name) == '1.0' assert version(pkg_name.lower()) == '1.0' assert version(pkg_name.upper()) == '1.0' + def test_unique_distributions(self): + """ + Two distributions varying only by non-normalized name on + the file system should resolve as the same. + """ + fixtures.build_files(self.make_pkg('abc'), self.site_dir) + before = list(_unique(distributions())) + + alt_site_dir = self.fixtures.enter_context(fixtures.tempdir()) + self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) + fixtures.build_files(self.make_pkg('ABC'), alt_site_dir) + after = list(_unique(distributions())) + + assert len(after) == len(before) + class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): @staticmethod @@ -128,11 +138,12 @@ def pkg_with_non_ascii_description(site_dir): Create minimal metadata for a package with non-ASCII in the description. """ - metadata_dir = site_dir / 'portend.dist-info' - metadata_dir.mkdir() - metadata = metadata_dir / 'METADATA' - with metadata.open('w', encoding='utf-8') as fp: - fp.write('Description: pôrˈtend') + contents = { + 'portend.dist-info': { + 'METADATA': 'Description: pôrˈtend', + }, + } + fixtures.build_files(contents, site_dir) return 'portend' @staticmethod @@ -141,19 +152,15 @@ def pkg_with_non_ascii_description_egg_info(site_dir): Create minimal metadata for an egg-info package with non-ASCII in the description. """ - metadata_dir = site_dir / 'portend.dist-info' - metadata_dir.mkdir() - metadata = metadata_dir / 'METADATA' - with metadata.open('w', encoding='utf-8') as fp: - fp.write( - textwrap.dedent( - """ + contents = { + 'portend.dist-info': { + 'METADATA': """ Name: portend - pôrˈtend - """ - ).strip() - ) + pôrˈtend""", + }, + } + fixtures.build_files(contents, site_dir) return 'portend' def test_metadata_loads(self): diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/test_metadata_api.py index c86bb4df19c5eb..69c78e9820c044 100644 --- a/Lib/test/test_importlib/test_metadata_api.py +++ b/Lib/test/test_importlib/test_metadata_api.py @@ -89,15 +89,15 @@ def test_entry_points_distribution(self): self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg')) self.assertEqual(ep.dist.version, "1.0.0") - def test_entry_points_unique_packages(self): + def test_entry_points_unique_packages_normalized(self): """ Entry points should only be exposed for the first package - on sys.path with a given name. + on sys.path with a given name (even when normalized). """ alt_site_dir = self.fixtures.enter_context(fixtures.tempdir()) self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) alt_pkg = { - "distinfo_pkg-1.1.0.dist-info": { + "DistInfo_pkg-1.1.0.dist-info": { "METADATA": """ Name: distinfo-pkg Version: 1.1.0 diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index c07ac2a64c289f..0b6dcc5eaf03d2 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -298,7 +298,7 @@ def writes_bytecode_files(fxn): """Decorator to protect sys.dont_write_bytecode from mutation and to skip tests that require it to be set to False.""" if sys.dont_write_bytecode: - return lambda *args, **kwargs: None + return unittest.skip("relies on writing bytecode")(fxn) @functools.wraps(fxn) def wrapper(*args, **kwargs): original = sys.dont_write_bytecode diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index be9f29e04ae110..a54e6eb53e680c 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -1421,6 +1421,13 @@ def wrapper(a, b): self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations, eval_str=True), {'x': int}) +class TestFormatAnnotation(unittest.TestCase): + def test_typing_replacement(self): + from test.typinganndata.ann_module9 import ann, ann1 + self.assertEqual(inspect.formatannotation(ann), 'Union[List[str], int]') + self.assertEqual(inspect.formatannotation(ann1), 'Union[List[testModule.typing.A], int]') + + class TestIsDataDescriptor(unittest.TestCase): def test_custom_descriptors(self): @@ -2953,8 +2960,6 @@ def foo(a): pass self.assertEqual(str(inspect.signature(foo)), '(a)') def test_signature_on_decorated(self): - import functools - def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs) -> int: @@ -2966,6 +2971,8 @@ class Foo: def bar(self, a, b): pass + bar = decorator(Foo().bar) + self.assertEqual(self.signature(Foo.bar), ((('self', ..., ..., "positional_or_keyword"), ('a', ..., ..., "positional_or_keyword"), @@ -2984,6 +2991,11 @@ def bar(self, a, b): # from "func" to "wrapper", hence no # return_annotation + self.assertEqual(self.signature(bar), + ((('a', ..., ..., "positional_or_keyword"), + ('b', ..., ..., "positional_or_keyword")), + ...)) + # Test that we handle method wrappers correctly def decorator(func): @functools.wraps(func) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index daccbae5b4a1da..79aa2da58622b2 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -888,6 +888,14 @@ def badopener(fname, flags): open('non-existent', 'r', opener=badopener) self.assertEqual(str(cm.exception), 'opener returned -2') + def test_opener_invalid_fd(self): + # Check that OSError is raised with error code EBADF if the + # opener returns an invalid file descriptor (see gh-82212). + fd = os_helper.make_bad_fd() + with self.assertRaises(OSError) as cm: + self.open('foo', opener=lambda name, flags: fd) + self.assertEqual(cm.exception.errno, errno.EBADF) + def test_fileio_closefd(self): # Issue #4841 with self.open(__file__, 'rb') as f1, \ @@ -3923,7 +3931,15 @@ def test_translate(self): self.assertEqual(decoder.decode(b"\r\r\n"), "\r\r\n") class CIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest): - pass + @support.cpython_only + def test_uninitialized(self): + uninitialized = self.IncrementalNewlineDecoder.__new__( + self.IncrementalNewlineDecoder) + self.assertRaises(ValueError, uninitialized.decode, b'bar') + self.assertRaises(ValueError, uninitialized.getstate) + self.assertRaises(ValueError, uninitialized.setstate, (b'foo', 0)) + self.assertRaises(ValueError, uninitialized.reset) + class PyIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest): pass diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index c9ae7dab387ced..a5388b2e5debd8 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -1652,7 +1652,7 @@ def testNth(self): self.assertRaises(IndexError, self.ipv6_scoped_network.__getitem__, 1 << 64) def testGetitem(self): - # http://code.google.com/p/ipaddr-py/issues/detail?id=15 + # https://code.google.com/p/ipaddr-py/issues/detail?id=15 addr = ipaddress.IPv4Network('172.31.255.128/255.255.255.240') self.assertEqual(28, addr.prefixlen) addr_list = list(addr) @@ -2277,6 +2277,39 @@ def testReservedIpv4(self): self.assertEqual(False, ipaddress.ip_address('128.0.0.0').is_loopback) self.assertEqual(True, ipaddress.ip_network('0.0.0.0').is_unspecified) + def testPrivateNetworks(self): + self.assertEqual(False, ipaddress.ip_network("0.0.0.0/0").is_private) + self.assertEqual(False, ipaddress.ip_network("1.0.0.0/8").is_private) + + self.assertEqual(True, ipaddress.ip_network("0.0.0.0/8").is_private) + self.assertEqual(True, ipaddress.ip_network("10.0.0.0/8").is_private) + self.assertEqual(True, ipaddress.ip_network("127.0.0.0/8").is_private) + self.assertEqual(True, ipaddress.ip_network("169.254.0.0/16").is_private) + self.assertEqual(True, ipaddress.ip_network("172.16.0.0/12").is_private) + self.assertEqual(True, ipaddress.ip_network("192.0.0.0/29").is_private) + self.assertEqual(True, ipaddress.ip_network("192.0.0.170/31").is_private) + self.assertEqual(True, ipaddress.ip_network("192.0.2.0/24").is_private) + self.assertEqual(True, ipaddress.ip_network("192.168.0.0/16").is_private) + self.assertEqual(True, ipaddress.ip_network("198.18.0.0/15").is_private) + self.assertEqual(True, ipaddress.ip_network("198.51.100.0/24").is_private) + self.assertEqual(True, ipaddress.ip_network("203.0.113.0/24").is_private) + self.assertEqual(True, ipaddress.ip_network("240.0.0.0/4").is_private) + self.assertEqual(True, ipaddress.ip_network("255.255.255.255/32").is_private) + + self.assertEqual(False, ipaddress.ip_network("::/0").is_private) + self.assertEqual(False, ipaddress.ip_network("::ff/128").is_private) + + self.assertEqual(True, ipaddress.ip_network("::1/128").is_private) + self.assertEqual(True, ipaddress.ip_network("::/128").is_private) + self.assertEqual(True, ipaddress.ip_network("::ffff:0:0/96").is_private) + self.assertEqual(True, ipaddress.ip_network("100::/64").is_private) + self.assertEqual(True, ipaddress.ip_network("2001::/23").is_private) + self.assertEqual(True, ipaddress.ip_network("2001:2::/48").is_private) + self.assertEqual(True, ipaddress.ip_network("2001:db8::/32").is_private) + self.assertEqual(True, ipaddress.ip_network("2001:10::/28").is_private) + self.assertEqual(True, ipaddress.ip_network("fc00::/7").is_private) + self.assertEqual(True, ipaddress.ip_network("fe80::/10").is_private) + def testReservedIpv6(self): self.assertEqual(True, ipaddress.ip_network('ffff::').is_multicast) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 554f602f6252c0..acbdcb5f302060 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -81,6 +81,16 @@ class BadIterableClass: def __iter__(self): raise ZeroDivisionError +class CallableIterClass: + def __init__(self): + self.i = 0 + def __call__(self): + i = self.i + self.i = i + 1 + if i > 100: + raise IndexError # Emergency stop + return i + # Main test suite class TestCase(unittest.TestCase): @@ -237,16 +247,7 @@ def __iter__(self): # Test two-argument iter() with callable instance def test_iter_callable(self): - class C: - def __init__(self): - self.i = 0 - def __call__(self): - i = self.i - self.i = i + 1 - if i > 100: - raise IndexError # Emergency stop - return i - self.check_iterator(iter(C(), 10), list(range(10)), pickle=False) + self.check_iterator(iter(CallableIterClass(), 10), list(range(10)), pickle=True) # Test two-argument iter() with function def test_iter_function(self): diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 238afbbd883d3a..311c2a3288dc8c 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -636,6 +636,7 @@ def test_cycle(self): self.assertRaises(TypeError, cycle, 5) self.assertEqual(list(islice(cycle(gen3()),10)), [0,1,2,0,1,2,0,1,2,0]) + def test_cycle_copy_pickle(self): # check copy, deepcopy, pickle c = cycle('abc') self.assertEqual(next(c), 'a') @@ -671,6 +672,37 @@ def test_cycle(self): d = pickle.loads(p) # rebuild the cycle object self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + def test_cycle_unpickle_compat(self): + testcases = [ + b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI0\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aK\x00tb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', + b'\x80\x04\x95=\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', + + b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI1\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00K\x01tb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', + b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', + + b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI00\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aI00\ntb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', + b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', + + b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI01\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00I01\ntb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', + b'\x80\x04\x95;\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', + ] + assert len(testcases) == 20 + for t in testcases: + it = pickle.loads(t) + self.assertEqual(take(10, it), [2, 3, 1, 2, 3, 1, 2, 3, 1, 2]) + def test_cycle_setstate(self): # Verify both modes for restoring state diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index eb9064edea9115..efc982e8b0eb04 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -2,73 +2,73 @@ # 2007-10-05 JSONDOCS = [ - # http://json.org/JSON_checker/test/fail1.json + # https://json.org/JSON_checker/test/fail1.json '"A JSON payload should be an object or array, not a string."', - # http://json.org/JSON_checker/test/fail2.json + # https://json.org/JSON_checker/test/fail2.json '["Unclosed array"', - # http://json.org/JSON_checker/test/fail3.json + # https://json.org/JSON_checker/test/fail3.json '{unquoted_key: "keys must be quoted"}', - # http://json.org/JSON_checker/test/fail4.json + # https://json.org/JSON_checker/test/fail4.json '["extra comma",]', - # http://json.org/JSON_checker/test/fail5.json + # https://json.org/JSON_checker/test/fail5.json '["double extra comma",,]', - # http://json.org/JSON_checker/test/fail6.json + # https://json.org/JSON_checker/test/fail6.json '[ , "<-- missing value"]', - # http://json.org/JSON_checker/test/fail7.json + # https://json.org/JSON_checker/test/fail7.json '["Comma after the close"],', - # http://json.org/JSON_checker/test/fail8.json + # https://json.org/JSON_checker/test/fail8.json '["Extra close"]]', - # http://json.org/JSON_checker/test/fail9.json + # https://json.org/JSON_checker/test/fail9.json '{"Extra comma": true,}', - # http://json.org/JSON_checker/test/fail10.json + # https://json.org/JSON_checker/test/fail10.json '{"Extra value after close": true} "misplaced quoted value"', - # http://json.org/JSON_checker/test/fail11.json + # https://json.org/JSON_checker/test/fail11.json '{"Illegal expression": 1 + 2}', - # http://json.org/JSON_checker/test/fail12.json + # https://json.org/JSON_checker/test/fail12.json '{"Illegal invocation": alert()}', - # http://json.org/JSON_checker/test/fail13.json + # https://json.org/JSON_checker/test/fail13.json '{"Numbers cannot have leading zeroes": 013}', - # http://json.org/JSON_checker/test/fail14.json + # https://json.org/JSON_checker/test/fail14.json '{"Numbers cannot be hex": 0x14}', - # http://json.org/JSON_checker/test/fail15.json + # https://json.org/JSON_checker/test/fail15.json '["Illegal backslash escape: \\x15"]', - # http://json.org/JSON_checker/test/fail16.json + # https://json.org/JSON_checker/test/fail16.json '[\\naked]', - # http://json.org/JSON_checker/test/fail17.json + # https://json.org/JSON_checker/test/fail17.json '["Illegal backslash escape: \\017"]', - # http://json.org/JSON_checker/test/fail18.json + # https://json.org/JSON_checker/test/fail18.json '[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', - # http://json.org/JSON_checker/test/fail19.json + # https://json.org/JSON_checker/test/fail19.json '{"Missing colon" null}', - # http://json.org/JSON_checker/test/fail20.json + # https://json.org/JSON_checker/test/fail20.json '{"Double colon":: null}', - # http://json.org/JSON_checker/test/fail21.json + # https://json.org/JSON_checker/test/fail21.json '{"Comma instead of colon", null}', - # http://json.org/JSON_checker/test/fail22.json + # https://json.org/JSON_checker/test/fail22.json '["Colon instead of comma": false]', - # http://json.org/JSON_checker/test/fail23.json + # https://json.org/JSON_checker/test/fail23.json '["Bad value", truth]', - # http://json.org/JSON_checker/test/fail24.json + # https://json.org/JSON_checker/test/fail24.json "['single quote']", - # http://json.org/JSON_checker/test/fail25.json + # https://json.org/JSON_checker/test/fail25.json '["\ttab\tcharacter\tin\tstring\t"]', - # http://json.org/JSON_checker/test/fail26.json + # https://json.org/JSON_checker/test/fail26.json '["tab\\ character\\ in\\ string\\ "]', - # http://json.org/JSON_checker/test/fail27.json + # https://json.org/JSON_checker/test/fail27.json '["line\nbreak"]', - # http://json.org/JSON_checker/test/fail28.json + # https://json.org/JSON_checker/test/fail28.json '["line\\\nbreak"]', - # http://json.org/JSON_checker/test/fail29.json + # https://json.org/JSON_checker/test/fail29.json '[0e]', - # http://json.org/JSON_checker/test/fail30.json + # https://json.org/JSON_checker/test/fail30.json '[0e+]', - # http://json.org/JSON_checker/test/fail31.json + # https://json.org/JSON_checker/test/fail31.json '[0e+-1]', - # http://json.org/JSON_checker/test/fail32.json + # https://json.org/JSON_checker/test/fail32.json '{"Comma instead if closing brace": true,', - # http://json.org/JSON_checker/test/fail33.json + # https://json.org/JSON_checker/test/fail33.json '["mismatch"}', - # http://code.google.com/p/simplejson/issues/detail?id=3 + # https://code.google.com/archive/p/simplejson/issues/3 '["A\u001FZ control characters in string"]', ] diff --git a/Lib/test/test_json/test_pass1.py b/Lib/test/test_json/test_pass1.py index 15e64b0aeae709..26bf3cdbd77303 100644 --- a/Lib/test/test_json/test_pass1.py +++ b/Lib/test/test_json/test_pass1.py @@ -1,7 +1,7 @@ from test.test_json import PyTest, CTest -# from http://json.org/JSON_checker/test/pass1.json +# from https://json.org/JSON_checker/test/pass1.json JSON = r''' [ "JSON Test Pattern pass1", diff --git a/Lib/test/test_json/test_pass2.py b/Lib/test/test_json/test_pass2.py index 35075249e3bc6f..9340de665aabe5 100644 --- a/Lib/test/test_json/test_pass2.py +++ b/Lib/test/test_json/test_pass2.py @@ -1,7 +1,7 @@ from test.test_json import PyTest, CTest -# from http://json.org/JSON_checker/test/pass2.json +# from https://json.org/JSON_checker/test/pass2.json JSON = r''' [[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] ''' diff --git a/Lib/test/test_json/test_pass3.py b/Lib/test/test_json/test_pass3.py index cd0cf170d275c5..0adccc1c2a5316 100644 --- a/Lib/test/test_json/test_pass3.py +++ b/Lib/test/test_json/test_pass3.py @@ -1,7 +1,7 @@ from test.test_json import PyTest, CTest -# from http://json.org/JSON_checker/test/pass3.json +# from https://json.org/JSON_checker/test/pass3.json JSON = r''' { "JSON Test Pattern pass3": { diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py index ba6856b3e2466f..3991a8b4606b9b 100644 --- a/Lib/test/test_launcher.py +++ b/Lib/test/test_launcher.py @@ -174,7 +174,7 @@ def find_py(cls): errors="ignore", ) as p: p.stdin.close() - version = next(p.stdout).splitlines()[0].rpartition(" ")[2] + version = next(p.stdout, "\n").splitlines()[0].rpartition(" ")[2] p.stdout.read() p.wait(10) if not sys.version.startswith(version): @@ -468,6 +468,15 @@ def test_py3_default_env(self): self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) self.assertEqual("X.Y-arm64.exe -X fake_arg_for_test -arg", data["stdout"].strip()) + def test_py_default_short_argv0(self): + with self.py_ini(TEST_PY_COMMANDS): + for argv0 in ['"py.exe"', 'py.exe', '"py"', 'py']: + with self.subTest(argv0): + data = self.run_py(["--version"], argv=f'{argv0} --version') + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual(f'X.Y.exe --version', data["stdout"].strip()) + def test_py_default_in_list(self): data = self.run_py(["-0"], env=TEST_PY_ENV) default = None @@ -517,6 +526,14 @@ def test_py_shebang(self): self.assertEqual("3.100", data["SearchInfo.tag"]) self.assertEqual(f"X.Y.exe -prearg {script} -postarg", data["stdout"].strip()) + def test_python_shebang(self): + with self.py_ini(TEST_PY_COMMANDS): + with self.script("#! python -prearg") as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y.exe -prearg {script} -postarg", data["stdout"].strip()) + def test_py2_shebang(self): with self.py_ini(TEST_PY_COMMANDS): with self.script("#! /usr/bin/python2 -prearg") as script: @@ -618,3 +635,42 @@ def test_install(self): self.assertIn("winget.exe", cmd) # Both command lines include the store ID self.assertIn("9PJPW5LDXLZ5", cmd) + + def test_literal_shebang_absolute(self): + with self.script(f"#! C:/some_random_app -witharg") as script: + data = self.run_py([script]) + self.assertEqual( + f"C:\\some_random_app -witharg {script}", + data["stdout"].strip(), + ) + + def test_literal_shebang_relative(self): + with self.script(f"#! ..\\some_random_app -witharg") as script: + data = self.run_py([script]) + self.assertEqual( + f"{script.parent.parent}\\some_random_app -witharg {script}", + data["stdout"].strip(), + ) + + def test_literal_shebang_quoted(self): + with self.script(f'#! "some random app" -witharg') as script: + data = self.run_py([script]) + self.assertEqual( + f'"{script.parent}\\some random app" -witharg {script}', + data["stdout"].strip(), + ) + + with self.script(f'#! some" random "app -witharg') as script: + data = self.run_py([script]) + self.assertEqual( + f'"{script.parent}\\some random app" -witharg {script}', + data["stdout"].strip(), + ) + + def test_literal_shebang_quoted_escape(self): + with self.script(f'#! some\\" random "app -witharg') as script: + data = self.run_py([script]) + self.assertEqual( + f'"{script.parent}\\some\\ random app" -witharg {script}', + data["stdout"].strip(), + ) diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index d092e0176c2616..77b37ca1fa4afe 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -1334,6 +1334,12 @@ def equivalent_python(n, length, byteorder, signed=False): b'\xff\xff\xff\xff\xff') self.assertRaises(OverflowError, (1).to_bytes, 0, 'big') + # gh-98783 + class SubStr(str): + pass + self.assertEqual((0).to_bytes(1, SubStr('big')), b'\x00') + self.assertEqual((0).to_bytes(0, SubStr('little')), b'') + def test_from_bytes(self): def check(tests, byteorder, signed=False): def equivalent_python(byte_array, byteorder, signed=False): @@ -1518,6 +1524,28 @@ def __init__(self, value): self.assertEqual(i, 1) self.assertEqual(getattr(i, 'foo', 'none'), 'bar') + class ValidBytes: + def __bytes__(self): + return b'\x01' + class InvalidBytes: + def __bytes__(self): + return 'abc' + class MissingBytes: ... + class RaisingBytes: + def __bytes__(self): + 1 / 0 + + self.assertEqual(int.from_bytes(ValidBytes()), 1) + self.assertRaises(TypeError, int.from_bytes, InvalidBytes()) + self.assertRaises(TypeError, int.from_bytes, MissingBytes()) + self.assertRaises(ZeroDivisionError, int.from_bytes, RaisingBytes()) + + # gh-98783 + class SubStr(str): + pass + self.assertEqual(int.from_bytes(b'', SubStr('big')), 0) + self.assertEqual(int.from_bytes(b'\x00', SubStr('little')), 0) + @support.cpython_only def test_from_bytes_small(self): # bpo-46361 diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index aae86cc257d7e1..a1a91f661ba253 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -259,6 +259,8 @@ def test_recursion_limit(self): #if os.name == 'nt' and hasattr(sys, 'gettotalrefcount'): if os.name == 'nt': MAX_MARSHAL_STACK_DEPTH = 1000 + elif sys.platform == 'wasi': + MAX_MARSHAL_STACK_DEPTH = 1500 else: MAX_MARSHAL_STACK_DEPTH = 2000 for i in range(MAX_MARSHAL_STACK_DEPTH - 2): @@ -352,7 +354,7 @@ def test_deterministic_sets(self): for elements in ( "float('nan'), b'a', b'b', b'c', 'x', 'y', 'z'", # Also test for bad interactions with backreferencing: - "('Spam', 0), ('Spam', 1), ('Spam', 2)", + "('Spam', 0), ('Spam', 1), ('Spam', 2), ('Spam', 3), ('Spam', 4), ('Spam', 5)", ): s = f"{kind}([{elements}])" with self.subTest(s): diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index cfaf3b3ea26a7e..bf0d0a56e6ac8b 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -1006,6 +1006,11 @@ class T(tuple): self.assertEqual(math.dist(p, q), 5*scale) self.assertEqual(math.dist(q, p), 5*scale) + def test_math_dist_leak(self): + # gh-98897: Check for error handling does not leak memory + with self.assertRaises(ValueError): + math.dist([1, 2], [3, 4, 5]) + def testIsqrt(self): # Test a variety of inputs, large and small. test_values = ( diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 5c032d59b13f16..e39b7260624899 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -177,6 +177,73 @@ def f(): for _ in range(1025): self.assertFalse(f()) + def test_load_shadowing_slot_should_raise_type_error(self): + class Class: + __slots__ = ("slot",) + + class Sneaky: + __slots__ = ("shadowed",) + shadowing = Class.slot + + def f(o): + o.shadowing + + o = Sneaky() + o.shadowed = 42 + + for _ in range(1025): + with self.assertRaises(TypeError): + f(o) + + def test_store_shadowing_slot_should_raise_type_error(self): + class Class: + __slots__ = ("slot",) + + class Sneaky: + __slots__ = ("shadowed",) + shadowing = Class.slot + + def f(o): + o.shadowing = 42 + + o = Sneaky() + + for _ in range(1025): + with self.assertRaises(TypeError): + f(o) + + def test_load_borrowed_slot_should_not_crash(self): + class Class: + __slots__ = ("slot",) + + class Sneaky: + borrowed = Class.slot + + def f(o): + o.borrowed + + o = Sneaky() + + for _ in range(1025): + with self.assertRaises(TypeError): + f(o) + + def test_store_borrowed_slot_should_not_crash(self): + class Class: + __slots__ = ("slot",) + + class Sneaky: + borrowed = Class.slot + + def f(o): + o.borrowed = 42 + + o = Sneaky() + + for _ in range(1025): + with self.assertRaises(TypeError): + f(o) + class TestLoadMethodCache(unittest.TestCase): def test_descriptor_added_after_optimization(self): diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 55c3283e26b1e9..48f419e62fbbed 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2104,6 +2104,52 @@ def inner(v): pass stdout, stderr = self.run_pdb_script(script, commands) self.assertFalse(stderr) + def test_gh_93696_frozen_list(self): + frozen_src = """ + def func(): + x = "Sentinel string for gh-93696" + print(x) + """ + host_program = """ + import os + import sys + + def _create_fake_frozen_module(): + with open('gh93696.py') as f: + src = f.read() + + # this function has a co_filename as if it were in a frozen module + dummy_mod = compile(src, "", "exec") + func_code = dummy_mod.co_consts[0] + + mod = type(sys)("gh93696") + mod.func = type(lambda: None)(func_code, mod.__dict__) + mod.__file__ = 'gh93696.py' + + return mod + + mod = _create_fake_frozen_module() + mod.func() + """ + commands = """ + break 20 + continue + step + list + quit + """ + with open('gh93696.py', 'w') as f: + f.write(textwrap.dedent(frozen_src)) + + with open('gh93696_host.py', 'w') as f: + f.write(textwrap.dedent(host_program)) + + self.addCleanup(os_helper.unlink, 'gh93696.py') + self.addCleanup(os_helper.unlink, 'gh93696_host.py') + stdout, stderr = self._run_pdb(["gh93696_host.py"], commands) + # verify that pdb found the source of the "frozen" function + self.assertIn('x = "Sentinel string for gh-93696"', stdout, "Sentinel statement not found") + class ChecklineTests(unittest.TestCase): def setUp(self): linecache.clearcache() # Pdb.checkline() uses linecache.getline() diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 9b2cd201f3c2fe..c9f27575b51575 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -269,6 +269,14 @@ def test_uname_slices(self): self.assertEqual(res[:], expected) self.assertEqual(res[:5], expected[:5]) + def test_uname_fields(self): + self.assertIn('processor', platform.uname()._fields) + + def test_uname_asdict(self): + res = platform.uname()._asdict() + self.assertEqual(len(res), 6) + self.assertIn('processor', res) + @unittest.skipIf(sys.platform in ['win32', 'OpenVMS'], "uname -p not used") @support.requires_subprocess() def test_uname_processor(self): diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index c644f881e460fe..8a1dd131928cff 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -178,6 +178,8 @@ def test_islink(self): def test_ismount(self): self.assertIs(posixpath.ismount("/"), True) self.assertIs(posixpath.ismount(b"/"), True) + self.assertIs(posixpath.ismount(FakePath("/")), True) + self.assertIs(posixpath.ismount(FakePath(b"/")), True) def test_ismount_non_existent(self): # Non-existent mountpoint. diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 5d946370ee177c..59d0b7b5c48963 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -661,6 +661,11 @@ def test_re_groupref_exists_errors(self): self.checkPatternError(r'()(?(2)a)', "invalid group reference 2", 5) + def test_re_groupref_exists_validation_bug(self): + for i in range(256): + with self.subTest(code=i): + re.compile(r'()(?(1)\x%02x?)' % i) + def test_re_groupref_overflow(self): from re._constants import MAXGROUPS self.checkTemplateError('()', r'\g<%s>' % MAXGROUPS, 'xx', diff --git a/Lib/test/test_sched.py b/Lib/test/test_sched.py index 32cc8105bc0557..eb52ac7983f6c8 100644 --- a/Lib/test/test_sched.py +++ b/Lib/test/test_sched.py @@ -92,10 +92,23 @@ def test_priority(self): l = [] fun = lambda x: l.append(x) scheduler = sched.scheduler(time.time, time.sleep) - for priority in [1, 2, 3, 4, 5]: - z = scheduler.enterabs(0.01, priority, fun, (priority,)) - scheduler.run() - self.assertEqual(l, [1, 2, 3, 4, 5]) + + cases = [ + ([1, 2, 3, 4, 5], [1, 2, 3, 4, 5]), + ([5, 4, 3, 2, 1], [1, 2, 3, 4, 5]), + ([2, 5, 3, 1, 4], [1, 2, 3, 4, 5]), + ([1, 2, 3, 2, 1], [1, 1, 2, 2, 3]), + ] + for priorities, expected in cases: + with self.subTest(priorities=priorities, expected=expected): + for priority in priorities: + scheduler.enterabs(0.01, priority, fun, (priority,)) + scheduler.run() + self.assertEqual(l, expected) + + # Cleanup: + self.assertTrue(scheduler.empty()) + l.clear() def test_cancel(self): l = [] diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index a2c4ab508195b3..055c9af2cc5092 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -752,18 +752,25 @@ def _copy(src, dst): @os_helper.skip_unless_symlink def test_copytree_dangling_symlinks(self): - # a dangling symlink raises an error at the end src_dir = self.mkdtemp() + valid_file = os.path.join(src_dir, 'test.txt') + write_file(valid_file, 'abc') + dir_a = os.path.join(src_dir, 'dir_a') + os.mkdir(dir_a) + for d in src_dir, dir_a: + os.symlink('IDONTEXIST', os.path.join(d, 'broken')) + os.symlink(valid_file, os.path.join(d, 'valid')) + + # A dangling symlink should raise an error. dst_dir = os.path.join(self.mkdtemp(), 'destination') - os.symlink('IDONTEXIST', os.path.join(src_dir, 'test.txt')) - os.mkdir(os.path.join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') self.assertRaises(Error, shutil.copytree, src_dir, dst_dir) - # a dangling symlink is ignored with the proper flag + # Dangling symlinks should be ignored with the proper flag. dst_dir = os.path.join(self.mkdtemp(), 'destination2') shutil.copytree(src_dir, dst_dir, ignore_dangling_symlinks=True) - self.assertNotIn('test.txt', os.listdir(dst_dir)) + for root, dirs, files in os.walk(dst_dir): + self.assertNotIn('broken', files) + self.assertIn('valid', files) # a dangling symlink is copied if symlinks=True dst_dir = os.path.join(self.mkdtemp(), 'destination3') diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index e357264eb1d165..5fe0f3124444ba 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -161,6 +161,18 @@ def test_file_parse_error_multiline(self): finally: os.unlink(TESTFN) + def test_tokenizer_fstring_warning_in_first_line(self): + source = "0b1and 2" + with open(TESTFN, "w") as fd: + fd.write("{}".format(source)) + try: + retcode, stdout, stderr = script_helper.assert_python_ok(TESTFN) + self.assertIn(b"SyntaxWarning: invalid binary litera", stderr) + self.assertEqual(stderr.count(source.encode()), 1) + finally: + os.unlink(TESTFN) + + class AbstractSourceEncodingTest: def test_default_coding(self): diff --git a/Lib/test/test_sqlite3/test_regression.py b/Lib/test/test_sqlite3/test_regression.py index 0b727cecb0e8cb..9a07e02ae1197e 100644 --- a/Lib/test/test_sqlite3/test_regression.py +++ b/Lib/test/test_sqlite3/test_regression.py @@ -469,6 +469,18 @@ def test_executescript_step_through_select(self): con.executescript("select step(t) from t") self.assertEqual(steps, values) + def test_custom_cursor_object_crash_gh_99886(self): + # This test segfaults on GH-99886 + class MyCursor(sqlite.Cursor): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # this can go before or after the super call; doesn't matter + self.some_attr = None + + with memory_database() as con: + cur = con.cursor(MyCursor) + cur.close() + del cur class RecursiveUseOfCursors(unittest.TestCase): # GH-80254: sqlite3 should not segfault for recursive use of cursors. diff --git a/Lib/test/test_sqlite3/test_types.py b/Lib/test/test_sqlite3/test_types.py index 177cd102350397..ed225486300d0e 100644 --- a/Lib/test/test_sqlite3/test_types.py +++ b/Lib/test/test_sqlite3/test_types.py @@ -106,9 +106,9 @@ def test_string_with_surrogates(self): @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') @support.bigmemtest(size=2**31, memuse=4, dry_run=False) def test_too_large_string(self, maxsize): - with self.assertRaises(sqlite.InterfaceError): + with self.assertRaises(sqlite.DataError): self.cur.execute("insert into test(s) values (?)", ('x'*(2**31-1),)) - with self.assertRaises(OverflowError): + with self.assertRaises(sqlite.DataError): self.cur.execute("insert into test(s) values (?)", ('x'*(2**31),)) self.cur.execute("select 1 from test") row = self.cur.fetchone() @@ -117,9 +117,9 @@ def test_too_large_string(self, maxsize): @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') @support.bigmemtest(size=2**31, memuse=3, dry_run=False) def test_too_large_blob(self, maxsize): - with self.assertRaises(sqlite.InterfaceError): + with self.assertRaises(sqlite.DataError): self.cur.execute("insert into test(s) values (?)", (b'x'*(2**31-1),)) - with self.assertRaises(OverflowError): + with self.assertRaises(sqlite.DataError): self.cur.execute("insert into test(s) values (?)", (b'x'*(2**31),)) self.cur.execute("select 1 from test") row = self.cur.fetchone() diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 6de98241c294d7..3e172e974e125d 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2986,14 +2986,19 @@ def __init__(self, mu, sigma): nd = NormalDist(100, 15) self.assertNotEqual(nd, lnd) - def test_pickle_and_copy(self): + def test_copy(self): nd = self.module.NormalDist(37.5, 5.625) nd1 = copy.copy(nd) self.assertEqual(nd, nd1) nd2 = copy.deepcopy(nd) self.assertEqual(nd, nd2) - nd3 = pickle.loads(pickle.dumps(nd)) - self.assertEqual(nd, nd3) + + def test_pickle(self): + nd = self.module.NormalDist(37.5, 5.625) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + pickled = pickle.loads(pickle.dumps(nd, protocol=proto)) + self.assertEqual(nd, pickled) def test_hashability(self): ND = self.module.NormalDist diff --git a/Lib/test/test_string_literals.py b/Lib/test/test_string_literals.py index 3a3830bcb6e96f..7247b7e48bc2b6 100644 --- a/Lib/test/test_string_literals.py +++ b/Lib/test/test_string_literals.py @@ -266,6 +266,13 @@ def test_eval_str_u(self): self.assertRaises(SyntaxError, eval, """ bu'' """) self.assertRaises(SyntaxError, eval, """ ub'' """) + def test_uppercase_prefixes(self): + self.assertEqual(eval(""" B'x' """), b'x') + self.assertEqual(eval(r""" R'\x01' """), r'\x01') + self.assertEqual(eval(r""" BR'\x01' """), br'\x01') + self.assertEqual(eval(""" F'{1+1}' """), f'{1+1}') + self.assertEqual(eval(r""" U'\U0001d120' """), u'\U0001d120') + def check_encoding(self, encoding, extra=""): modname = "xx_" + encoding.replace("-", "_") fn = os.path.join(self.tmpdir, modname + ".py") diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index f6854922a5b878..abd0dd8b25699b 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -238,6 +238,12 @@ def test_check_output_input_none_universal_newlines(self): input=None, universal_newlines=True) self.assertNotIn('XX', output) + def test_check_output_input_none_encoding_errors(self): + output = subprocess.check_output( + [sys.executable, "-c", "print('foo')"], + input=None, encoding='utf-8', errors='ignore') + self.assertIn('foo', output) + def test_check_output_stdout_arg(self): # check_output() refuses to accept 'stdout' argument with self.assertRaises(ValueError) as c: @@ -2826,7 +2832,7 @@ def test_close_fds(self): @unittest.skipIf(sys.platform.startswith("freebsd") and os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, - "Requires fdescfs mounted on /dev/fd on FreeBSD.") + "Requires fdescfs mounted on /dev/fd on FreeBSD") def test_close_fds_when_max_fd_is_lowered(self): """Confirm that issue21618 is fixed (may fail under valgrind).""" fd_status = support.findfile("fd_status.py", subdir="subprocessdata") @@ -3203,7 +3209,7 @@ def __int__(self): 1, 2, 3, 4, True, True, 0, None, None, None, -1, - None, "no vfork") + None, True) self.assertIn('fds_to_keep', str(c.exception)) finally: if not gc_enabled: diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index ae1066924b3cf5..400092ee2c896f 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1985,6 +1985,16 @@ def test_generator_in_function_call(self): "Generator expression must be parenthesized", lineno=1, end_lineno=1, offset=11, end_offset=53) + def test_except_then_except_star(self): + self._check_error("try: pass\nexcept ValueError: pass\nexcept* TypeError: pass", + r"cannot have both 'except' and 'except\*' on the same 'try'", + lineno=3, end_lineno=3, offset=1, end_offset=8) + + def test_except_star_then_except(self): + self._check_error("try: pass\nexcept* ValueError: pass\nexcept TypeError: pass", + r"cannot have both 'except' and 'except\*' on the same 'try'", + lineno=3, end_lineno=3, offset=1, end_offset=7) + def test_empty_line_after_linecont(self): # See issue-40847 s = r"""\ diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 9f1aa81dbcd249..aa61f8b185883b 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1721,6 +1721,20 @@ def g(frame, event, arg): finally: sys.settrace(existing) + def test_line_event_raises_before_opcode_event(self): + exception = ValueError("BOOM!") + def trace(frame, event, arg): + if event == "line": + raise exception + frame.f_trace_opcodes = True + return trace + def f(): + pass + with self.assertRaises(ValueError) as caught: + sys.settrace(trace) + f() + self.assertIs(caught.exception, exception) + # 'Jump' tests: assigning to frame.f_lineno within a trace function # moves the execution position - it's how debuggers implement a Jump diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 578ac1db504455..d96371d24269f7 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -439,6 +439,7 @@ def test_platform_in_subprocess(self): self.assertEqual(status, 0) self.assertEqual(my_platform, test_platform) + @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") def test_srcdir(self): # See Issues #15322, #15364. srcdir = sysconfig.get_config_var('srcdir') diff --git a/Lib/test/test_syslog.py b/Lib/test/test_syslog.py index fe09bd39f8b7fd..2125ec58d87e03 100644 --- a/Lib/test/test_syslog.py +++ b/Lib/test/test_syslog.py @@ -1,5 +1,9 @@ -from test.support import import_helper +from test.support import import_helper, threading_helper syslog = import_helper.import_module("syslog") #skip if not supported +from test import support +import sys +import threading +import time import unittest # XXX(nnorwitz): This test sucks. I don't know of a platform independent way @@ -8,6 +12,9 @@ class Test(unittest.TestCase): + def tearDown(self): + syslog.closelog() + def test_openlog(self): syslog.openlog('python') # Issue #6697. @@ -18,22 +25,59 @@ def test_syslog(self): syslog.syslog('test message from python test_syslog') syslog.syslog(syslog.LOG_ERR, 'test error from python test_syslog') + def test_syslog_implicit_open(self): + syslog.closelog() # Make sure log is closed + syslog.syslog('test message from python test_syslog') + syslog.syslog(syslog.LOG_ERR, 'test error from python test_syslog') + def test_closelog(self): syslog.openlog('python') syslog.closelog() + syslog.closelog() # idempotent operation def test_setlogmask(self): - syslog.setlogmask(syslog.LOG_DEBUG) + mask = syslog.LOG_UPTO(syslog.LOG_WARNING) + oldmask = syslog.setlogmask(mask) + self.assertEqual(syslog.setlogmask(0), mask) + self.assertEqual(syslog.setlogmask(oldmask), mask) def test_log_mask(self): - syslog.LOG_MASK(syslog.LOG_INFO) - - def test_log_upto(self): - syslog.LOG_UPTO(syslog.LOG_INFO) + mask = syslog.LOG_UPTO(syslog.LOG_WARNING) + self.assertTrue(mask & syslog.LOG_MASK(syslog.LOG_WARNING)) + self.assertTrue(mask & syslog.LOG_MASK(syslog.LOG_ERR)) + self.assertFalse(mask & syslog.LOG_MASK(syslog.LOG_INFO)) def test_openlog_noargs(self): syslog.openlog() syslog.syslog('test message from python test_syslog') + @threading_helper.requires_working_threading() + def test_syslog_threaded(self): + start = threading.Event() + stop = False + def opener(): + start.wait(10) + i = 1 + while not stop: + syslog.openlog(f'python-test-{i}') # new string object + i += 1 + def logger(): + start.wait(10) + while not stop: + syslog.syslog('test message from python test_syslog') + + orig_si = sys.getswitchinterval() + support.setswitchinterval(1e-9) + try: + threads = [threading.Thread(target=opener)] + threads += [threading.Thread(target=logger) for k in range(10)] + with threading_helper.start_threads(threads): + start.set() + time.sleep(0.1) + stop = True + finally: + sys.setswitchinterval(orig_si) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index e0389c5dc474f5..cdfd426807bc36 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -734,6 +734,18 @@ def test_zlib_error_does_not_leak(self): with self.assertRaises(tarfile.ReadError): tarfile.open(self.tarname) + def test_next_on_empty_tarfile(self): + fd = io.BytesIO() + tf = tarfile.open(fileobj=fd, mode="w") + tf.close() + + fd.seek(0) + with tarfile.open(fileobj=fd, mode="r|") as tf: + self.assertEqual(tf.next(), None) + + fd.seek(0) + with tarfile.open(fileobj=fd, mode="r") as tf: + self.assertEqual(tf.next(), None) class MiscReadTest(MiscReadTestBase, unittest.TestCase): test_fail_comp = None diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 1272e1e9be002e..63c2501cfe2338 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3,13 +3,15 @@ from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP, STRING, ENDMARKER, ENCODING, tok_name, detect_encoding, open as tokenize_open, Untokenizer, generate_tokens, - NEWLINE, _generate_tokens_from_c_tokenizer) + NEWLINE, _generate_tokens_from_c_tokenizer, DEDENT) from io import BytesIO, StringIO import unittest from textwrap import dedent from unittest import TestCase, mock from test.test_grammar import (VALID_UNDERSCORE_LITERALS, INVALID_UNDERSCORE_LITERALS) +from test.support import os_helper +from test.support.script_helper import run_test_script, make_script import os import token @@ -2512,6 +2514,26 @@ def get_tokens(string): self.assertRaises(SyntaxError, get_tokens, "("*1000+"a"+")"*1000) self.assertRaises(SyntaxError, get_tokens, "]") + def test_max_indent(self): + MAXINDENT = 100 + + def generate_source(indents): + source = ''.join((' ' * x) + 'if True:\n' for x in range(indents)) + source += ' ' * indents + 'pass\n' + return source + + valid = generate_source(MAXINDENT - 1) + tokens = list(_generate_tokens_from_c_tokenizer(valid)) + self.assertEqual(tokens[-1].type, DEDENT) + compile(valid, "", "exec") + + invalid = generate_source(MAXINDENT) + tokens = list(_generate_tokens_from_c_tokenizer(invalid)) + self.assertEqual(tokens[-1].type, NEWLINE) + self.assertRaises( + IndentationError, compile, invalid, "", "exec" + ) + def test_continuation_lines_indentation(self): def get_tokens(string): return [(kind, string) for (kind, string, *_) in _generate_tokens_from_c_tokenizer(string)] @@ -2611,5 +2633,19 @@ def fib(n): self.assertEqual(get_tokens(code), get_tokens(code_no_cont)) +class CTokenizerBufferTests(unittest.TestCase): + def test_newline_at_the_end_of_buffer(self): + # See issue 99581: Make sure that if we need to add a new line at the + # end of the buffer, we have enough space in the buffer, specially when + # the current line is as long as the buffer space available. + test_script = f"""\ + #coding: latin-1 + #{"a"*10000} + #{"a"*10002}""" + with os_helper.temp_dir() as temp_dir: + file_name = make_script(temp_dir, 'foo', test_script) + run_test_script(file_name) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index 7f18edaaa8ca60..c083a04475e726 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -155,6 +155,26 @@ class C: ''')) self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + def test_moduledocstring(self): + for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'): + with self.subTest(doc): + msgids = self.extract_docstrings_from_str(dedent('''\ + %s + ''' % doc)) + self.assertIn('doc', msgids) + + def test_moduledocstring_bytes(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + b"""doc""" + ''')) + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_moduledocstring_fstring(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"""doc""" + ''')) + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + def test_msgid(self): msgids = self.extract_docstrings_from_str( '''_("""doc""" r'str' u"ing")''') diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 5f712111ca14e0..fad2b3b8379ffc 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -1,4 +1,5 @@ import os +from pickle import dump import sys from test.support import captured_stdout from test.support.os_helper import (TESTFN, rmtree, unlink) @@ -412,6 +413,15 @@ def test_issue9936(self): self.assertIn(modname, coverage) self.assertEqual(coverage[modname], (5, 100)) + def test_coverageresults_update(self): + # Update empty CoverageResults with a non-empty infile. + infile = TESTFN + '-infile' + with open(infile, 'wb') as f: + dump(({}, {}, {'caller': 1}), f, protocol=1) + self.addCleanup(unlink, infile) + results = trace.CoverageResults({}, {}, infile, {}) + self.assertEqual(results.callers, {'caller': 1}) + ### Tests that don't mess with sys.settrace and can be traced ### themselves TODO: Skip tests that do mess with sys.settrace when ### regrtest is invoked with -T option. diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 94ccc3f48d3ea2..eadc9c440e32cc 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -532,6 +532,23 @@ def f_with_binary_operator(): result_lines = self.get_exception(f_with_binary_operator) self.assertEqual(result_lines, expected_error.splitlines()) + def test_caret_for_binary_operators_with_unicode(self): + def f_with_binary_operator(): + áóí = 20 + return 10 + áóí / 0 + 30 + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' + ' return 10 + áóí / 0 + 30\n' + ' ~~~~^~~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + def test_caret_for_binary_operators_two_char(self): def f_with_binary_operator(): divisor = 20 @@ -566,6 +583,23 @@ def f_with_subscript(): result_lines = self.get_exception(f_with_subscript) self.assertEqual(result_lines, expected_error.splitlines()) + def test_caret_for_subscript_unicode(self): + def f_with_subscript(): + some_dict = {'ó': {'á': {'í': {'theta': 1}}}} + return some_dict['ó']['á']['í']['beta'] + + lineno_f = f_with_subscript.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_subscript\n' + " return some_dict['ó']['á']['í']['beta']\n" + ' ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^\n' + ) + result_lines = self.get_exception(f_with_subscript) + self.assertEqual(result_lines, expected_error.splitlines()) + def test_traceback_specialization_with_syntax_error(self): bytecode = compile("1 / 0 / 1 / 2\n", TESTFN, "exec") @@ -778,6 +812,56 @@ def f(): ] self.assertEqual(actual, expected) + def test_wide_characters_unicode_with_problematic_byte_offset(self): + def f(): + ｗｉｄｔｈ + + actual = self.get_exception(f) + expected = [ + f"Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + f" callable()", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + f" ｗｉｄｔｈ", + ] + self.assertEqual(actual, expected) + + + def test_byte_offset_with_wide_characters_middle(self): + def f(): + ｗｉｄｔｈ = 1 + raise ValueError(ｗｉｄｔｈ) + + actual = self.get_exception(f) + expected = [ + f"Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + f" callable()", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", + f" raise ValueError(ｗｉｄｔｈ)", + ] + self.assertEqual(actual, expected) + + def test_byte_offset_multiline(self): + def f(): + ｗｗｗ = 1 + ｔｈ = 0 + + print(1, ｗｗｗ( + ｔｈ)) + + actual = self.get_exception(f) + expected = [ + f"Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + f" callable()", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 4}, in f", + f" print(1, ｗｗｗ(", + f" ^^^^", + ] + self.assertEqual(actual, expected) + + @cpython_only @requires_debug_ranges() class CPythonTracebackErrorCaretTests(TracebackErrorLocationCaretTests): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 776a6f003c0691..efa4cb3d4e2a97 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -42,7 +42,7 @@ import weakref import types -from test.support import import_helper, captured_stderr +from test.support import import_helper, captured_stderr, cpython_only from test import mod_generics_cache from test import _typed_dict_helper @@ -50,6 +50,8 @@ py_typing = import_helper.import_fresh_module('typing', blocked=['_typing']) c_typing = import_helper.import_fresh_module('typing', fresh=['_typing']) +CANNOT_SUBCLASS_TYPE = 'Cannot subclass special typing classes' + class BaseTestCase(TestCase): @@ -109,6 +111,12 @@ def test_any_instance_type_error(self): def test_repr(self): self.assertEqual(repr(Any), 'typing.Any') + class Sub(Any): pass + self.assertEqual( + repr(Sub), + ".Sub'>", + ) + def test_errors(self): with self.assertRaises(TypeError): issubclass(42, Any) @@ -581,10 +589,9 @@ def test_no_duplicates_if_replacement_not_in_templates(self): class GenericAliasSubstitutionTests(BaseTestCase): """Tests for type variable substitution in generic aliases. - Note that the expected results here are tentative, based on a - still-being-worked-out spec for what we allow at runtime (given that - implementation of *full* substitution logic at runtime would add too much - complexity to typing.py). This spec is currently being discussed at + For variadic cases, these tests should be regarded as the source of truth, + since we hadn't realised the full complexity of variadic substitution + at the time of finalizing PEP 646. For full discussion, see https://github.com/python/cpython/issues/91162. """ @@ -671,9 +678,6 @@ class C(Generic[T1, T2]): pass ('generic[T1, T2]', '[tuple_type[int, ...]]', 'TypeError'), ('generic[T1, T2]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'generic[tuple_type[int, ...], tuple_type[str, ...]]'), - # Should raise TypeError according to the tentative spec: unpacked - # types cannot be used as arguments to aliases that expect a fixed - # number of arguments. ('generic[T1, T2]', '[*tuple_type[int, ...]]', 'TypeError'), ('generic[T1, T2]', '[int, *tuple_type[str, ...]]', 'TypeError'), ('generic[T1, T2]', '[*tuple_type[int, ...], str]', 'TypeError'), @@ -681,10 +685,12 @@ class C(Generic[T1, T2]): pass ('generic[T1, T2]', '[*Ts]', 'TypeError'), ('generic[T1, T2]', '[T, *Ts]', 'TypeError'), ('generic[T1, T2]', '[*Ts, T]', 'TypeError'), - # Should raise TypeError according to the tentative spec: unpacked - # types cannot be used as arguments to generics that expect a fixed - # number of arguments. - # (None of the things in `generics` were defined using *Ts.) + # This one isn't technically valid - none of the things that + # `generic` can be (defined in `generics` above) are variadic, so we + # shouldn't really be able to do `generic[T1, *tuple_type[int, ...]]`. + # So even if type checkers shouldn't allow it, we allow it at + # runtime, in accordance with a general philosophy of "Keep the + # runtime lenient so people can experiment with typing constructs". ('generic[T1, *tuple_type[int, ...]]', '[str]', 'generic[str, *tuple_type[int, ...]]'), ] @@ -746,8 +752,6 @@ class C(Generic[*Ts]): pass generics = ['C', 'tuple', 'Tuple'] tuple_types = ['tuple', 'Tuple'] - # The majority of these have three separate cases for C, tuple and - # Tuple because tuple currently behaves differently. tests = [ # Alias # Args # Expected result ('generic[*Ts]', '[()]', 'generic[()]'), @@ -765,22 +769,48 @@ class C(Generic[*Ts]): pass ('generic[*Ts]', '[*Ts]', 'generic[*Ts]'), ('generic[*Ts]', '[T, *Ts]', 'generic[T, *Ts]'), ('generic[*Ts]', '[*Ts, T]', 'generic[*Ts, T]'), + ('generic[T, *Ts]', '[()]', 'TypeError'), ('generic[T, *Ts]', '[int]', 'generic[int]'), ('generic[T, *Ts]', '[int, str]', 'generic[int, str]'), ('generic[T, *Ts]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[list[T], *Ts]', '[()]', 'TypeError'), ('generic[list[T], *Ts]', '[int]', 'generic[list[int]]'), ('generic[list[T], *Ts]', '[int, str]', 'generic[list[int], str]'), ('generic[list[T], *Ts]', '[int, str, bool]', 'generic[list[int], str, bool]'), + ('generic[*Ts, T]', '[()]', 'TypeError'), ('generic[*Ts, T]', '[int]', 'generic[int]'), ('generic[*Ts, T]', '[int, str]', 'generic[int, str]'), ('generic[*Ts, T]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[*Ts, list[T]]', '[()]', 'TypeError'), ('generic[*Ts, list[T]]', '[int]', 'generic[list[int]]'), ('generic[*Ts, list[T]]', '[int, str]', 'generic[int, list[str]]'), ('generic[*Ts, list[T]]', '[int, str, bool]', 'generic[int, str, list[bool]]'), + ('generic[T1, T2, *Ts]', '[()]', 'TypeError'), + ('generic[T1, T2, *Ts]', '[int]', 'TypeError'), + ('generic[T1, T2, *Ts]', '[int, str]', 'generic[int, str]'), + ('generic[T1, T2, *Ts]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[T1, T2, *Ts]', '[int, str, bool, bytes]', 'generic[int, str, bool, bytes]'), + + ('generic[*Ts, T1, T2]', '[()]', 'TypeError'), + ('generic[*Ts, T1, T2]', '[int]', 'TypeError'), + ('generic[*Ts, T1, T2]', '[int, str]', 'generic[int, str]'), + ('generic[*Ts, T1, T2]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[*Ts, T1, T2]', '[int, str, bool, bytes]', 'generic[int, str, bool, bytes]'), + + ('generic[T1, *Ts, T2]', '[()]', 'TypeError'), + ('generic[T1, *Ts, T2]', '[int]', 'TypeError'), + ('generic[T1, *Ts, T2]', '[int, str]', 'generic[int, str]'), + ('generic[T1, *Ts, T2]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[T1, *Ts, T2]', '[int, str, bool, bytes]', 'generic[int, str, bool, bytes]'), + ('generic[T, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...]]'), + ('generic[T, *Ts]', '[str, *tuple_type[int, ...]]', 'generic[str, *tuple_type[int, ...]]'), + ('generic[T, *Ts]', '[*tuple_type[int, ...], str]', 'generic[int, *tuple_type[int, ...], str]'), ('generic[*Ts, T]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], int]'), + ('generic[*Ts, T]', '[str, *tuple_type[int, ...]]', 'generic[str, *tuple_type[int, ...], int]'), + ('generic[*Ts, T]', '[*tuple_type[int, ...], str]', 'generic[*tuple_type[int, ...], str]'), ('generic[T1, *Ts, T2]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...], int]'), ('generic[T, str, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, str, *tuple_type[int, ...]]'), ('generic[*Ts, str, T]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], str, int]'), @@ -819,13 +849,19 @@ class C(Generic[*Ts]): pass class UnpackTests(BaseTestCase): def test_accepts_single_type(self): + (*tuple[int],) Unpack[Tuple[int]] def test_rejects_multiple_types(self): with self.assertRaises(TypeError): Unpack[Tuple[int], Tuple[str]] + # We can't do the equivalent for `*` here - + # *(Tuple[int], Tuple[str]) is just plain tuple unpacking, + # which is valid. def test_rejects_multiple_parameterization(self): + with self.assertRaises(TypeError): + (*tuple[int],)[0][tuple[int]] with self.assertRaises(TypeError): Unpack[Tuple[int]][Tuple[int]] @@ -864,19 +900,20 @@ def test_cannot_call_instance(self): def test_unpacked_typevartuple_is_equal_to_itself(self): Ts = TypeVarTuple('Ts') + self.assertEqual((*Ts,)[0], (*Ts,)[0]) self.assertEqual(Unpack[Ts], Unpack[Ts]) def test_parameterised_tuple_is_equal_to_itself(self): Ts = TypeVarTuple('Ts') - self.assertEqual(tuple[Unpack[Ts]], tuple[Unpack[Ts]]) + self.assertEqual(tuple[*Ts], tuple[*Ts]) self.assertEqual(Tuple[Unpack[Ts]], Tuple[Unpack[Ts]]) def tests_tuple_arg_ordering_matters(self): Ts1 = TypeVarTuple('Ts1') Ts2 = TypeVarTuple('Ts2') self.assertNotEqual( - tuple[Unpack[Ts1], Unpack[Ts2]], - tuple[Unpack[Ts2], Unpack[Ts1]], + tuple[*Ts1, *Ts2], + tuple[*Ts2, *Ts1], ) self.assertNotEqual( Tuple[Unpack[Ts1], Unpack[Ts2]], @@ -885,8 +922,8 @@ def tests_tuple_arg_ordering_matters(self): def test_tuple_args_and_parameters_are_correct(self): Ts = TypeVarTuple('Ts') - t1 = tuple[Unpack[Ts]] - self.assertEqual(t1.__args__, (Unpack[Ts],)) + t1 = tuple[*Ts] + self.assertEqual(t1.__args__, (*Ts,)) self.assertEqual(t1.__parameters__, (Ts,)) t2 = Tuple[Unpack[Ts]] self.assertEqual(t2.__args__, (Unpack[Ts],)) @@ -896,128 +933,218 @@ def test_var_substitution(self): Ts = TypeVarTuple('Ts') T = TypeVar('T') T2 = TypeVar('T2') - class G(Generic[Unpack[Ts]]): pass + class G1(Generic[*Ts]): pass + class G2(Generic[Unpack[Ts]]): pass - for A in G, Tuple, tuple: - B = A[Unpack[Ts]] + for A in G1, G2, Tuple, tuple: + B = A[*Ts] self.assertEqual(B[()], A[()]) self.assertEqual(B[float], A[float]) self.assertEqual(B[float, str], A[float, str]) - C = List[A[Unpack[Ts]]] - self.assertEqual(C[()], List[A[()]]) - self.assertEqual(C[float], List[A[float]]) - self.assertEqual(C[float, str], List[A[float, str]]) + C = A[Unpack[Ts]] + self.assertEqual(C[()], A[()]) + self.assertEqual(C[float], A[float]) + self.assertEqual(C[float, str], A[float, str]) + + D = list[A[*Ts]] + self.assertEqual(D[()], list[A[()]]) + self.assertEqual(D[float], list[A[float]]) + self.assertEqual(D[float, str], list[A[float, str]]) - D = A[T, Unpack[Ts], T2] + E = List[A[Unpack[Ts]]] + self.assertEqual(E[()], List[A[()]]) + self.assertEqual(E[float], List[A[float]]) + self.assertEqual(E[float, str], List[A[float, str]]) + + F = A[T, *Ts, T2] with self.assertRaises(TypeError): - D[()] + F[()] with self.assertRaises(TypeError): - D[float] - self.assertEqual(D[float, str], A[float, str]) - self.assertEqual(D[float, str, int], A[float, str, int]) - self.assertEqual(D[float, str, int, bytes], A[float, str, int, bytes]) + F[float] + self.assertEqual(F[float, str], A[float, str]) + self.assertEqual(F[float, str, int], A[float, str, int]) + self.assertEqual(F[float, str, int, bytes], A[float, str, int, bytes]) - E = Tuple[List[T], A[Unpack[Ts]], List[T2]] + G = A[T, Unpack[Ts], T2] with self.assertRaises(TypeError): - E[()] + G[()] with self.assertRaises(TypeError): - E[float] + G[float] + self.assertEqual(G[float, str], A[float, str]) + self.assertEqual(G[float, str, int], A[float, str, int]) + self.assertEqual(G[float, str, int, bytes], A[float, str, int, bytes]) + + H = tuple[list[T], A[*Ts], list[T2]] + with self.assertRaises(TypeError): + H[()] + with self.assertRaises(TypeError): + H[float] + if A != Tuple: + self.assertEqual(H[float, str], + tuple[list[float], A[()], list[str]]) + self.assertEqual(H[float, str, int], + tuple[list[float], A[str], list[int]]) + self.assertEqual(H[float, str, int, bytes], + tuple[list[float], A[str, int], list[bytes]]) + + I = Tuple[List[T], A[Unpack[Ts]], List[T2]] + with self.assertRaises(TypeError): + I[()] + with self.assertRaises(TypeError): + I[float] if A != Tuple: - self.assertEqual(E[float, str], + self.assertEqual(I[float, str], Tuple[List[float], A[()], List[str]]) - self.assertEqual(E[float, str, int], + self.assertEqual(I[float, str, int], Tuple[List[float], A[str], List[int]]) - self.assertEqual(E[float, str, int, bytes], + self.assertEqual(I[float, str, int, bytes], Tuple[List[float], A[str, int], List[bytes]]) def test_bad_var_substitution(self): Ts = TypeVarTuple('Ts') T = TypeVar('T') T2 = TypeVar('T2') - class G(Generic[Unpack[Ts]]): pass + class G1(Generic[*Ts]): pass + class G2(Generic[Unpack[Ts]]): pass - for A in G, Tuple, tuple: + for A in G1, G2, Tuple, tuple: B = A[Ts] with self.assertRaises(TypeError): B[int, str] C = A[T, T2] + with self.assertRaises(TypeError): + C[*Ts] with self.assertRaises(TypeError): C[Unpack[Ts]] - def test_repr_is_correct(self): - Ts = TypeVarTuple('Ts') - T = TypeVar('T') - T2 = TypeVar('T2') - class G(Generic[Unpack[Ts]]): pass - - for A in G, Tuple: - B = A[T, Unpack[Ts], str, T2] + B = A[T, *Ts, str, T2] + with self.assertRaises(TypeError): + B[int, *Ts] with self.assertRaises(TypeError): - B[int, Unpack[Ts]] + B[int, *Ts, *Ts] + C = A[T, Unpack[Ts], str, T2] + with self.assertRaises(TypeError): + C[int, Unpack[Ts]] with self.assertRaises(TypeError): C[int, Unpack[Ts], Unpack[Ts]] def test_repr_is_correct(self): Ts = TypeVarTuple('Ts') + T = TypeVar('T') + T2 = TypeVar('T2') + + class G1(Generic[*Ts]): pass + class G2(Generic[Unpack[Ts]]): pass + self.assertEqual(repr(Ts), 'Ts') + + self.assertEqual(repr((*Ts,)[0]), '*Ts') self.assertEqual(repr(Unpack[Ts]), '*Ts') - self.assertEqual(repr(tuple[Unpack[Ts]]), 'tuple[*Ts]') + + self.assertEqual(repr(tuple[*Ts]), 'tuple[*Ts]') self.assertEqual(repr(Tuple[Unpack[Ts]]), 'typing.Tuple[*Ts]') - self.assertEqual(repr(Unpack[tuple[Unpack[Ts]]]), '*tuple[*Ts]') + + self.assertEqual(repr(*tuple[*Ts]), '*tuple[*Ts]') self.assertEqual(repr(Unpack[Tuple[Unpack[Ts]]]), '*typing.Tuple[*Ts]') def test_variadic_class_repr_is_correct(self): Ts = TypeVarTuple('Ts') - class A(Generic[Unpack[Ts]]): pass + class A(Generic[*Ts]): pass + class B(Generic[Unpack[Ts]]): pass self.assertEndsWith(repr(A[()]), 'A[()]') + self.assertEndsWith(repr(B[()]), 'B[()]') self.assertEndsWith(repr(A[float]), 'A[float]') + self.assertEndsWith(repr(B[float]), 'B[float]') self.assertEndsWith(repr(A[float, str]), 'A[float, str]') - self.assertEndsWith(repr(A[Unpack[tuple[int, ...]]]), + self.assertEndsWith(repr(B[float, str]), 'B[float, str]') + + self.assertEndsWith(repr(A[*tuple[int, ...]]), 'A[*tuple[int, ...]]') - self.assertEndsWith(repr(A[float, Unpack[tuple[int, ...]]]), + self.assertEndsWith(repr(B[Unpack[Tuple[int, ...]]]), + 'B[*typing.Tuple[int, ...]]') + + self.assertEndsWith(repr(A[float, *tuple[int, ...]]), 'A[float, *tuple[int, ...]]') - self.assertEndsWith(repr(A[Unpack[tuple[int, ...]], str]), + self.assertEndsWith(repr(A[float, Unpack[Tuple[int, ...]]]), + 'A[float, *typing.Tuple[int, ...]]') + + self.assertEndsWith(repr(A[*tuple[int, ...], str]), 'A[*tuple[int, ...], str]') - self.assertEndsWith(repr(A[float, Unpack[tuple[int, ...]], str]), + self.assertEndsWith(repr(B[Unpack[Tuple[int, ...]], str]), + 'B[*typing.Tuple[int, ...], str]') + + self.assertEndsWith(repr(A[float, *tuple[int, ...], str]), 'A[float, *tuple[int, ...], str]') + self.assertEndsWith(repr(B[float, Unpack[Tuple[int, ...]], str]), + 'B[float, *typing.Tuple[int, ...], str]') def test_variadic_class_alias_repr_is_correct(self): Ts = TypeVarTuple('Ts') class A(Generic[Unpack[Ts]]): pass - B = A[Unpack[Ts]] + B = A[*Ts] self.assertEndsWith(repr(B), 'A[*Ts]') self.assertEndsWith(repr(B[()]), 'A[()]') self.assertEndsWith(repr(B[float]), 'A[float]') self.assertEndsWith(repr(B[float, str]), 'A[float, str]') - C = A[Unpack[Ts], int] - self.assertEndsWith(repr(C), 'A[*Ts, int]') - self.assertEndsWith(repr(C[()]), 'A[int]') - self.assertEndsWith(repr(C[float]), 'A[float, int]') - self.assertEndsWith(repr(C[float, str]), 'A[float, str, int]') + C = A[Unpack[Ts]] + self.assertEndsWith(repr(C), 'A[*Ts]') + self.assertEndsWith(repr(C[()]), 'A[()]') + self.assertEndsWith(repr(C[float]), 'A[float]') + self.assertEndsWith(repr(C[float, str]), 'A[float, str]') - D = A[int, Unpack[Ts]] - self.assertEndsWith(repr(D), 'A[int, *Ts]') + D = A[*Ts, int] + self.assertEndsWith(repr(D), 'A[*Ts, int]') self.assertEndsWith(repr(D[()]), 'A[int]') - self.assertEndsWith(repr(D[float]), 'A[int, float]') - self.assertEndsWith(repr(D[float, str]), 'A[int, float, str]') - - E = A[int, Unpack[Ts], str] - self.assertEndsWith(repr(E), 'A[int, *Ts, str]') - self.assertEndsWith(repr(E[()]), 'A[int, str]') - self.assertEndsWith(repr(E[float]), 'A[int, float, str]') - self.assertEndsWith(repr(E[float, str]), 'A[int, float, str, str]') - - F = A[Unpack[Ts], Unpack[tuple[str, ...]]] - self.assertEndsWith(repr(F), 'A[*Ts, *tuple[str, ...]]') - self.assertEndsWith(repr(F[()]), 'A[*tuple[str, ...]]') - self.assertEndsWith(repr(F[float]), 'A[float, *tuple[str, ...]]') - self.assertEndsWith(repr(F[float, str]), 'A[float, str, *tuple[str, ...]]') + self.assertEndsWith(repr(D[float]), 'A[float, int]') + self.assertEndsWith(repr(D[float, str]), 'A[float, str, int]') + + E = A[Unpack[Ts], int] + self.assertEndsWith(repr(E), 'A[*Ts, int]') + self.assertEndsWith(repr(E[()]), 'A[int]') + self.assertEndsWith(repr(E[float]), 'A[float, int]') + self.assertEndsWith(repr(E[float, str]), 'A[float, str, int]') + + F = A[int, *Ts] + self.assertEndsWith(repr(F), 'A[int, *Ts]') + self.assertEndsWith(repr(F[()]), 'A[int]') + self.assertEndsWith(repr(F[float]), 'A[int, float]') + self.assertEndsWith(repr(F[float, str]), 'A[int, float, str]') + + G = A[int, Unpack[Ts]] + self.assertEndsWith(repr(G), 'A[int, *Ts]') + self.assertEndsWith(repr(G[()]), 'A[int]') + self.assertEndsWith(repr(G[float]), 'A[int, float]') + self.assertEndsWith(repr(G[float, str]), 'A[int, float, str]') + + H = A[int, *Ts, str] + self.assertEndsWith(repr(H), 'A[int, *Ts, str]') + self.assertEndsWith(repr(H[()]), 'A[int, str]') + self.assertEndsWith(repr(H[float]), 'A[int, float, str]') + self.assertEndsWith(repr(H[float, str]), 'A[int, float, str, str]') + + I = A[int, Unpack[Ts], str] + self.assertEndsWith(repr(I), 'A[int, *Ts, str]') + self.assertEndsWith(repr(I[()]), 'A[int, str]') + self.assertEndsWith(repr(I[float]), 'A[int, float, str]') + self.assertEndsWith(repr(I[float, str]), 'A[int, float, str, str]') + + J = A[*Ts, *tuple[str, ...]] + self.assertEndsWith(repr(J), 'A[*Ts, *tuple[str, ...]]') + self.assertEndsWith(repr(J[()]), 'A[*tuple[str, ...]]') + self.assertEndsWith(repr(J[float]), 'A[float, *tuple[str, ...]]') + self.assertEndsWith(repr(J[float, str]), 'A[float, str, *tuple[str, ...]]') + + K = A[Unpack[Ts], Unpack[Tuple[str, ...]]] + self.assertEndsWith(repr(K), 'A[*Ts, *typing.Tuple[str, ...]]') + self.assertEndsWith(repr(K[()]), 'A[*typing.Tuple[str, ...]]') + self.assertEndsWith(repr(K[float]), 'A[float, *typing.Tuple[str, ...]]') + self.assertEndsWith(repr(K[float, str]), 'A[float, str, *typing.Tuple[str, ...]]') def test_cannot_subclass_class(self): with self.assertRaises(TypeError): @@ -1027,30 +1154,69 @@ def test_cannot_subclass_instance(self): Ts = TypeVarTuple('Ts') with self.assertRaises(TypeError): class C(Ts): pass - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(Unpack)): pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(*Ts)): pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(Unpack[Ts])): pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.Unpack'): + class C(Unpack): pass + with self.assertRaisesRegex(TypeError, r'Cannot subclass \*Ts'): + class C(*Ts): pass + with self.assertRaisesRegex(TypeError, r'Cannot subclass \*Ts'): class C(Unpack[Ts]): pass def test_variadic_class_args_are_correct(self): T = TypeVar('T') Ts = TypeVarTuple('Ts') - class A(Generic[Unpack[Ts]]): pass - B = A[()] - self.assertEqual(B.__args__, ()) - C = A[int] - self.assertEqual(C.__args__, (int,)) - D = A[int, str] - self.assertEqual(D.__args__, (int, str)) - E = A[T] - self.assertEqual(E.__args__, (T,)) - F = A[Unpack[Ts]] - self.assertEqual(F.__args__, (Unpack[Ts],)) - G = A[T, Unpack[Ts]] - self.assertEqual(G.__args__, (T, Unpack[Ts])) - H = A[Unpack[Ts], T] - self.assertEqual(H.__args__, (Unpack[Ts], T)) + class A(Generic[*Ts]): pass + class B(Generic[Unpack[Ts]]): pass + + C = A[()] + D = B[()] + self.assertEqual(C.__args__, ()) + self.assertEqual(D.__args__, ()) + + E = A[int] + F = B[int] + self.assertEqual(E.__args__, (int,)) + self.assertEqual(F.__args__, (int,)) + + G = A[int, str] + H = B[int, str] + self.assertEqual(G.__args__, (int, str)) + self.assertEqual(H.__args__, (int, str)) + + I = A[T] + J = B[T] + self.assertEqual(I.__args__, (T,)) + self.assertEqual(J.__args__, (T,)) + + K = A[*Ts] + L = B[Unpack[Ts]] + self.assertEqual(K.__args__, (*Ts,)) + self.assertEqual(L.__args__, (Unpack[Ts],)) + + M = A[T, *Ts] + N = B[T, Unpack[Ts]] + self.assertEqual(M.__args__, (T, *Ts)) + self.assertEqual(N.__args__, (T, Unpack[Ts])) + + O = A[*Ts, T] + P = B[Unpack[Ts], T] + self.assertEqual(O.__args__, (*Ts, T)) + self.assertEqual(P.__args__, (Unpack[Ts], T)) def test_variadic_class_origin_is_correct(self): Ts = TypeVarTuple('Ts') + + class C(Generic[*Ts]): pass + self.assertIs(C[int].__origin__, C) + self.assertIs(C[T].__origin__, C) + self.assertIs(C[Unpack[Ts]].__origin__, C) + class D(Generic[Unpack[Ts]]): pass self.assertIs(D[int].__origin__, D) self.assertIs(D[T].__origin__, D) @@ -1059,21 +1225,21 @@ class D(Generic[Unpack[Ts]]): pass def test_tuple_args_are_correct(self): Ts = TypeVarTuple('Ts') - self.assertEqual(tuple[Unpack[Ts]].__args__, (Unpack[Ts],)) + self.assertEqual(tuple[*Ts].__args__, (*Ts,)) self.assertEqual(Tuple[Unpack[Ts]].__args__, (Unpack[Ts],)) - self.assertEqual(tuple[Unpack[Ts], int].__args__, (Unpack[Ts], int)) + self.assertEqual(tuple[*Ts, int].__args__, (*Ts, int)) self.assertEqual(Tuple[Unpack[Ts], int].__args__, (Unpack[Ts], int)) - self.assertEqual(tuple[int, Unpack[Ts]].__args__, (int, Unpack[Ts])) + self.assertEqual(tuple[int, *Ts].__args__, (int, *Ts)) self.assertEqual(Tuple[int, Unpack[Ts]].__args__, (int, Unpack[Ts])) - self.assertEqual(tuple[int, Unpack[Ts], str].__args__, - (int, Unpack[Ts], str)) + self.assertEqual(tuple[int, *Ts, str].__args__, + (int, *Ts, str)) self.assertEqual(Tuple[int, Unpack[Ts], str].__args__, (int, Unpack[Ts], str)) - self.assertEqual(tuple[Unpack[Ts], int].__args__, (Unpack[Ts], int)) + self.assertEqual(tuple[*Ts, int].__args__, (*Ts, int)) self.assertEqual(Tuple[Unpack[Ts]].__args__, (Unpack[Ts],)) def test_callable_args_are_correct(self): @@ -1083,63 +1249,97 @@ def test_callable_args_are_correct(self): # TypeVarTuple in the arguments - a = Callable[[Unpack[Ts]], None] - self.assertEqual(a.__args__, (Unpack[Ts], type(None))) + a = Callable[[*Ts], None] + b = Callable[[Unpack[Ts]], None] + self.assertEqual(a.__args__, (*Ts, type(None))) + self.assertEqual(b.__args__, (Unpack[Ts], type(None))) - b = Callable[[int, Unpack[Ts]], None] - self.assertEqual(b.__args__, (int, Unpack[Ts], type(None))) + c = Callable[[int, *Ts], None] + d = Callable[[int, Unpack[Ts]], None] + self.assertEqual(c.__args__, (int, *Ts, type(None))) + self.assertEqual(d.__args__, (int, Unpack[Ts], type(None))) - c = Callable[[Unpack[Ts], int], None] - self.assertEqual(c.__args__, (Unpack[Ts], int, type(None))) + e = Callable[[*Ts, int], None] + f = Callable[[Unpack[Ts], int], None] + self.assertEqual(e.__args__, (*Ts, int, type(None))) + self.assertEqual(f.__args__, (Unpack[Ts], int, type(None))) - d = Callable[[str, Unpack[Ts], int], None] - self.assertEqual(d.__args__, (str, Unpack[Ts], int, type(None))) + g = Callable[[str, *Ts, int], None] + h = Callable[[str, Unpack[Ts], int], None] + self.assertEqual(g.__args__, (str, *Ts, int, type(None))) + self.assertEqual(h.__args__, (str, Unpack[Ts], int, type(None))) # TypeVarTuple as the return - e = Callable[[None], Unpack[Ts]] - self.assertEqual(e.__args__, (type(None), Unpack[Ts])) + i = Callable[[None], *Ts] + j = Callable[[None], Unpack[Ts]] + self.assertEqual(i.__args__, (type(None), *Ts)) + self.assertEqual(i.__args__, (type(None), Unpack[Ts])) - f = Callable[[None], tuple[int, Unpack[Ts]]] - self.assertEqual(f.__args__, (type(None), tuple[int, Unpack[Ts]])) + k = Callable[[None], tuple[int, *Ts]] + l = Callable[[None], Tuple[int, Unpack[Ts]]] + self.assertEqual(k.__args__, (type(None), tuple[int, *Ts])) + self.assertEqual(l.__args__, (type(None), Tuple[int, Unpack[Ts]])) - g = Callable[[None], tuple[Unpack[Ts], int]] - self.assertEqual(g.__args__, (type(None), tuple[Unpack[Ts], int])) + m = Callable[[None], tuple[*Ts, int]] + n = Callable[[None], Tuple[Unpack[Ts], int]] + self.assertEqual(m.__args__, (type(None), tuple[*Ts, int])) + self.assertEqual(n.__args__, (type(None), Tuple[Unpack[Ts], int])) - h = Callable[[None], tuple[str, Unpack[Ts], int]] - self.assertEqual(h.__args__, (type(None), tuple[str, Unpack[Ts], int])) + o = Callable[[None], tuple[str, *Ts, int]] + p = Callable[[None], Tuple[str, Unpack[Ts], int]] + self.assertEqual(o.__args__, (type(None), tuple[str, *Ts, int])) + self.assertEqual(p.__args__, (type(None), Tuple[str, Unpack[Ts], int])) # TypeVarTuple in both - i = Callable[[Unpack[Ts]], Unpack[Ts]] - self.assertEqual(i.__args__, (Unpack[Ts], Unpack[Ts])) + q = Callable[[*Ts], *Ts] + r = Callable[[Unpack[Ts]], Unpack[Ts]] + self.assertEqual(q.__args__, (*Ts, *Ts)) + self.assertEqual(r.__args__, (Unpack[Ts], Unpack[Ts])) - j = Callable[[Unpack[Ts1]], Unpack[Ts2]] - self.assertEqual(j.__args__, (Unpack[Ts1], Unpack[Ts2])) + s = Callable[[*Ts1], *Ts2] + u = Callable[[Unpack[Ts1]], Unpack[Ts2]] + self.assertEqual(s.__args__, (*Ts1, *Ts2)) + self.assertEqual(u.__args__, (Unpack[Ts1], Unpack[Ts2])) def test_variadic_class_with_duplicate_typevartuples_fails(self): Ts1 = TypeVarTuple('Ts1') Ts2 = TypeVarTuple('Ts2') + + with self.assertRaises(TypeError): + class C(Generic[*Ts1, *Ts1]): pass with self.assertRaises(TypeError): class C(Generic[Unpack[Ts1], Unpack[Ts1]]): pass + + with self.assertRaises(TypeError): + class C(Generic[*Ts1, *Ts2, *Ts1]): pass with self.assertRaises(TypeError): class C(Generic[Unpack[Ts1], Unpack[Ts2], Unpack[Ts1]]): pass def test_type_concatenation_in_variadic_class_argument_list_succeeds(self): Ts = TypeVarTuple('Ts') class C(Generic[Unpack[Ts]]): pass + + C[int, *Ts] C[int, Unpack[Ts]] + + C[*Ts, int] C[Unpack[Ts], int] + + C[int, *Ts, str] C[int, Unpack[Ts], str] + + C[int, bool, *Ts, float, str] C[int, bool, Unpack[Ts], float, str] def test_type_concatenation_in_tuple_argument_list_succeeds(self): Ts = TypeVarTuple('Ts') - tuple[int, Unpack[Ts]] - tuple[Unpack[Ts], int] - tuple[int, Unpack[Ts], str] - tuple[int, bool, Unpack[Ts], float, str] + tuple[int, *Ts] + tuple[*Ts, int] + tuple[int, *Ts, str] + tuple[int, bool, *Ts, float, str] Tuple[int, Unpack[Ts]] Tuple[Unpack[Ts], int] @@ -1153,6 +1353,8 @@ class C(Generic[Ts]): pass def test_variadic_class_definition_using_concrete_types_fails(self): Ts = TypeVarTuple('Ts') + with self.assertRaises(TypeError): + class F(Generic[*Ts, int]): pass with self.assertRaises(TypeError): class E(Generic[Unpack[Ts], int]): pass @@ -1161,32 +1363,50 @@ def test_variadic_class_with_2_typevars_accepts_2_or_more_args(self): T1 = TypeVar('T1') T2 = TypeVar('T2') - class A(Generic[T1, T2, Unpack[Ts]]): pass + class A(Generic[T1, T2, *Ts]): pass A[int, str] A[int, str, float] A[int, str, float, bool] - class B(Generic[T1, Unpack[Ts], T2]): pass + class B(Generic[T1, T2, Unpack[Ts]]): pass B[int, str] B[int, str, float] B[int, str, float, bool] - class C(Generic[Unpack[Ts], T1, T2]): pass + class C(Generic[T1, *Ts, T2]): pass C[int, str] C[int, str, float] C[int, str, float, bool] + class D(Generic[T1, Unpack[Ts], T2]): pass + D[int, str] + D[int, str, float] + D[int, str, float, bool] + + class E(Generic[*Ts, T1, T2]): pass + E[int, str] + E[int, str, float] + E[int, str, float, bool] + + class F(Generic[Unpack[Ts], T1, T2]): pass + F[int, str] + F[int, str, float] + F[int, str, float, bool] + def test_variadic_args_annotations_are_correct(self): Ts = TypeVarTuple('Ts') + def f(*args: Unpack[Ts]): pass + def g(*args: *Ts): pass self.assertEqual(f.__annotations__, {'args': Unpack[Ts]}) + self.assertEqual(g.__annotations__, {'args': (*Ts,)[0]}) def test_variadic_args_with_ellipsis_annotations_are_correct(self): Ts = TypeVarTuple('Ts') - def a(*args: Unpack[tuple[int, ...]]): pass + def a(*args: *tuple[int, ...]): pass self.assertEqual(a.__annotations__, - {'args': Unpack[tuple[int, ...]]}) + {'args': (*tuple[int, ...],)[0]}) def b(*args: Unpack[Tuple[int, ...]]): pass self.assertEqual(b.__annotations__, @@ -1195,30 +1415,30 @@ def b(*args: Unpack[Tuple[int, ...]]): pass def test_concatenation_in_variadic_args_annotations_are_correct(self): Ts = TypeVarTuple('Ts') - # Unpacking using `Unpack`, native `tuple` type + # Unpacking using `*`, native `tuple` type - def a(*args: Unpack[tuple[int, Unpack[Ts]]]): pass + def a(*args: *tuple[int, *Ts]): pass self.assertEqual( a.__annotations__, - {'args': Unpack[tuple[int, Unpack[Ts]]]}, + {'args': (*tuple[int, *Ts],)[0]}, ) - def b(*args: Unpack[tuple[Unpack[Ts], int]]): pass + def b(*args: *tuple[*Ts, int]): pass self.assertEqual( b.__annotations__, - {'args': Unpack[tuple[Unpack[Ts], int]]}, + {'args': (*tuple[*Ts, int],)[0]}, ) - def c(*args: Unpack[tuple[str, Unpack[Ts], int]]): pass + def c(*args: *tuple[str, *Ts, int]): pass self.assertEqual( c.__annotations__, - {'args': Unpack[tuple[str, Unpack[Ts], int]]}, + {'args': (*tuple[str, *Ts, int],)[0]}, ) - def d(*args: Unpack[tuple[int, bool, Unpack[Ts], float, str]]): pass + def d(*args: *tuple[int, bool, *Ts, float, str]): pass self.assertEqual( d.__annotations__, - {'args': Unpack[tuple[int, bool, Unpack[Ts], float, str]]}, + {'args': (*tuple[int, bool, *Ts, float, str],)[0]}, ) # Unpacking using `Unpack`, `Tuple` type from typing.py @@ -1249,47 +1469,78 @@ def h(*args: Unpack[Tuple[int, bool, Unpack[Ts], float, str]]): pass def test_variadic_class_same_args_results_in_equalty(self): Ts = TypeVarTuple('Ts') - class C(Generic[Unpack[Ts]]): pass + class C(Generic[*Ts]): pass + class D(Generic[Unpack[Ts]]): pass self.assertEqual(C[int], C[int]) + self.assertEqual(D[int], D[int]) Ts1 = TypeVarTuple('Ts1') Ts2 = TypeVarTuple('Ts2') + + self.assertEqual( + C[*Ts1], + C[*Ts1], + ) + self.assertEqual( + D[Unpack[Ts1]], + D[Unpack[Ts1]], + ) + self.assertEqual( - C[Unpack[Ts1]], - C[Unpack[Ts1]], + C[*Ts1, *Ts2], + C[*Ts1, *Ts2], ) self.assertEqual( - C[Unpack[Ts1], Unpack[Ts2]], - C[Unpack[Ts1], Unpack[Ts2]], + D[Unpack[Ts1], Unpack[Ts2]], + D[Unpack[Ts1], Unpack[Ts2]], ) + self.assertEqual( - C[int, Unpack[Ts1], Unpack[Ts2]], - C[int, Unpack[Ts1], Unpack[Ts2]], + C[int, *Ts1, *Ts2], + C[int, *Ts1, *Ts2], + ) + self.assertEqual( + D[int, Unpack[Ts1], Unpack[Ts2]], + D[int, Unpack[Ts1], Unpack[Ts2]], ) def test_variadic_class_arg_ordering_matters(self): Ts = TypeVarTuple('Ts') - class C(Generic[Unpack[Ts]]): pass + class C(Generic[*Ts]): pass + class D(Generic[Unpack[Ts]]): pass self.assertNotEqual( C[int, str], C[str, int], ) + self.assertNotEqual( + D[int, str], + D[str, int], + ) Ts1 = TypeVarTuple('Ts1') Ts2 = TypeVarTuple('Ts2') + self.assertNotEqual( - C[Unpack[Ts1], Unpack[Ts2]], - C[Unpack[Ts2], Unpack[Ts1]], + C[*Ts1, *Ts2], + C[*Ts2, *Ts1], + ) + self.assertNotEqual( + D[Unpack[Ts1], Unpack[Ts2]], + D[Unpack[Ts2], Unpack[Ts1]], ) def test_variadic_class_arg_typevartuple_identity_matters(self): Ts = TypeVarTuple('Ts') - class C(Generic[Unpack[Ts]]): pass Ts1 = TypeVarTuple('Ts1') Ts2 = TypeVarTuple('Ts2') - self.assertNotEqual(C[Unpack[Ts1]], C[Unpack[Ts2]]) + + class C(Generic[*Ts]): pass + class D(Generic[Unpack[Ts]]): pass + + self.assertNotEqual(C[*Ts1], C[*Ts2]) + self.assertNotEqual(D[Unpack[Ts1]], D[Unpack[Ts2]]) class TypeVarTuplePicklingTests(BaseTestCase): @@ -1309,10 +1560,15 @@ def test_pickling_then_unpickling_results_in_same_identity(self, proto): def test_pickling_then_unpickling_unpacked_results_in_same_identity(self, proto): global global_Ts # See explanation at start of class. global_Ts = TypeVarTuple('global_Ts') - unpacked1 = Unpack[global_Ts] + + unpacked1 = (*global_Ts,)[0] unpacked2 = pickle.loads(pickle.dumps(unpacked1, proto)) self.assertIs(unpacked1, unpacked2) + unpacked3 = Unpack[global_Ts] + unpacked4 = pickle.loads(pickle.dumps(unpacked3, proto)) + self.assertIs(unpacked3, unpacked4) + @all_pickle_protocols def test_pickling_then_unpickling_tuple_with_typevartuple_equality( self, proto @@ -1321,17 +1577,19 @@ def test_pickling_then_unpickling_tuple_with_typevartuple_equality( global_T = TypeVar('global_T') global_Ts = TypeVarTuple('global_Ts') - a1 = Tuple[Unpack[global_Ts]] - a2 = pickle.loads(pickle.dumps(a1, proto)) - self.assertEqual(a1, a2) + tuples = [ + tuple[*global_Ts], + Tuple[Unpack[global_Ts]], - a1 = Tuple[T, Unpack[global_Ts]] - a2 = pickle.loads(pickle.dumps(a1, proto)) - self.assertEqual(a1, a2) + tuple[T, *global_Ts], + Tuple[T, Unpack[global_Ts]], - a1 = Tuple[int, Unpack[global_Ts]] - a2 = pickle.loads(pickle.dumps(a1, proto)) - self.assertEqual(a1, a2) + tuple[int, *global_Ts], + Tuple[int, Unpack[global_Ts]], + ] + for t in tuples: + t2 = pickle.loads(pickle.dumps(t, proto)) + self.assertEqual(t, t2) class UnionTests(BaseTestCase): @@ -2660,7 +2918,7 @@ class CP(P[int]): self.assertEqual(x.bar, 'abc') self.assertEqual(x.x, 1) self.assertEqual(x.__dict__, {'foo': 42, 'bar': 'abc'}) - s = pickle.dumps(P) + s = pickle.dumps(P, proto) D = pickle.loads(s) class E: @@ -3554,11 +3812,11 @@ class D(C): self.assertEqual(D.__parameters__, ()) - with self.assertRaises(Exception): + with self.assertRaises(TypeError): D[int] - with self.assertRaises(Exception): + with self.assertRaises(TypeError): D[Any] - with self.assertRaises(Exception): + with self.assertRaises(TypeError): D[T] def test_new_with_args(self): @@ -3676,6 +3934,34 @@ class A: # C version of GenericAlias self.assertEqual(list[A()].__parameters__, (T,)) + def test_non_generic_subscript(self): + T = TypeVar('T') + class G(Generic[T]): + pass + class A: + __parameters__ = (T,) + + for s in (int, G, A, List, list, + TypeVar, TypeVarTuple, ParamSpec, + types.GenericAlias, types.UnionType): + + for t in Tuple, tuple: + with self.subTest(tuple=t, sub=s): + self.assertEqual(t[s, T][int], t[s, int]) + self.assertEqual(t[T, s][int], t[int, s]) + a = t[s] + with self.assertRaises(TypeError): + a[int] + + for c in Callable, collections.abc.Callable: + with self.subTest(callable=c, sub=s): + self.assertEqual(c[[s], T][int], c[[s], int]) + self.assertEqual(c[[T], s][int], c[[int], s]) + a = c[[s], s] + with self.assertRaises(TypeError): + a[int] + + class ClassVarTests(BaseTestCase): def test_basics(self): @@ -4361,7 +4647,7 @@ def method(self): ... class OverloadTests(BaseTestCase): def test_overload_fails(self): - with self.assertRaises(RuntimeError): + with self.assertRaises(NotImplementedError): @overload def blah(): @@ -4379,6 +4665,21 @@ def blah(): blah() + @cpython_only # gh-98713 + def test_overload_on_compiled_functions(self): + with patch("typing._overload_registry", + defaultdict(lambda: defaultdict(dict))): + # The registry starts out empty: + self.assertEqual(typing._overload_registry, {}) + + # This should just not fail: + overload(sum) + overload(print) + + # No overloads are recorded (but, it still has a side-effect): + self.assertEqual(typing.get_overloads(sum), []) + self.assertEqual(typing.get_overloads(print), []) + def set_up_overloads(self): def blah(): pass @@ -4413,6 +4714,9 @@ def some_other_func(): pass other_overload = some_other_func def some_other_func(): pass self.assertEqual(list(get_overloads(some_other_func)), [other_overload]) + # Unrelated function still has no overloads: + def not_overloaded(): pass + self.assertEqual(list(get_overloads(not_overloaded)), []) # Make sure that after we clear all overloads, the registry is # completely empty. @@ -4882,6 +5186,7 @@ def h(x: collections.abc.Callable[P, int]): ... class GetUtilitiesTestCase(TestCase): def test_get_origin(self): T = TypeVar('T') + Ts = TypeVarTuple('Ts') P = ParamSpec('P') class C(Generic[T]): pass self.assertIs(get_origin(C[int]), C) @@ -4905,6 +5210,10 @@ class C(Generic[T]): pass self.assertIs(get_origin(P.kwargs), P) self.assertIs(get_origin(Required[int]), Required) self.assertIs(get_origin(NotRequired[int]), NotRequired) + self.assertIs(get_origin((*Ts,)[0]), Unpack) + self.assertIs(get_origin(Unpack[Ts]), Unpack) + self.assertIs(get_origin((*tuple[*Ts],)[0]), tuple) + self.assertIs(get_origin(Unpack[Tuple[Unpack[Ts]]]), Unpack) def test_get_args(self): T = TypeVar('T') @@ -4944,6 +5253,16 @@ class C(Generic[T]): pass self.assertEqual(get_args(list | str), (list, str)) self.assertEqual(get_args(Required[int]), (int,)) self.assertEqual(get_args(NotRequired[int]), (int,)) + self.assertEqual(get_args(TypeAlias), ()) + self.assertEqual(get_args(TypeGuard[int]), (int,)) + Ts = TypeVarTuple('Ts') + self.assertEqual(get_args(Ts), ()) + self.assertEqual(get_args((*Ts,)[0]), (Ts,)) + self.assertEqual(get_args(Unpack[Ts]), (Ts,)) + self.assertEqual(get_args(tuple[*Ts]), (*Ts,)) + self.assertEqual(get_args(tuple[Unpack[Ts]]), (Unpack[Ts],)) + self.assertEqual(get_args((*tuple[*Ts],)[0]), (*Ts,)) + self.assertEqual(get_args(Unpack[tuple[Unpack[Ts]]]), (tuple[Unpack[Ts]],)) class CollectionsAbcTests(BaseTestCase): @@ -6609,69 +6928,112 @@ def test_typevar_subst(self): T1 = TypeVar('T1') T2 = TypeVar('T2') - A = Annotated[Tuple[Unpack[Ts]], dec] - self.assertEqual(A[int], Annotated[Tuple[int], dec]) - self.assertEqual(A[str, int], Annotated[Tuple[str, int], dec]) + A = Annotated[tuple[*Ts], dec] + self.assertEqual(A[int], Annotated[tuple[int], dec]) + self.assertEqual(A[str, int], Annotated[tuple[str, int], dec]) with self.assertRaises(TypeError): - Annotated[Unpack[Ts], dec] + Annotated[*Ts, dec] - B = Annotated[Tuple[T, Unpack[Ts]], dec] + B = Annotated[Tuple[Unpack[Ts]], dec] self.assertEqual(B[int], Annotated[Tuple[int], dec]) - self.assertEqual(B[int, str], Annotated[Tuple[int, str], dec]) - self.assertEqual( - B[int, str, float], - Annotated[Tuple[int, str, float], dec] - ) + self.assertEqual(B[str, int], Annotated[Tuple[str, int], dec]) with self.assertRaises(TypeError): - B[()] + Annotated[Unpack[Ts], dec] - C = Annotated[Tuple[Unpack[Ts], T], dec] - self.assertEqual(C[int], Annotated[Tuple[int], dec]) - self.assertEqual(C[int, str], Annotated[Tuple[int, str], dec]) + C = Annotated[tuple[T, *Ts], dec] + self.assertEqual(C[int], Annotated[tuple[int], dec]) + self.assertEqual(C[int, str], Annotated[tuple[int, str], dec]) self.assertEqual( C[int, str, float], - Annotated[Tuple[int, str, float], dec] + Annotated[tuple[int, str, float], dec] ) with self.assertRaises(TypeError): C[()] - D = Annotated[Tuple[T1, Unpack[Ts], T2], dec] + D = Annotated[Tuple[T, Unpack[Ts]], dec] + self.assertEqual(D[int], Annotated[Tuple[int], dec]) self.assertEqual(D[int, str], Annotated[Tuple[int, str], dec]) self.assertEqual( D[int, str, float], Annotated[Tuple[int, str, float], dec] ) + with self.assertRaises(TypeError): + D[()] + + E = Annotated[tuple[*Ts, T], dec] + self.assertEqual(E[int], Annotated[tuple[int], dec]) + self.assertEqual(E[int, str], Annotated[tuple[int, str], dec]) self.assertEqual( - D[int, str, bool, float], - Annotated[Tuple[int, str, bool, float], dec] + E[int, str, float], + Annotated[tuple[int, str, float], dec] ) with self.assertRaises(TypeError): - D[int] + E[()] - # Now let's try creating an alias from an alias. - - Ts2 = TypeVarTuple('Ts2') - T3 = TypeVar('T3') - T4 = TypeVar('T4') + F = Annotated[Tuple[Unpack[Ts], T], dec] + self.assertEqual(F[int], Annotated[Tuple[int], dec]) + self.assertEqual(F[int, str], Annotated[Tuple[int, str], dec]) + self.assertEqual( + F[int, str, float], + Annotated[Tuple[int, str, float], dec] + ) + with self.assertRaises(TypeError): + F[()] - E = D[T3, Unpack[Ts2], T4] + G = Annotated[tuple[T1, *Ts, T2], dec] + self.assertEqual(G[int, str], Annotated[tuple[int, str], dec]) self.assertEqual( - E, - Annotated[Tuple[T3, Unpack[Ts2], T4], dec] + G[int, str, float], + Annotated[tuple[int, str, float], dec] ) self.assertEqual( - E[int, str], Annotated[Tuple[int, str], dec] + G[int, str, bool, float], + Annotated[tuple[int, str, bool, float], dec] ) + with self.assertRaises(TypeError): + G[int] + + H = Annotated[Tuple[T1, Unpack[Ts], T2], dec] + self.assertEqual(H[int, str], Annotated[Tuple[int, str], dec]) self.assertEqual( - E[int, str, float], + H[int, str, float], Annotated[Tuple[int, str, float], dec] ) self.assertEqual( - E[int, str, bool, float], + H[int, str, bool, float], Annotated[Tuple[int, str, bool, float], dec] ) with self.assertRaises(TypeError): - E[int] + H[int] + + # Now let's try creating an alias from an alias. + + Ts2 = TypeVarTuple('Ts2') + T3 = TypeVar('T3') + T4 = TypeVar('T4') + + # G is Annotated[tuple[T1, *Ts, T2], dec]. + I = G[T3, *Ts2, T4] + J = G[T3, Unpack[Ts2], T4] + + for x, y in [ + (I, Annotated[tuple[T3, *Ts2, T4], dec]), + (J, Annotated[tuple[T3, Unpack[Ts2], T4], dec]), + (I[int, str], Annotated[tuple[int, str], dec]), + (J[int, str], Annotated[tuple[int, str], dec]), + (I[int, str, float], Annotated[tuple[int, str, float], dec]), + (J[int, str, float], Annotated[tuple[int, str, float], dec]), + (I[int, str, bool, float], + Annotated[tuple[int, str, bool, float], dec]), + (J[int, str, bool, float], + Annotated[tuple[int, str, bool, float], dec]), + ]: + self.assertEqual(x, y) + + with self.assertRaises(TypeError): + I[int] + with self.assertRaises(TypeError): + J[int] def test_annotated_in_other_types(self): X = List[Annotated[T, 5]] @@ -6850,6 +7212,65 @@ class X(Generic[P, P2]): self.assertEqual(G1.__args__, ((int, str), (bytes,))) self.assertEqual(G2.__args__, ((int,), (str, bytes))) + def test_typevartuple_and_paramspecs_in_user_generics(self): + Ts = TypeVarTuple("Ts") + P = ParamSpec("P") + + class X(Generic[*Ts, P]): + f: Callable[P, int] + g: Tuple[*Ts] + + G1 = X[int, [bytes]] + self.assertEqual(G1.__args__, (int, (bytes,))) + G2 = X[int, str, [bytes]] + self.assertEqual(G2.__args__, (int, str, (bytes,))) + G3 = X[[bytes]] + self.assertEqual(G3.__args__, ((bytes,),)) + G4 = X[[]] + self.assertEqual(G4.__args__, ((),)) + with self.assertRaises(TypeError): + X[()] + + class Y(Generic[P, *Ts]): + f: Callable[P, int] + g: Tuple[*Ts] + + G1 = Y[[bytes], int] + self.assertEqual(G1.__args__, ((bytes,), int)) + G2 = Y[[bytes], int, str] + self.assertEqual(G2.__args__, ((bytes,), int, str)) + G3 = Y[[bytes]] + self.assertEqual(G3.__args__, ((bytes,),)) + G4 = Y[[]] + self.assertEqual(G4.__args__, ((),)) + with self.assertRaises(TypeError): + Y[()] + + def test_typevartuple_and_paramspecs_in_generic_aliases(self): + P = ParamSpec('P') + T = TypeVar('T') + Ts = TypeVarTuple('Ts') + + for C in Callable, collections.abc.Callable: + with self.subTest(generic=C): + A = C[P, Tuple[*Ts]] + B = A[[int, str], bytes, float] + self.assertEqual(B.__args__, (int, str, Tuple[bytes, float])) + + class X(Generic[T, P]): + pass + + A = X[Tuple[*Ts], P] + B = A[bytes, float, [int, str]] + self.assertEqual(B.__args__, (Tuple[bytes, float], (int, str,))) + + class Y(Generic[P, T]): + pass + + A = Y[P, Tuple[*Ts]] + B = A[[int, str], bytes, float] + self.assertEqual(B.__args__, ((int, str,), Tuple[bytes, float])) + def test_var_substitution(self): T = TypeVar("T") P = ParamSpec("P") diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 90bd75f550dff6..b3c02c1b998475 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -16,7 +16,6 @@ import unicodedata import unittest import warnings -from test.support import import_helper from test.support import warnings_helper from test import support, string_tests from test.support.script_helper import assert_python_failure @@ -257,6 +256,20 @@ def test_find(self): self.checkequalnofix(9, 'abcdefghiabc', 'find', 'abc', 1) self.checkequalnofix(-1, 'abcdefghiabc', 'find', 'def', 4) + # test utf-8 non-ascii char + self.checkequal(0, 'тест', 'find', 'т') + self.checkequal(3, 'тест', 'find', 'т', 1) + self.checkequal(-1, 'тест', 'find', 'т', 1, 3) + self.checkequal(-1, 'тест', 'find', 'e') # english `e` + # test utf-8 non-ascii slice + self.checkequal(1, 'тест тест', 'find', 'ес') + self.checkequal(1, 'тест тест', 'find', 'ес', 1) + self.checkequal(1, 'тест тест', 'find', 'ес', 1, 3) + self.checkequal(6, 'тест тест', 'find', 'ес', 2) + self.checkequal(-1, 'тест тест', 'find', 'ес', 6, 7) + self.checkequal(-1, 'тест тест', 'find', 'ес', 7) + self.checkequal(-1, 'тест тест', 'find', 'ec') # english `ec` + self.assertRaises(TypeError, 'hello'.find) self.assertRaises(TypeError, 'hello'.find, 42) # test mixed kinds @@ -287,6 +300,19 @@ def test_rfind(self): self.checkequalnofix(9, 'abcdefghiabc', 'rfind', 'abc') self.checkequalnofix(12, 'abcdefghiabc', 'rfind', '') self.checkequalnofix(12, 'abcdefghiabc', 'rfind', '') + # test utf-8 non-ascii char + self.checkequal(1, 'тест', 'rfind', 'е') + self.checkequal(1, 'тест', 'rfind', 'е', 1) + self.checkequal(-1, 'тест', 'rfind', 'е', 2) + self.checkequal(-1, 'тест', 'rfind', 'e') # english `e` + # test utf-8 non-ascii slice + self.checkequal(6, 'тест тест', 'rfind', 'ес') + self.checkequal(6, 'тест тест', 'rfind', 'ес', 1) + self.checkequal(1, 'тест тест', 'rfind', 'ес', 1, 3) + self.checkequal(6, 'тест тест', 'rfind', 'ес', 2) + self.checkequal(-1, 'тест тест', 'rfind', 'ес', 6, 7) + self.checkequal(-1, 'тест тест', 'rfind', 'ес', 7) + self.checkequal(-1, 'тест тест', 'rfind', 'ec') # english `ec` # test mixed kinds self.checkequal(0, 'a' + '\u0102' * 100, 'rfind', 'a') self.checkequal(0, 'a' + '\U00100304' * 100, 'rfind', 'a') @@ -441,10 +467,10 @@ def test_split(self): def test_rsplit(self): string_tests.CommonTest.test_rsplit(self) # test mixed kinds - for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): + for left, right in ('ba', 'юё', '\u0101\u0100', '\U00010301\U00010300'): left *= 9 right *= 9 - for delim in ('c', '\u0102', '\U00010302'): + for delim in ('c', 'ы', '\u0102', '\U00010302'): self.checkequal([left + right], left + right, 'rsplit', delim) self.checkequal([left, right], @@ -454,6 +480,10 @@ def test_rsplit(self): self.checkequal([left, right], left + delim * 2 + right, 'rsplit', delim *2) + # Check `None` as well: + self.checkequal([left + right], + left + right, 'rsplit', None) + def test_partition(self): string_tests.MixinStrUnicodeUserStringTest.test_partition(self) # test mixed kinds @@ -2593,447 +2623,6 @@ def test_check_encoding_errors(self): self.assertEqual(proc.rc, 10, proc) -class CAPITest(unittest.TestCase): - - # Test PyUnicode_FromFormat() - def test_from_format(self): - import_helper.import_module('ctypes') - from ctypes import ( - c_char_p, - pythonapi, py_object, sizeof, - c_int, c_long, c_longlong, c_ssize_t, - c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) - name = "PyUnicode_FromFormat" - _PyUnicode_FromFormat = getattr(pythonapi, name) - _PyUnicode_FromFormat.argtypes = (c_char_p,) - _PyUnicode_FromFormat.restype = py_object - - def PyUnicode_FromFormat(format, *args): - cargs = tuple( - py_object(arg) if isinstance(arg, str) else arg - for arg in args) - return _PyUnicode_FromFormat(format, *cargs) - - def check_format(expected, format, *args): - text = PyUnicode_FromFormat(format, *args) - self.assertEqual(expected, text) - - # ascii format, non-ascii argument - check_format('ascii\x7f=unicode\xe9', - b'ascii\x7f=%U', 'unicode\xe9') - - # non-ascii format, ascii argument: ensure that PyUnicode_FromFormatV() - # raises an error - self.assertRaisesRegex(ValueError, - r'^PyUnicode_FromFormatV\(\) expects an ASCII-encoded format ' - 'string, got a non-ASCII byte: 0xe9$', - PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii') - - # test "%c" - check_format('\uabcd', - b'%c', c_int(0xabcd)) - check_format('\U0010ffff', - b'%c', c_int(0x10ffff)) - with self.assertRaises(OverflowError): - PyUnicode_FromFormat(b'%c', c_int(0x110000)) - # Issue #18183 - check_format('\U00010000\U00100000', - b'%c%c', c_int(0x10000), c_int(0x100000)) - - # test "%" - check_format('%', - b'%') - check_format('%', - b'%%') - check_format('%s', - b'%%s') - check_format('[%]', - b'[%%]') - check_format('%abc', - b'%%%s', b'abc') - - # truncated string - check_format('abc', - b'%.3s', b'abcdef') - check_format('abc[\ufffd', - b'%.5s', 'abc[\u20ac]'.encode('utf8')) - check_format("'\\u20acABC'", - b'%A', '\u20acABC') - check_format("'\\u20", - b'%.5A', '\u20acABCDEF') - check_format("'\u20acABC'", - b'%R', '\u20acABC') - check_format("'\u20acA", - b'%.3R', '\u20acABCDEF') - check_format('\u20acAB', - b'%.3S', '\u20acABCDEF') - check_format('\u20acAB', - b'%.3U', '\u20acABCDEF') - check_format('\u20acAB', - b'%.3V', '\u20acABCDEF', None) - check_format('abc[\ufffd', - b'%.5V', None, 'abc[\u20ac]'.encode('utf8')) - - # following tests comes from #7330 - # test width modifier and precision modifier with %S - check_format("repr= abc", - b'repr=%5S', 'abc') - check_format("repr=ab", - b'repr=%.2S', 'abc') - check_format("repr= ab", - b'repr=%5.2S', 'abc') - - # test width modifier and precision modifier with %R - check_format("repr= 'abc'", - b'repr=%8R', 'abc') - check_format("repr='ab", - b'repr=%.3R', 'abc') - check_format("repr= 'ab", - b'repr=%5.3R', 'abc') - - # test width modifier and precision modifier with %A - check_format("repr= 'abc'", - b'repr=%8A', 'abc') - check_format("repr='ab", - b'repr=%.3A', 'abc') - check_format("repr= 'ab", - b'repr=%5.3A', 'abc') - - # test width modifier and precision modifier with %s - check_format("repr= abc", - b'repr=%5s', b'abc') - check_format("repr=ab", - b'repr=%.2s', b'abc') - check_format("repr= ab", - b'repr=%5.2s', b'abc') - - # test width modifier and precision modifier with %U - check_format("repr= abc", - b'repr=%5U', 'abc') - check_format("repr=ab", - b'repr=%.2U', 'abc') - check_format("repr= ab", - b'repr=%5.2U', 'abc') - - # test width modifier and precision modifier with %V - check_format("repr= abc", - b'repr=%5V', 'abc', b'123') - check_format("repr=ab", - b'repr=%.2V', 'abc', b'123') - check_format("repr= ab", - b'repr=%5.2V', 'abc', b'123') - check_format("repr= 123", - b'repr=%5V', None, b'123') - check_format("repr=12", - b'repr=%.2V', None, b'123') - check_format("repr= 12", - b'repr=%5.2V', None, b'123') - - # test integer formats (%i, %d, %u) - check_format('010', - b'%03i', c_int(10)) - check_format('0010', - b'%0.4i', c_int(10)) - check_format('-123', - b'%i', c_int(-123)) - check_format('-123', - b'%li', c_long(-123)) - check_format('-123', - b'%lli', c_longlong(-123)) - check_format('-123', - b'%zi', c_ssize_t(-123)) - - check_format('-123', - b'%d', c_int(-123)) - check_format('-123', - b'%ld', c_long(-123)) - check_format('-123', - b'%lld', c_longlong(-123)) - check_format('-123', - b'%zd', c_ssize_t(-123)) - - check_format('123', - b'%u', c_uint(123)) - check_format('123', - b'%lu', c_ulong(123)) - check_format('123', - b'%llu', c_ulonglong(123)) - check_format('123', - b'%zu', c_size_t(123)) - - # test long output - min_longlong = -(2 ** (8 * sizeof(c_longlong) - 1)) - max_longlong = -min_longlong - 1 - check_format(str(min_longlong), - b'%lld', c_longlong(min_longlong)) - check_format(str(max_longlong), - b'%lld', c_longlong(max_longlong)) - max_ulonglong = 2 ** (8 * sizeof(c_ulonglong)) - 1 - check_format(str(max_ulonglong), - b'%llu', c_ulonglong(max_ulonglong)) - PyUnicode_FromFormat(b'%p', c_void_p(-1)) - - # test padding (width and/or precision) - check_format('123'.rjust(10, '0'), - b'%010i', c_int(123)) - check_format('123'.rjust(100), - b'%100i', c_int(123)) - check_format('123'.rjust(100, '0'), - b'%.100i', c_int(123)) - check_format('123'.rjust(80, '0').rjust(100), - b'%100.80i', c_int(123)) - - check_format('123'.rjust(10, '0'), - b'%010u', c_uint(123)) - check_format('123'.rjust(100), - b'%100u', c_uint(123)) - check_format('123'.rjust(100, '0'), - b'%.100u', c_uint(123)) - check_format('123'.rjust(80, '0').rjust(100), - b'%100.80u', c_uint(123)) - - check_format('123'.rjust(10, '0'), - b'%010x', c_int(0x123)) - check_format('123'.rjust(100), - b'%100x', c_int(0x123)) - check_format('123'.rjust(100, '0'), - b'%.100x', c_int(0x123)) - check_format('123'.rjust(80, '0').rjust(100), - b'%100.80x', c_int(0x123)) - - # test %A - check_format(r"%A:'abc\xe9\uabcd\U0010ffff'", - b'%%A:%A', 'abc\xe9\uabcd\U0010ffff') - - # test %V - check_format('repr=abc', - b'repr=%V', 'abc', b'xyz') - - # Test string decode from parameter of %s using utf-8. - # b'\xe4\xba\xba\xe6\xb0\x91' is utf-8 encoded byte sequence of - # '\u4eba\u6c11' - check_format('repr=\u4eba\u6c11', - b'repr=%V', None, b'\xe4\xba\xba\xe6\xb0\x91') - - #Test replace error handler. - check_format('repr=abc\ufffd', - b'repr=%V', None, b'abc\xff') - - # not supported: copy the raw format string. these tests are just here - # to check for crashes and should not be considered as specifications - check_format('%s', - b'%1%s', b'abc') - check_format('%1abc', - b'%1abc') - check_format('%+i', - b'%+i', c_int(10)) - check_format('%.%s', - b'%.%s', b'abc') - - # Issue #33817: empty strings - check_format('', - b'') - check_format('', - b'%s', b'') - - # Test PyUnicode_AsWideChar() - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_aswidechar(self): - from _testcapi import unicode_aswidechar - import_helper.import_module('ctypes') - from ctypes import c_wchar, sizeof - - wchar, size = unicode_aswidechar('abcdef', 2) - self.assertEqual(size, 2) - self.assertEqual(wchar, 'ab') - - wchar, size = unicode_aswidechar('abc', 3) - self.assertEqual(size, 3) - self.assertEqual(wchar, 'abc') - - wchar, size = unicode_aswidechar('abc', 4) - self.assertEqual(size, 3) - self.assertEqual(wchar, 'abc\0') - - wchar, size = unicode_aswidechar('abc', 10) - self.assertEqual(size, 3) - self.assertEqual(wchar, 'abc\0') - - wchar, size = unicode_aswidechar('abc\0def', 20) - self.assertEqual(size, 7) - self.assertEqual(wchar, 'abc\0def\0') - - nonbmp = chr(0x10ffff) - if sizeof(c_wchar) == 2: - buflen = 3 - nchar = 2 - else: # sizeof(c_wchar) == 4 - buflen = 2 - nchar = 1 - wchar, size = unicode_aswidechar(nonbmp, buflen) - self.assertEqual(size, nchar) - self.assertEqual(wchar, nonbmp + '\0') - - # Test PyUnicode_AsWideCharString() - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_aswidecharstring(self): - from _testcapi import unicode_aswidecharstring - import_helper.import_module('ctypes') - from ctypes import c_wchar, sizeof - - wchar, size = unicode_aswidecharstring('abc') - self.assertEqual(size, 3) - self.assertEqual(wchar, 'abc\0') - - wchar, size = unicode_aswidecharstring('abc\0def') - self.assertEqual(size, 7) - self.assertEqual(wchar, 'abc\0def\0') - - nonbmp = chr(0x10ffff) - if sizeof(c_wchar) == 2: - nchar = 2 - else: # sizeof(c_wchar) == 4 - nchar = 1 - wchar, size = unicode_aswidecharstring(nonbmp) - self.assertEqual(size, nchar) - self.assertEqual(wchar, nonbmp + '\0') - - # Test PyUnicode_AsUCS4() - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_asucs4(self): - from _testcapi import unicode_asucs4 - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', - 'a\ud800b\udfffc', '\ud834\udd1e']: - l = len(s) - self.assertEqual(unicode_asucs4(s, l, True), s+'\0') - self.assertEqual(unicode_asucs4(s, l, False), s+'\uffff') - self.assertEqual(unicode_asucs4(s, l+1, True), s+'\0\uffff') - self.assertEqual(unicode_asucs4(s, l+1, False), s+'\0\uffff') - self.assertRaises(SystemError, unicode_asucs4, s, l-1, True) - self.assertRaises(SystemError, unicode_asucs4, s, l-2, False) - s = '\0'.join([s, s]) - self.assertEqual(unicode_asucs4(s, len(s), True), s+'\0') - self.assertEqual(unicode_asucs4(s, len(s), False), s+'\uffff') - - # Test PyUnicode_AsUTF8() - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_asutf8(self): - from _testcapi import unicode_asutf8 - - bmp = '\u0100' - bmp2 = '\uffff' - nonbmp = chr(0x10ffff) - - self.assertEqual(unicode_asutf8(bmp), b'\xc4\x80') - self.assertEqual(unicode_asutf8(bmp2), b'\xef\xbf\xbf') - self.assertEqual(unicode_asutf8(nonbmp), b'\xf4\x8f\xbf\xbf') - self.assertRaises(UnicodeEncodeError, unicode_asutf8, 'a\ud800b\udfffc') - - # Test PyUnicode_AsUTF8AndSize() - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_asutf8andsize(self): - from _testcapi import unicode_asutf8andsize - - bmp = '\u0100' - bmp2 = '\uffff' - nonbmp = chr(0x10ffff) - - self.assertEqual(unicode_asutf8andsize(bmp), (b'\xc4\x80', 2)) - self.assertEqual(unicode_asutf8andsize(bmp2), (b'\xef\xbf\xbf', 3)) - self.assertEqual(unicode_asutf8andsize(nonbmp), (b'\xf4\x8f\xbf\xbf', 4)) - self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, 'a\ud800b\udfffc') - - # Test PyUnicode_FindChar() - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_findchar(self): - from _testcapi import unicode_findchar - - for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": - for i, ch in enumerate(str): - self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), 1), i) - self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), -1), i) - - str = "!>_= end - self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, 1), -1) - self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, 1), -1) - # negative - self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, 1), 0) - self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, -1), 0) - - # Test PyUnicode_CopyCharacters() - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_copycharacters(self): - from _testcapi import unicode_copycharacters - - strings = [ - 'abcde', '\xa1\xa2\xa3\xa4\xa5', - '\u4f60\u597d\u4e16\u754c\uff01', - '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' - ] - - for idx, from_ in enumerate(strings): - # wide -> narrow: exceed maxchar limitation - for to in strings[:idx]: - self.assertRaises( - SystemError, - unicode_copycharacters, to, 0, from_, 0, 5 - ) - # same kind - for from_start in range(5): - self.assertEqual( - unicode_copycharacters(from_, 0, from_, from_start, 5), - (from_[from_start:from_start+5].ljust(5, '\0'), - 5-from_start) - ) - for to_start in range(5): - self.assertEqual( - unicode_copycharacters(from_, to_start, from_, to_start, 5), - (from_[to_start:to_start+5].rjust(5, '\0'), - 5-to_start) - ) - # narrow -> wide - # Tests omitted since this creates invalid strings. - - s = strings[0] - self.assertRaises(IndexError, unicode_copycharacters, s, 6, s, 0, 5) - self.assertRaises(IndexError, unicode_copycharacters, s, -1, s, 0, 5) - self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, 6, 5) - self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, -1, 5) - self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, 5) - self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, -1) - self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0) - - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_pep393_utf8_caching_bug(self): - # Issue #25709: Problem with string concatenation and utf-8 cache - from _testcapi import getargs_s_hash - for k in 0x24, 0xa4, 0x20ac, 0x1f40d: - s = '' - for i in range(5): - # Due to CPython specific optimization the 's' string can be - # resized in-place. - s += chr(k) - # Parsing with the "s#" format code calls indirectly - # PyUnicode_AsUTF8AndSize() which creates the UTF-8 - # encoded string cached in the Unicode object. - self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) - # Check that the second call returns the same result - self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) - class StringModuleTest(unittest.TestCase): def test_formatter_parser(self): def parse(format): diff --git a/Lib/test/test_unicode_file_functions.py b/Lib/test/test_unicode_file_functions.py index 54916dec4eafa3..47619c8807bafe 100644 --- a/Lib/test/test_unicode_file_functions.py +++ b/Lib/test/test_unicode_file_functions.py @@ -6,6 +6,7 @@ import warnings from unicodedata import normalize from test.support import os_helper +from test import support filenames = [ @@ -123,6 +124,10 @@ def test_open(self): # NFKD in Python is useless, because darwin will normalize it later and so # open(), os.stat(), etc. don't raise any exception. @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "test fails on Emscripten/WASI when host platform is macOS." + ) def test_normalize(self): files = set(self.files) others = set() diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 3514697f548e65..9e0097c892e7f7 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -12,7 +12,8 @@ import unicodedata import unittest from test.support import (open_urlresource, requires_resource, script_helper, - cpython_only, check_disallow_instantiation) + cpython_only, check_disallow_instantiation, + ResourceDenied) class UnicodeMethodsTest(unittest.TestCase): @@ -95,6 +96,13 @@ def test_function_checksum(self): result = h.hexdigest() self.assertEqual(result, self.expectedchecksum) + @requires_resource('cpu') + def test_name_inverse_lookup(self): + for i in range(sys.maxunicode + 1): + char = chr(i) + if looked_name := self.db.name(char, None): + self.assertEqual(self.db.lookup(looked_name), char) + def test_digit(self): self.assertEqual(self.db.digit('A', None), None) self.assertEqual(self.db.digit('9'), 9) @@ -338,8 +346,8 @@ def test_normalization(self): except PermissionError: self.skipTest(f"Permission error when downloading {TESTDATAURL} " f"into the test data directory") - except (OSError, HTTPException): - self.fail(f"Could not retrieve {TESTDATAURL}") + except (OSError, HTTPException) as exc: + self.skipTest(f"Failed to download {TESTDATAURL}: {exc}") with testdata: self.run_normalization_tests(testdata) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 2f629c72ae784e..b4261107235f81 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -653,18 +653,39 @@ def test_attributes_bad_port(self): """Check handling of invalid ports.""" for bytes in (False, True): for parse in (urllib.parse.urlsplit, urllib.parse.urlparse): - for port in ("foo", "1.5", "-1", "0x10"): + for port in ("foo", "1.5", "-1", "0x10", "-0", "1_1", " 1", "1 ", "६"): with self.subTest(bytes=bytes, parse=parse, port=port): netloc = "www.example.net:" + port url = "http://" + netloc if bytes: - netloc = netloc.encode("ascii") - url = url.encode("ascii") + if netloc.isascii() and port.isascii(): + netloc = netloc.encode("ascii") + url = url.encode("ascii") + else: + continue p = parse(url) self.assertEqual(p.netloc, netloc) with self.assertRaises(ValueError): p.port + def test_attributes_bad_scheme(self): + """Check handling of invalid schemes.""" + for bytes in (False, True): + for parse in (urllib.parse.urlsplit, urllib.parse.urlparse): + for scheme in (".", "+", "-", "0", "http&", "६http"): + with self.subTest(bytes=bytes, parse=parse, scheme=scheme): + url = scheme + "://www.example.net" + if bytes: + if url.isascii(): + url = url.encode("ascii") + else: + continue + p = parse(url) + if bytes: + self.assertEqual(p.scheme, b"") + else: + self.assertEqual(p.scheme, "") + def test_attributes_without_netloc(self): # This example is straight from RFC 3261. It looks like it # should allow the username, hostname, and port to be filled @@ -1195,6 +1216,7 @@ def test_splitnport(self): self.assertEqual(splitnport('127.0.0.1', 55), ('127.0.0.1', 55)) self.assertEqual(splitnport('parrot:cheese'), ('parrot', None)) self.assertEqual(splitnport('parrot:cheese', 55), ('parrot', None)) + self.assertEqual(splitnport('parrot: +1_0 '), ('parrot', None)) def test_splitquery(self): # Normal cases are exercised by other tests; ensure that we also diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index decf021c647770..86ce60fef13975 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -216,7 +216,7 @@ def test_upgrade_dependencies(self): if sys.platform == 'win32': expect_exe = os.path.normcase(os.path.realpath(expect_exe)) - def pip_cmd_checker(cmd): + def pip_cmd_checker(cmd, **kwargs): cmd[0] = os.path.normcase(cmd[0]) self.assertEqual( cmd, @@ -232,7 +232,7 @@ def pip_cmd_checker(cmd): ) fake_context = builder.ensure_directories(fake_env_dir) - with patch('venv.subprocess.check_call', pip_cmd_checker): + with patch('venv.subprocess.check_output', pip_cmd_checker): builder.upgrade_dependencies(fake_context) @requireVenvCreate @@ -537,6 +537,80 @@ def test_pathsep_error(self): self.assertRaises(ValueError, venv.create, bad_itempath) self.assertRaises(ValueError, venv.create, pathlib.Path(bad_itempath)) + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') + @requireVenvCreate + def test_zippath_from_non_installed_posix(self): + """ + Test that when create venv from non-installed python, the zip path + value is as expected. + """ + rmtree(self.env_dir) + # First try to create a non-installed python. It's not a real full + # functional non-installed python, but enough for this test. + platlibdir = sys.platlibdir + non_installed_dir = os.path.realpath(tempfile.mkdtemp()) + self.addCleanup(rmtree, non_installed_dir) + bindir = os.path.join(non_installed_dir, self.bindir) + os.mkdir(bindir) + shutil.copy2(sys.executable, bindir) + libdir = os.path.join(non_installed_dir, platlibdir, self.lib[1]) + os.makedirs(libdir) + landmark = os.path.join(libdir, "os.py") + stdlib_zip = "python%d%d.zip" % sys.version_info[:2] + zip_landmark = os.path.join(non_installed_dir, + platlibdir, + stdlib_zip) + additional_pythonpath_for_non_installed = [] + # Copy stdlib files to the non-installed python so venv can + # correctly calculate the prefix. + for eachpath in sys.path: + if eachpath.endswith(".zip"): + if os.path.isfile(eachpath): + shutil.copyfile( + eachpath, + os.path.join(non_installed_dir, platlibdir)) + elif os.path.isfile(os.path.join(eachpath, "os.py")): + for name in os.listdir(eachpath): + if name == "site-packages": + continue + fn = os.path.join(eachpath, name) + if os.path.isfile(fn): + shutil.copy(fn, libdir) + elif os.path.isdir(fn): + shutil.copytree(fn, os.path.join(libdir, name)) + else: + additional_pythonpath_for_non_installed.append( + eachpath) + cmd = [os.path.join(non_installed_dir, self.bindir, self.exe), + "-m", + "venv", + "--without-pip", + self.env_dir] + # Our fake non-installed python is not fully functional because + # it cannot find the extensions. Set PYTHONPATH so it can run the + # venv module correctly. + pythonpath = os.pathsep.join( + additional_pythonpath_for_non_installed) + # For python built with shared enabled. We need to set + # LD_LIBRARY_PATH so the non-installed python can find and link + # libpython.so + ld_library_path = sysconfig.get_config_var("LIBDIR") + if not ld_library_path or sysconfig.is_python_build(): + ld_library_path = os.path.abspath(os.path.dirname(sys.executable)) + if sys.platform == 'darwin': + ld_library_path_env = "DYLD_LIBRARY_PATH" + else: + ld_library_path_env = "LD_LIBRARY_PATH" + subprocess.check_call(cmd, + env={"PYTHONPATH": pythonpath, + ld_library_path_env: ld_library_path}) + envpy = os.path.join(self.env_dir, self.bindir, self.exe) + # Now check the venv created from the non-installed python has + # correct zip path in pythonpath. + cmd = [envpy, '-S', '-c', 'import sys; print(sys.path)'] + out, err = check_output(cmd) + self.assertTrue(zip_landmark.encode() in out) + @requireVenvCreate class EnsurePipTest(BaseTest): """Test venv module installation of pip.""" @@ -673,8 +747,8 @@ def nicer_error(self): try: yield except subprocess.CalledProcessError as exc: - out = exc.output.decode(errors="replace") - err = exc.stderr.decode(errors="replace") + out = (exc.output or b'').decode(errors="replace") + err = (exc.stderr or b'').decode(errors="replace") self.fail( f"{exc}\n\n" f"**Subprocess Output**\n{out}\n\n" diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 0f960b82bfaebc..61a644406a6c20 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -489,7 +489,14 @@ def test_warn_explicit_non_ascii_filename(self): module=self.module) as w: self.module.resetwarnings() self.module.filterwarnings("always", category=UserWarning) - for filename in ("nonascii\xe9\u20ac", "surrogate\udc80"): + filenames = ["nonascii\xe9\u20ac"] + if not support.is_emscripten: + # JavaScript does not like surrogates. + # Invalid UTF-8 leading byte 0x80 encountered when + # deserializing a UTF-8 string in wasm memory to a JS + # string! + filenames.append("surrogate\udc80") + for filename in filenames: try: os.fsencode(filename) except UnicodeEncodeError: diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 3a9573d6e70559..7c5920797d2538 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -597,7 +597,7 @@ class C(object): # deallocation of c2. del c2 - def test_callback_in_cycle_1(self): + def test_callback_in_cycle(self): import gc class J(object): @@ -637,40 +637,11 @@ def acallback(self, ignore): del I, J, II gc.collect() - def test_callback_in_cycle_2(self): + def test_callback_reachable_one_way(self): import gc - # This is just like test_callback_in_cycle_1, except that II is an - # old-style class. The symptom is different then: an instance of an - # old-style class looks in its own __dict__ first. 'J' happens to - # get cleared from I.__dict__ before 'wr', and 'J' was never in II's - # __dict__, so the attribute isn't found. The difference is that - # the old-style II doesn't have a NULL __mro__ (it doesn't have any - # __mro__), so no segfault occurs. Instead it got: - # test_callback_in_cycle_2 (__main__.ReferencesTestCase) ... - # Exception exceptions.AttributeError: - # "II instance has no attribute 'J'" in > ignored - - class J(object): - pass - - class II: - def acallback(self, ignore): - self.J - - I = II() - I.J = J - I.wr = weakref.ref(J, I.acallback) - - del I, J, II - gc.collect() - - def test_callback_in_cycle_3(self): - import gc - - # This one broke the first patch that fixed the last two. In this - # case, the objects reachable from the callback aren't also reachable + # This one broke the first patch that fixed the previous test. In this case, + # the objects reachable from the callback aren't also reachable # from the object (c1) *triggering* the callback: you can get to # c1 from c2, but not vice-versa. The result was that c2's __dict__ # got tp_clear'ed by the time the c2.cb callback got invoked. @@ -690,10 +661,10 @@ def cb(self, ignore): del c1, c2 gc.collect() - def test_callback_in_cycle_4(self): + def test_callback_different_classes(self): import gc - # Like test_callback_in_cycle_3, except c2 and c1 have different + # Like test_callback_reachable_one_way, except c2 and c1 have different # classes. c2's class (C) isn't reachable from c1 then, so protecting # objects reachable from the dying object (c1) isn't enough to stop # c2's class (C) from getting tp_clear'ed before c2.cb is invoked. diff --git a/Lib/test/typinganndata/__init__.py b/Lib/test/typinganndata/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/typinganndata/ann_module9.py b/Lib/test/typinganndata/ann_module9.py new file mode 100644 index 00000000000000..952217393e1ff7 --- /dev/null +++ b/Lib/test/typinganndata/ann_module9.py @@ -0,0 +1,14 @@ +# Test ``inspect.formatannotation`` +# https://github.com/python/cpython/issues/96073 + +from typing import Union, List + +ann = Union[List[str], int] + +# mock typing._type_repr behaviour +class A: ... + +A.__module__ = 'testModule.typing' +A.__qualname__ = 'A' + +ann1 = Union[List[A], int] diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 296320235afddb..a8e7bf490ad463 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -2619,7 +2619,7 @@ def __init__(self, master, widgetName, cnf={}, kw={}, extra=()): if kw: cnf = _cnfmerge((cnf, kw)) self.widgetName = widgetName - BaseWidget._setup(self, master, cnf) + self._setup(master, cnf) if self._tclCommands is None: self._tclCommands = [] classes = [(k, v) for k, v in cnf.items() if isinstance(k, type)] @@ -3038,6 +3038,8 @@ def type(self, tagOrId): return self.tk.call(self._w, 'type', tagOrId) or None +_checkbutton_count = 0 + class Checkbutton(Widget): """Checkbutton widget which is either in on- or off-state.""" @@ -3053,6 +3055,14 @@ def __init__(self, master=None, cnf={}, **kw): underline, variable, width, wraplength.""" Widget.__init__(self, master, 'checkbutton', cnf, kw) + def _setup(self, master, cnf): + if not cnf.get('name'): + global _checkbutton_count + name = self.__class__.__name__.lower() + _checkbutton_count += 1 + cnf['name'] = f'!{name}{_checkbutton_count}' + super()._setup(master, cnf) + def deselect(self): """Put the button in off-state.""" self.tk.call(self._w, 'deselect') @@ -3638,7 +3648,7 @@ def count(self, index1, index2, *args): # new in Tk 8.5 "lines", "xpixels" and "ypixels". There is an additional possible option "update", which if given then all subsequent options ensure that any possible out of date information is recalculated.""" - args = ['-%s' % arg for arg in args if not arg.startswith('-')] + args = ['-%s' % arg for arg in args] args += [index1, index2] res = self.tk.call(self._w, 'count', *args) or None if res is not None and len(args) <= 3: diff --git a/Lib/tkinter/dialog.py b/Lib/tkinter/dialog.py index 8ae214011727cc..36ae6c277cb66a 100644 --- a/Lib/tkinter/dialog.py +++ b/Lib/tkinter/dialog.py @@ -11,7 +11,7 @@ class Dialog(Widget): def __init__(self, master=None, cnf={}, **kw): cnf = _cnfmerge((cnf, kw)) self.widgetName = '__dialog__' - Widget._setup(self, master, cnf) + self._setup(master, cnf) self.num = self.tk.getint( self.tk.call( 'tk_dialog', self._w, diff --git a/Lib/tkinter/test/test_tkinter/test_text.py b/Lib/tkinter/test/test_tkinter/test_text.py index 482f150df559fc..ea557586c7708b 100644 --- a/Lib/tkinter/test/test_tkinter/test_text.py +++ b/Lib/tkinter/test/test_tkinter/test_text.py @@ -40,6 +40,58 @@ def test_search(self): self.assertEqual(text.search('-test', '1.0', 'end'), '1.2') self.assertEqual(text.search('test', '1.0', 'end'), '1.3') + def test_count(self): + # XXX Some assertions do not check against the intended result, + # but instead check the current result to prevent regression. + text = self.text + text.insert('1.0', + 'Lorem ipsum dolor sit amet,\n' + 'consectetur adipiscing elit,\n' + 'sed do eiusmod tempor incididunt\n' + 'ut labore et dolore magna aliqua.') + + options = ('chars', 'indices', 'lines', + 'displaychars', 'displayindices', 'displaylines', + 'xpixels', 'ypixels') + if self.wantobjects: + self.assertEqual(len(text.count('1.0', 'end', *options)), 8) + else: + text.count('1.0', 'end', *options) + self.assertEqual(text.count('1.0', 'end', 'chars', 'lines'), (124, 4) + if self.wantobjects else '124 4') + self.assertEqual(text.count('1.3', '4.5', 'chars', 'lines'), (92, 3) + if self.wantobjects else '92 3') + self.assertEqual(text.count('4.5', '1.3', 'chars', 'lines'), (-92, -3) + if self.wantobjects else '-92 -3') + self.assertEqual(text.count('1.3', '1.3', 'chars', 'lines'), (0, 0) + if self.wantobjects else '0 0') + self.assertEqual(text.count('1.0', 'end', 'lines'), (4,) + if self.wantobjects else ('4',)) + self.assertEqual(text.count('end', '1.0', 'lines'), (-4,) + if self.wantobjects else ('-4',)) + self.assertEqual(text.count('1.3', '1.5', 'lines'), None + if self.wantobjects else ('0',)) + self.assertEqual(text.count('1.3', '1.3', 'lines'), None + if self.wantobjects else ('0',)) + self.assertEqual(text.count('1.0', 'end'), (124,) # 'indices' by default + if self.wantobjects else ('124',)) + self.assertRaises(tkinter.TclError, text.count, '1.0', 'end', 'spam') + self.assertRaises(tkinter.TclError, text.count, '1.0', 'end', '-lines') + + self.assertIsInstance(text.count('1.3', '1.5', 'ypixels'), tuple) + self.assertIsInstance(text.count('1.3', '1.5', 'update', 'ypixels'), int + if self.wantobjects else str) + self.assertEqual(text.count('1.3', '1.3', 'update', 'ypixels'), None + if self.wantobjects else '0') + self.assertEqual(text.count('1.3', '1.5', 'update', 'indices'), 2 + if self.wantobjects else '2') + self.assertEqual(text.count('1.3', '1.3', 'update', 'indices'), None + if self.wantobjects else '0') + self.assertEqual(text.count('1.3', '1.5', 'update'), (2,) + if self.wantobjects else ('2',)) + self.assertEqual(text.count('1.3', '1.3', 'update'), None + if self.wantobjects else ('0',)) + if __name__ == "__main__": unittest.main() diff --git a/Lib/tkinter/test/test_tkinter/test_widgets.py b/Lib/tkinter/test/test_tkinter/test_widgets.py index fe8ecfeb326554..da321a1daedb54 100644 --- a/Lib/tkinter/test/test_tkinter/test_widgets.py +++ b/Lib/tkinter/test/test_tkinter/test_widgets.py @@ -212,6 +212,32 @@ def test_configure_onvalue(self): widget = self.create() self.checkParams(widget, 'onvalue', 1, 2.3, '', 'any string') + def test_unique_variables(self): + frames = [] + buttons = [] + for i in range(2): + f = tkinter.Frame(self.root) + f.pack() + frames.append(f) + for j in 'AB': + b = tkinter.Checkbutton(f, text=j) + b.pack() + buttons.append(b) + variables = [str(b['variable']) for b in buttons] + self.assertEqual(len(set(variables)), 4, variables) + + def test_same_name(self): + f1 = tkinter.Frame(self.root) + f2 = tkinter.Frame(self.root) + b1 = tkinter.Checkbutton(f1, name='test', text='Test1') + b2 = tkinter.Checkbutton(f2, name='test', text='Test2') + + v = tkinter.IntVar(self.root, name='test') + b1.select() + self.assertEqual(v.get(), 1) + b2.deselect() + self.assertEqual(v.get(), 0) + @add_standard_options(StandardOptionsTests) class RadiobuttonTest(AbstractLabelTest, unittest.TestCase): @@ -733,6 +759,164 @@ def test_configure_yscrollincrement(self): self.checkPixelsParam(widget, 'yscrollincrement', 10, 0, 11.2, 13.6, -10, '0.1i') + def _test_option_joinstyle(self, c, factory): + for joinstyle in 'bevel', 'miter', 'round': + i = factory(joinstyle=joinstyle) + self.assertEqual(c.itemcget(i, 'joinstyle'), joinstyle) + self.assertRaises(TclError, factory, joinstyle='spam') + + def _test_option_smooth(self, c, factory): + for smooth in 1, True, '1', 'true', 'yes', 'on': + i = factory(smooth=smooth) + self.assertEqual(c.itemcget(i, 'smooth'), 'true') + for smooth in 0, False, '0', 'false', 'no', 'off': + i = factory(smooth=smooth) + self.assertEqual(c.itemcget(i, 'smooth'), '0') + i = factory(smooth=True, splinestep=30) + self.assertEqual(c.itemcget(i, 'smooth'), 'true') + self.assertEqual(c.itemcget(i, 'splinestep'), '30') + i = factory(smooth='raw', splinestep=30) + self.assertEqual(c.itemcget(i, 'smooth'), 'raw') + self.assertEqual(c.itemcget(i, 'splinestep'), '30') + self.assertRaises(TclError, factory, smooth='spam') + + def test_create_rectangle(self): + c = self.create() + i1 = c.create_rectangle(20, 30, 60, 10) + self.assertEqual(c.coords(i1), [20.0, 10.0, 60.0, 30.0]) + self.assertEqual(c.bbox(i1), (19, 9, 61, 31)) + + i2 = c.create_rectangle([21, 31, 61, 11]) + self.assertEqual(c.coords(i2), [21.0, 11.0, 61.0, 31.0]) + self.assertEqual(c.bbox(i2), (20, 10, 62, 32)) + + i3 = c.create_rectangle((22, 32), (62, 12)) + self.assertEqual(c.coords(i3), [22.0, 12.0, 62.0, 32.0]) + self.assertEqual(c.bbox(i3), (21, 11, 63, 33)) + + i4 = c.create_rectangle([(23, 33), (63, 13)]) + self.assertEqual(c.coords(i4), [23.0, 13.0, 63.0, 33.0]) + self.assertEqual(c.bbox(i4), (22, 12, 64, 34)) + + self.assertRaises(TclError, c.create_rectangle, 20, 30, 60) + self.assertRaises(TclError, c.create_rectangle, [20, 30, 60]) + self.assertRaises(TclError, c.create_rectangle, 20, 30, 40, 50, 60, 10) + self.assertRaises(TclError, c.create_rectangle, [20, 30, 40, 50, 60, 10]) + self.assertRaises(TclError, c.create_rectangle, 20, 30) + self.assertRaises(TclError, c.create_rectangle, [20, 30]) + self.assertRaises(IndexError, c.create_rectangle) + self.assertRaises(IndexError, c.create_rectangle, []) + + def test_create_line(self): + c = self.create() + i1 = c.create_line(20, 30, 40, 50, 60, 10) + self.assertEqual(c.coords(i1), [20.0, 30.0, 40.0, 50.0, 60.0, 10.0]) + self.assertEqual(c.bbox(i1), (18, 8, 62, 52)) + self.assertEqual(c.itemcget(i1, 'arrow'), 'none') + self.assertEqual(c.itemcget(i1, 'arrowshape'), '8 10 3') + self.assertEqual(c.itemcget(i1, 'capstyle'), 'butt') + self.assertEqual(c.itemcget(i1, 'joinstyle'), 'round') + self.assertEqual(c.itemcget(i1, 'smooth'), '0') + self.assertEqual(c.itemcget(i1, 'splinestep'), '12') + + i2 = c.create_line([21, 31, 41, 51, 61, 11]) + self.assertEqual(c.coords(i2), [21.0, 31.0, 41.0, 51.0, 61.0, 11.0]) + self.assertEqual(c.bbox(i2), (19, 9, 63, 53)) + + i3 = c.create_line((22, 32), (42, 52), (62, 12)) + self.assertEqual(c.coords(i3), [22.0, 32.0, 42.0, 52.0, 62.0, 12.0]) + self.assertEqual(c.bbox(i3), (20, 10, 64, 54)) + + i4 = c.create_line([(23, 33), (43, 53), (63, 13)]) + self.assertEqual(c.coords(i4), [23.0, 33.0, 43.0, 53.0, 63.0, 13.0]) + self.assertEqual(c.bbox(i4), (21, 11, 65, 55)) + + self.assertRaises(TclError, c.create_line, 20, 30, 60) + self.assertRaises(TclError, c.create_line, [20, 30, 60]) + self.assertRaises(TclError, c.create_line, 20, 30) + self.assertRaises(TclError, c.create_line, [20, 30]) + self.assertRaises(IndexError, c.create_line) + self.assertRaises(IndexError, c.create_line, []) + + for arrow in 'none', 'first', 'last', 'both': + i = c.create_line(20, 30, 60, 10, arrow=arrow) + self.assertEqual(c.itemcget(i, 'arrow'), arrow) + i = c.create_line(20, 30, 60, 10, arrow='first', arrowshape=[10, 15, 5]) + self.assertEqual(c.itemcget(i, 'arrowshape'), '10 15 5') + self.assertRaises(TclError, c.create_line, 20, 30, 60, 10, arrow='spam') + + for capstyle in 'butt', 'projecting', 'round': + i = c.create_line(20, 30, 60, 10, capstyle=capstyle) + self.assertEqual(c.itemcget(i, 'capstyle'), capstyle) + self.assertRaises(TclError, c.create_line, 20, 30, 60, 10, capstyle='spam') + + self._test_option_joinstyle(c, + lambda **kwargs: c.create_line(20, 30, 40, 50, 60, 10, **kwargs)) + self._test_option_smooth(c, + lambda **kwargs: c.create_line(20, 30, 60, 10, **kwargs)) + + def test_create_polygon(self): + c = self.create() + i1 = c.create_polygon(20, 30, 40, 50, 60, 10) + self.assertEqual(c.coords(i1), [20.0, 30.0, 40.0, 50.0, 60.0, 10.0]) + self.assertEqual(c.bbox(i1), (19, 9, 61, 51)) + self.assertEqual(c.itemcget(i1, 'joinstyle'), 'round') + self.assertEqual(c.itemcget(i1, 'smooth'), '0') + self.assertEqual(c.itemcget(i1, 'splinestep'), '12') + + i2 = c.create_polygon([21, 31, 41, 51, 61, 11]) + self.assertEqual(c.coords(i2), [21.0, 31.0, 41.0, 51.0, 61.0, 11.0]) + self.assertEqual(c.bbox(i2), (20, 10, 62, 52)) + + i3 = c.create_polygon((22, 32), (42, 52), (62, 12)) + self.assertEqual(c.coords(i3), [22.0, 32.0, 42.0, 52.0, 62.0, 12.0]) + self.assertEqual(c.bbox(i3), (21, 11, 63, 53)) + + i4 = c.create_polygon([(23, 33), (43, 53), (63, 13)]) + self.assertEqual(c.coords(i4), [23.0, 33.0, 43.0, 53.0, 63.0, 13.0]) + self.assertEqual(c.bbox(i4), (22, 12, 64, 54)) + + self.assertRaises(TclError, c.create_polygon, 20, 30, 60) + self.assertRaises(TclError, c.create_polygon, [20, 30, 60]) + self.assertRaises(IndexError, c.create_polygon) + self.assertRaises(IndexError, c.create_polygon, []) + + self._test_option_joinstyle(c, + lambda **kwargs: c.create_polygon(20, 30, 40, 50, 60, 10, **kwargs)) + self._test_option_smooth(c, + lambda **kwargs: c.create_polygon(20, 30, 40, 50, 60, 10, **kwargs)) + + def test_coords(self): + c = self.create() + i = c.create_line(20, 30, 40, 50, 60, 10, tags='x') + self.assertEqual(c.coords(i), [20.0, 30.0, 40.0, 50.0, 60.0, 10.0]) + self.assertEqual(c.coords('x'), [20.0, 30.0, 40.0, 50.0, 60.0, 10.0]) + self.assertEqual(c.bbox(i), (18, 8, 62, 52)) + + c.coords(i, 50, 60, 70, 80, 90, 40) + self.assertEqual(c.coords(i), [50.0, 60.0, 70.0, 80.0, 90.0, 40.0]) + self.assertEqual(c.bbox(i), (48, 38, 92, 82)) + + c.coords(i, [21, 31, 41, 51, 61, 11]) + self.assertEqual(c.coords(i), [21.0, 31.0, 41.0, 51.0, 61.0, 11.0]) + + c.coords(i, 20, 30, 60, 10) + self.assertEqual(c.coords(i), [20.0, 30.0, 60.0, 10.0]) + self.assertEqual(c.bbox(i), (18, 8, 62, 32)) + + self.assertRaises(TclError, c.coords, i, 20, 30, 60) + self.assertRaises(TclError, c.coords, i, [20, 30, 60]) + self.assertRaises(TclError, c.coords, i, 20, 30) + self.assertRaises(TclError, c.coords, i, [20, 30]) + + c.coords(i, '20', '30c', '60i', '10p') + coords = c.coords(i) + self.assertIsInstance(coords, list) + self.assertEqual(len(coords), 4) + self.assertEqual(coords[0], 20) + for i in range(4): + self.assertIsInstance(coords[i], float) + @requires_tcl(8, 6) def test_moveto(self): widget = self.create() diff --git a/Lib/tkinter/test/test_ttk/test_widgets.py b/Lib/tkinter/test/test_ttk/test_widgets.py index c14c321ca2687d..96d2afcf90ea81 100644 --- a/Lib/tkinter/test/test_ttk/test_widgets.py +++ b/Lib/tkinter/test/test_ttk/test_widgets.py @@ -275,6 +275,21 @@ def cb_test(): self.assertEqual(cbtn['offvalue'], cbtn.tk.globalgetvar(cbtn['variable'])) + def test_unique_variables(self): + frames = [] + buttons = [] + for i in range(2): + f = ttk.Frame(self.root) + f.pack() + frames.append(f) + for j in 'AB': + b = ttk.Checkbutton(f, text=j) + b.pack() + buttons.append(b) + variables = [str(b['variable']) for b in buttons] + print(variables) + self.assertEqual(len(set(variables)), 4, variables) + @add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) class EntryTest(AbstractWidgetTest, unittest.TestCase): diff --git a/Lib/tkinter/tix.py b/Lib/tkinter/tix.py index 44ecae1a326831..ce218265d4a824 100644 --- a/Lib/tkinter/tix.py +++ b/Lib/tkinter/tix.py @@ -310,7 +310,7 @@ def __init__ (self, master=None, widgetName=None, del cnf[k] self.widgetName = widgetName - Widget._setup(self, master, cnf) + self._setup(master, cnf) # If widgetName is None, this is a dummy creation call where the # corresponding Tk widget has already been created by Tix diff --git a/Lib/trace.py b/Lib/trace.py index 2cf3643878d4b8..213e46517d683d 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -172,7 +172,7 @@ def __init__(self, counts=None, calledfuncs=None, infile=None, try: with open(self.infile, 'rb') as f: counts, calledfuncs, callers = pickle.load(f) - self.update(self.__class__(counts, calledfuncs, callers)) + self.update(self.__class__(counts, calledfuncs, callers=callers)) except (OSError, EOFError, ValueError) as err: print(("Skipping counts file %r: %s" % (self.infile, err)), file=sys.stderr) diff --git a/Lib/traceback.py b/Lib/traceback.py index 55f8080044053e..0182bb7525bbf1 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -475,32 +475,32 @@ def format_frame_summary(self, frame_summary): frame_summary.colno is not None and frame_summary.end_colno is not None ): - colno = _byte_offset_to_character_offset( - frame_summary._original_line, frame_summary.colno) - end_colno = _byte_offset_to_character_offset( - frame_summary._original_line, frame_summary.end_colno) + start_offset = _byte_offset_to_character_offset( + frame_summary._original_line, frame_summary.colno) + 1 + end_offset = _byte_offset_to_character_offset( + frame_summary._original_line, frame_summary.end_colno) + 1 anchors = None if frame_summary.lineno == frame_summary.end_lineno: with suppress(Exception): anchors = _extract_caret_anchors_from_line_segment( - frame_summary._original_line[colno - 1:end_colno - 1] + frame_summary._original_line[start_offset - 1:end_offset - 1] ) else: - end_colno = stripped_characters + len(stripped_line) + end_offset = stripped_characters + len(stripped_line) # show indicators if primary char doesn't span the frame line - if end_colno - colno < len(stripped_line) or ( + if end_offset - start_offset < len(stripped_line) or ( anchors and anchors.right_start_offset - anchors.left_end_offset > 0): row.append(' ') - row.append(' ' * (colno - stripped_characters)) + row.append(' ' * (start_offset - stripped_characters)) if anchors: row.append(anchors.primary_char * (anchors.left_end_offset)) row.append(anchors.secondary_char * (anchors.right_start_offset - anchors.left_end_offset)) - row.append(anchors.primary_char * (end_colno - colno - anchors.right_start_offset)) + row.append(anchors.primary_char * (end_offset - start_offset - anchors.right_start_offset)) else: - row.append('^' * (end_colno - colno)) + row.append('^' * (end_offset - start_offset)) row.append('\n') @@ -560,10 +560,7 @@ def format(self): def _byte_offset_to_character_offset(str, offset): as_utf8 = str.encode('utf-8') - if offset > len(as_utf8): - offset = len(as_utf8) - - return len(as_utf8[:offset + 1].decode("utf-8")) + return len(as_utf8[:offset].decode("utf-8", errors="replace")) _Anchors = collections.namedtuple( @@ -588,12 +585,15 @@ def _extract_caret_anchors_from_line_segment(segment): if len(tree.body) != 1: return None + normalize = lambda offset: _byte_offset_to_character_offset(segment, offset) statement = tree.body[0] match statement: case ast.Expr(expr): match expr: case ast.BinOp(): - operator_str = segment[expr.left.end_col_offset:expr.right.col_offset] + operator_start = normalize(expr.left.end_col_offset) + operator_end = normalize(expr.right.col_offset) + operator_str = segment[operator_start:operator_end] operator_offset = len(operator_str) - len(operator_str.lstrip()) left_anchor = expr.left.end_col_offset + operator_offset @@ -603,9 +603,11 @@ def _extract_caret_anchors_from_line_segment(segment): and not operator_str[operator_offset + 1].isspace() ): right_anchor += 1 - return _Anchors(left_anchor, right_anchor) + return _Anchors(normalize(left_anchor), normalize(right_anchor)) case ast.Subscript(): - return _Anchors(expr.value.end_col_offset, expr.slice.end_col_offset + 1) + subscript_start = normalize(expr.value.end_col_offset) + subscript_end = normalize(expr.slice.end_col_offset + 1) + return _Anchors(subscript_start, subscript_end) return None diff --git a/Lib/turtle.py b/Lib/turtle.py index a8876e76bce40a..6abf9f7f65af0d 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -596,7 +596,6 @@ def _write(self, pos, txt, align, font, pencolor): item = self.cv.create_text(x-1, -y, text = txt, anchor = anchor[align], fill = pencolor, font = font) x0, y0, x1, y1 = self.cv.bbox(item) - self.cv.update() return item, x1-1 ## def _dot(self, pos, size, color): @@ -3419,6 +3418,7 @@ def _write(self, txt, align, font): """ item, end = self.screen._write(self._position, txt, align, font, self._pencolor) + self._update() self.items.append(item) if self.undobuffer: self.undobuffer.push(("wri", item)) diff --git a/Lib/turtledemo/clock.py b/Lib/turtledemo/clock.py index 62c8851606b34e..9f8585bd11e053 100755 --- a/Lib/turtledemo/clock.py +++ b/Lib/turtledemo/clock.py @@ -109,7 +109,6 @@ def tick(): writer.write(datum(t), align="center", font=("Courier", 14, "bold")) writer.forward(85) - tracer(True) second_hand.setheading(6*sekunde) # or here minute_hand.setheading(6*minute) hour_hand.setheading(30*stunde) diff --git a/Lib/typing.py b/Lib/typing.py index 354976caaaa007..9818237de43510 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -284,25 +284,6 @@ def _unpack_args(args): newargs.append(arg) return newargs -def _prepare_paramspec_params(cls, params): - """Prepares the parameters for a Generic containing ParamSpec - variables (internal helper). - """ - # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. - if (len(cls.__parameters__) == 1 - and params and not _is_param_expr(params[0])): - assert isinstance(cls.__parameters__[0], ParamSpec) - return (params,) - else: - _check_generic(cls, params, len(cls.__parameters__)) - _params = [] - # Convert lists to tuples to help other libraries cache the results. - for p, tvar in zip(params, cls.__parameters__): - if isinstance(tvar, ParamSpec) and isinstance(p, list): - p = tuple(p) - _params.append(p) - return tuple(_params) - def _deduplicate(params): # Weed out strict duplicates, preserving the first of each occurrence. all_params = set(params) @@ -493,7 +474,9 @@ def __instancecheck__(self, obj): return super().__instancecheck__(obj) def __repr__(self): - return "typing.Any" + if self is Any: + return "typing.Any" + return super().__repr__() # respect to subclasses class Any(metaclass=_AnyMeta): @@ -1224,7 +1207,18 @@ def __typing_subst__(self, arg): return arg def __typing_prepare_subst__(self, alias, args): - return _prepare_paramspec_params(alias, args) + params = alias.__parameters__ + i = params.index(self) + if i >= len(args): + raise TypeError(f"Too few arguments for {alias}") + # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. + if len(params) == 1 and not _is_param_expr(args[0]): + assert i == 0 + args = (args,) + # Convert lists to tuples to help other libraries cache the results. + elif isinstance(args[i], list): + args = (*args[:i], tuple(args[i]), *args[i+1:]) + return args def _is_dunder(attr): return attr.startswith('__') and attr.endswith('__') @@ -1423,6 +1417,10 @@ def _determine_new_args(self, args): new_args = [] for old_arg in self.__args__: + if isinstance(old_arg, type): + new_args.append(old_arg) + continue + substfunc = getattr(old_arg, '__typing_subst__', None) if substfunc: new_arg = substfunc(new_arg_by_param[old_arg]) @@ -1783,23 +1781,13 @@ def __class_getitem__(cls, params): if not isinstance(params, tuple): params = (params,) - if not params: - # We're only ok with `params` being empty if the class's only type - # parameter is a `TypeVarTuple` (which can contain zero types). - class_params = getattr(cls, "__parameters__", None) - only_class_parameter_is_typevartuple = ( - class_params is not None - and len(class_params) == 1 - and isinstance(class_params[0], TypeVarTuple) - ) - if not only_class_parameter_is_typevartuple: - raise TypeError( - f"Parameter list to {cls.__qualname__}[...] cannot be empty" - ) - params = tuple(_type_convert(p) for p in params) if cls in (Generic, Protocol): # Generic and Protocol can only be subscripted with unique type variables. + if not params: + raise TypeError( + f"Parameter list to {cls.__qualname__}[...] cannot be empty" + ) if not all(_is_typevar_like(p) for p in params): raise TypeError( f"Parameters to {cls.__name__}[...] must all be type variables " @@ -1809,13 +1797,20 @@ def __class_getitem__(cls, params): f"Parameters to {cls.__name__}[...] must all be unique") else: # Subscripting a regular Generic subclass. - if any(isinstance(t, ParamSpec) for t in cls.__parameters__): - params = _prepare_paramspec_params(cls, params) - elif not any(isinstance(p, TypeVarTuple) for p in cls.__parameters__): - # We only run this if there are no TypeVarTuples, because we - # don't check variadic generic arity at runtime (to reduce - # complexity of typing.py). - _check_generic(cls, params, len(cls.__parameters__)) + for param in cls.__parameters__: + prepare = getattr(param, '__typing_prepare_subst__', None) + if prepare is not None: + params = prepare(cls, params) + _check_generic(cls, params, len(cls.__parameters__)) + + new_args = [] + for param, new_arg in zip(cls.__parameters__, params): + if isinstance(param, TypeVarTuple): + new_args.extend(new_arg) + else: + new_args.append(new_arg) + params = tuple(new_args) + return _GenericAlias(cls, params, _paramspec_tvars=True) diff --git a/Lib/unittest/async_case.py b/Lib/unittest/async_case.py index 3457e92e5da298..bd2a471156065b 100644 --- a/Lib/unittest/async_case.py +++ b/Lib/unittest/async_case.py @@ -88,7 +88,7 @@ def _callSetUp(self): def _callTestMethod(self, method): if self._callMaybeAsync(method) is not None: - warnings.warn(f'It is deprecated to return a value!=None from a ' + warnings.warn(f'It is deprecated to return a value that is not None from a ' f'test case ({method})', DeprecationWarning, stacklevel=4) def _callTearDown(self): diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index ffc8f19ddd38d3..c4aa2d7721966e 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -384,11 +384,11 @@ class TestCase(object): # of difflib. See #11763. _diffThreshold = 2**16 - # Attribute used by TestSuite for classSetUp - - _classSetupFailed = False - - _class_cleanups = [] + def __init_subclass__(cls, *args, **kwargs): + # Attribute used by TestSuite for classSetUp + cls._classSetupFailed = False + cls._class_cleanups = [] + super().__init_subclass__(*args, **kwargs) def __init__(self, methodName='runTest'): """Create an instance of the class that will use the named test @@ -577,7 +577,7 @@ def _callSetUp(self): def _callTestMethod(self, method): if method() is not None: - warnings.warn(f'It is deprecated to return a value!=None from a ' + warnings.warn(f'It is deprecated to return a value that is not None from a ' f'test case ({method})', DeprecationWarning, stacklevel=3) def _callTearDown(self): diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index cd46fea5162aba..6720e5bc22dc56 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -35,6 +35,7 @@ from types import CodeType, ModuleType, MethodType from unittest.util import safe_repr from functools import wraps, partial +from threading import RLock class InvalidSpecError(Exception): @@ -402,6 +403,14 @@ def __init__(self, /, *args, **kwargs): class NonCallableMock(Base): """A non-callable version of `Mock`""" + # Store a mutex as a class attribute in order to protect concurrent access + # to mock attributes. Using a class attribute allows all NonCallableMock + # instances to share the mutex for simplicity. + # + # See https://github.com/python/cpython/issues/98624 for why this is + # necessary. + _lock = RLock() + def __new__(cls, /, *args, **kw): # every instance has its own class # so we can create magic methods on the @@ -644,35 +653,36 @@ def __getattr__(self, name): f"{name!r} is not a valid assertion. Use a spec " f"for the mock if {name!r} is meant to be an attribute.") - result = self._mock_children.get(name) - if result is _deleted: - raise AttributeError(name) - elif result is None: - wraps = None - if self._mock_wraps is not None: - # XXXX should we get the attribute without triggering code - # execution? - wraps = getattr(self._mock_wraps, name) - - result = self._get_child_mock( - parent=self, name=name, wraps=wraps, _new_name=name, - _new_parent=self - ) - self._mock_children[name] = result - - elif isinstance(result, _SpecState): - try: - result = create_autospec( - result.spec, result.spec_set, result.instance, - result.parent, result.name + with NonCallableMock._lock: + result = self._mock_children.get(name) + if result is _deleted: + raise AttributeError(name) + elif result is None: + wraps = None + if self._mock_wraps is not None: + # XXXX should we get the attribute without triggering code + # execution? + wraps = getattr(self._mock_wraps, name) + + result = self._get_child_mock( + parent=self, name=name, wraps=wraps, _new_name=name, + _new_parent=self ) - except InvalidSpecError: - target_name = self.__dict__['_mock_name'] or self - raise InvalidSpecError( - f'Cannot autospec attr {name!r} from target ' - f'{target_name!r} as it has already been mocked out. ' - f'[target={self!r}, attr={result.spec!r}]') - self._mock_children[name] = result + self._mock_children[name] = result + + elif isinstance(result, _SpecState): + try: + result = create_autospec( + result.spec, result.spec_set, result.instance, + result.parent, result.name + ) + except InvalidSpecError: + target_name = self.__dict__['_mock_name'] or self + raise InvalidSpecError( + f'Cannot autospec attr {name!r} from target ' + f'{target_name!r} as it has already been mocked out. ' + f'[target={self!r}, attr={result.spec!r}]') + self._mock_children[name] = result return result @@ -1799,6 +1809,12 @@ def __init__(self, in_dict, values=(), clear=False, **kwargs): def __call__(self, f): if isinstance(f, type): return self.decorate_class(f) + if inspect.iscoroutinefunction(f): + return self.decorate_async_callable(f) + return self.decorate_callable(f) + + + def decorate_callable(self, f): @wraps(f) def _inner(*args, **kw): self._patch_dict() @@ -1810,6 +1826,18 @@ def _inner(*args, **kw): return _inner + def decorate_async_callable(self, f): + @wraps(f) + async def _inner(*args, **kw): + self._patch_dict() + try: + return await f(*args, **kw) + finally: + self._unpatch_dict() + + return _inner + + def decorate_class(self, klass): for attr in dir(klass): attr_value = getattr(klass, attr) diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 3da7005e603f4a..5ca4c23238b419 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -196,6 +196,7 @@ def _clean_tracebacks(self, exctype, value, tb, test): ret = None first = True excs = [(exctype, value, tb)] + seen = {id(value)} # Detect loops in chained exceptions. while excs: (exctype, value, tb) = excs.pop() # Skip test runner traceback levels @@ -214,8 +215,9 @@ def _clean_tracebacks(self, exctype, value, tb, test): if value is not None: for c in (value.__cause__, value.__context__): - if c is not None: + if c is not None and id(c) not in seen: excs.append((type(c), c, c.__traceback__)) + seen.add(id(c)) return ret def _is_relevant_tb_level(self, tb): diff --git a/Lib/unittest/test/test_async_case.py b/Lib/unittest/test/test_async_case.py index d7d4dc91316c6c..a465103b59b6ec 100644 --- a/Lib/unittest/test/test_async_case.py +++ b/Lib/unittest/test/test_async_case.py @@ -49,69 +49,87 @@ def setUp(self): self.addCleanup(support.gc_collect) def test_full_cycle(self): + expected = ['setUp', + 'asyncSetUp', + 'test', + 'asyncTearDown', + 'tearDown', + 'cleanup6', + 'cleanup5', + 'cleanup4', + 'cleanup3', + 'cleanup2', + 'cleanup1'] class Test(unittest.IsolatedAsyncioTestCase): def setUp(self): self.assertEqual(events, []) events.append('setUp') VAR.set(VAR.get() + ('setUp',)) + self.addCleanup(self.on_cleanup1) + self.addAsyncCleanup(self.on_cleanup2) async def asyncSetUp(self): - self.assertEqual(events, ['setUp']) + self.assertEqual(events, expected[:1]) events.append('asyncSetUp') VAR.set(VAR.get() + ('asyncSetUp',)) - self.addAsyncCleanup(self.on_cleanup1) + self.addCleanup(self.on_cleanup3) + self.addAsyncCleanup(self.on_cleanup4) async def test_func(self): - self.assertEqual(events, ['setUp', - 'asyncSetUp']) + self.assertEqual(events, expected[:2]) events.append('test') VAR.set(VAR.get() + ('test',)) - self.addAsyncCleanup(self.on_cleanup2) + self.addCleanup(self.on_cleanup5) + self.addAsyncCleanup(self.on_cleanup6) async def asyncTearDown(self): - self.assertEqual(events, ['setUp', - 'asyncSetUp', - 'test']) + self.assertEqual(events, expected[:3]) VAR.set(VAR.get() + ('asyncTearDown',)) events.append('asyncTearDown') def tearDown(self): - self.assertEqual(events, ['setUp', - 'asyncSetUp', - 'test', - 'asyncTearDown']) + self.assertEqual(events, expected[:4]) events.append('tearDown') VAR.set(VAR.get() + ('tearDown',)) - async def on_cleanup1(self): - self.assertEqual(events, ['setUp', - 'asyncSetUp', - 'test', - 'asyncTearDown', - 'tearDown', - 'cleanup2']) + def on_cleanup1(self): + self.assertEqual(events, expected[:10]) events.append('cleanup1') VAR.set(VAR.get() + ('cleanup1',)) nonlocal cvar cvar = VAR.get() async def on_cleanup2(self): - self.assertEqual(events, ['setUp', - 'asyncSetUp', - 'test', - 'asyncTearDown', - 'tearDown']) + self.assertEqual(events, expected[:9]) events.append('cleanup2') VAR.set(VAR.get() + ('cleanup2',)) + def on_cleanup3(self): + self.assertEqual(events, expected[:8]) + events.append('cleanup3') + VAR.set(VAR.get() + ('cleanup3',)) + + async def on_cleanup4(self): + self.assertEqual(events, expected[:7]) + events.append('cleanup4') + VAR.set(VAR.get() + ('cleanup4',)) + + def on_cleanup5(self): + self.assertEqual(events, expected[:6]) + events.append('cleanup5') + VAR.set(VAR.get() + ('cleanup5',)) + + async def on_cleanup6(self): + self.assertEqual(events, expected[:5]) + events.append('cleanup6') + VAR.set(VAR.get() + ('cleanup6',)) + events = [] cvar = () test = Test("test_func") result = test.run() self.assertEqual(result.errors, []) self.assertEqual(result.failures, []) - expected = ['setUp', 'asyncSetUp', 'test', - 'asyncTearDown', 'tearDown', 'cleanup2', 'cleanup1'] self.assertEqual(events, expected) self.assertEqual(cvar, tuple(expected)) @@ -277,25 +295,36 @@ async def on_cleanup2(self): self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup2', 'cleanup1']) def test_deprecation_of_return_val_from_test(self): - # Issue 41322 - deprecate return of value!=None from a test + # Issue 41322 - deprecate return of value that is not None from a test + class Nothing: + def __eq__(self, o): + return o is None class Test(unittest.IsolatedAsyncioTestCase): async def test1(self): return 1 async def test2(self): yield 1 + async def test3(self): + return Nothing() with self.assertWarns(DeprecationWarning) as w: Test('test1').run() - self.assertIn('It is deprecated to return a value!=None', str(w.warning)) + self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test1', str(w.warning)) self.assertEqual(w.filename, __file__) with self.assertWarns(DeprecationWarning) as w: Test('test2').run() - self.assertIn('It is deprecated to return a value!=None', str(w.warning)) + self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test2', str(w.warning)) self.assertEqual(w.filename, __file__) + with self.assertWarns(DeprecationWarning) as w: + Test('test3').run() + self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) + self.assertIn('test3', str(w.warning)) + self.assertEqual(w.filename, __file__) + def test_cleanups_interleave_order(self): events = [] diff --git a/Lib/unittest/test/test_case.py b/Lib/unittest/test/test_case.py index 374a255255566f..78303b359f5e10 100644 --- a/Lib/unittest/test/test_case.py +++ b/Lib/unittest/test/test_case.py @@ -307,25 +307,36 @@ def test(self): Foo('test').run() def test_deprecation_of_return_val_from_test(self): - # Issue 41322 - deprecate return of value!=None from a test + # Issue 41322 - deprecate return of value that is not None from a test + class Nothing: + def __eq__(self, o): + return o is None class Foo(unittest.TestCase): def test1(self): return 1 def test2(self): yield 1 + def test3(self): + return Nothing() with self.assertWarns(DeprecationWarning) as w: Foo('test1').run() - self.assertIn('It is deprecated to return a value!=None', str(w.warning)) + self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test1', str(w.warning)) self.assertEqual(w.filename, __file__) with self.assertWarns(DeprecationWarning) as w: Foo('test2').run() - self.assertIn('It is deprecated to return a value!=None', str(w.warning)) + self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test2', str(w.warning)) self.assertEqual(w.filename, __file__) + with self.assertWarns(DeprecationWarning) as w: + Foo('test3').run() + self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) + self.assertIn('test3', str(w.warning)) + self.assertEqual(w.filename, __file__) + def _check_call_order__subtests(self, result, events, expected_events): class Foo(Test.LoggingTestCase): def test(self): diff --git a/Lib/unittest/test/test_result.py b/Lib/unittest/test/test_result.py index b0cc3d867d6e57..9320b0a44b51f9 100644 --- a/Lib/unittest/test/test_result.py +++ b/Lib/unittest/test/test_result.py @@ -275,6 +275,62 @@ def get_exc_info(): self.assertEqual(len(dropped), 1) self.assertIn("raise self.failureException(msg)", dropped[0]) + def test_addFailure_filter_traceback_frames_chained_exception_self_loop(self): + class Foo(unittest.TestCase): + def test_1(self): + pass + + def get_exc_info(): + try: + loop = Exception("Loop") + loop.__cause__ = loop + loop.__context__ = loop + raise loop + except: + return sys.exc_info() + + exc_info_tuple = get_exc_info() + + test = Foo('test_1') + result = unittest.TestResult() + result.startTest(test) + result.addFailure(test, exc_info_tuple) + result.stopTest(test) + + formatted_exc = result.failures[0][1] + self.assertEqual(formatted_exc.count("Exception: Loop\n"), 1) + + def test_addFailure_filter_traceback_frames_chained_exception_cycle(self): + class Foo(unittest.TestCase): + def test_1(self): + pass + + def get_exc_info(): + try: + # Create two directionally opposed cycles + # __cause__ in one direction, __context__ in the other + A, B, C = Exception("A"), Exception("B"), Exception("C") + edges = [(C, B), (B, A), (A, C)] + for ex1, ex2 in edges: + ex1.__cause__ = ex2 + ex2.__context__ = ex1 + raise C + except: + return sys.exc_info() + + exc_info_tuple = get_exc_info() + + test = Foo('test_1') + result = unittest.TestResult() + result.startTest(test) + result.addFailure(test, exc_info_tuple) + result.stopTest(test) + + formatted_exc = result.failures[0][1] + self.assertEqual(formatted_exc.count("Exception: A\n"), 1) + self.assertEqual(formatted_exc.count("Exception: B\n"), 1) + self.assertEqual(formatted_exc.count("Exception: C\n"), 1) + # "addError(test, err)" # ... # "Called when the test case test raises an unexpected exception err diff --git a/Lib/unittest/test/test_runner.py b/Lib/unittest/test/test_runner.py index d3488b40e82bde..e02b2dadc8fdbb 100644 --- a/Lib/unittest/test/test_runner.py +++ b/Lib/unittest/test/test_runner.py @@ -134,11 +134,13 @@ def testCleanupInRun(self): class TestableTest(unittest.TestCase): def setUp(self): ordering.append('setUp') + test.addCleanup(cleanup2) if blowUp: raise Exception('foo') def testNothing(self): ordering.append('test') + test.addCleanup(cleanup3) def tearDown(self): ordering.append('tearDown') @@ -149,8 +151,9 @@ def cleanup1(): ordering.append('cleanup1') def cleanup2(): ordering.append('cleanup2') + def cleanup3(): + ordering.append('cleanup3') test.addCleanup(cleanup1) - test.addCleanup(cleanup2) def success(some_test): self.assertEqual(some_test, test) @@ -160,7 +163,7 @@ def success(some_test): result.addSuccess = success test.run(result) - self.assertEqual(ordering, ['setUp', 'test', 'tearDown', + self.assertEqual(ordering, ['setUp', 'test', 'tearDown', 'cleanup3', 'cleanup2', 'cleanup1', 'success']) blowUp = True @@ -168,7 +171,7 @@ def success(some_test): test = TestableTest('testNothing') test.addCleanup(cleanup1) test.run(result) - self.assertEqual(ordering, ['setUp', 'cleanup1']) + self.assertEqual(ordering, ['setUp', 'cleanup2', 'cleanup1']) def testTestCaseDebugExecutesCleanups(self): ordering = [] @@ -180,9 +183,11 @@ def setUp(self): def testNothing(self): ordering.append('test') + self.addCleanup(cleanup3) def tearDown(self): ordering.append('tearDown') + test.addCleanup(cleanup4) test = TestableTest('testNothing') @@ -191,9 +196,14 @@ def cleanup1(): test.addCleanup(cleanup2) def cleanup2(): ordering.append('cleanup2') + def cleanup3(): + ordering.append('cleanup3') + def cleanup4(): + ordering.append('cleanup4') test.debug() - self.assertEqual(ordering, ['setUp', 'test', 'tearDown', 'cleanup1', 'cleanup2']) + self.assertEqual(ordering, ['setUp', 'test', 'tearDown', 'cleanup4', + 'cleanup3', 'cleanup1', 'cleanup2']) def test_enterContext(self): @@ -352,13 +362,14 @@ def testNothing(self): ordering.append('test') @classmethod def tearDownClass(cls): + ordering.append('tearDownClass') raise Exception('TearDownClassExc') suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) with self.assertRaises(Exception) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownClassExc') - self.assertEqual(ordering, ['setUpClass', 'test']) + self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass']) self.assertTrue(TestableTest._class_cleanups) TestableTest._class_cleanups.clear() @@ -368,7 +379,7 @@ def tearDownClass(cls): with self.assertRaises(Exception) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownClassExc') - self.assertEqual(ordering, ['setUpClass', 'test']) + self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass']) self.assertTrue(TestableTest._class_cleanups) TestableTest._class_cleanups.clear() @@ -536,6 +547,33 @@ def testNothing(self): self.assertEqual(TestableTest._class_cleanups, []) + def test_run_nested_test(self): + ordering = [] + + class InnerTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + ordering.append('inner setup') + cls.addClassCleanup(ordering.append, 'inner cleanup') + def test(self): + ordering.append('inner test') + + class OuterTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + ordering.append('outer setup') + cls.addClassCleanup(ordering.append, 'outer cleanup') + def test(self): + ordering.append('start outer test') + runTests(InnerTest) + ordering.append('end outer test') + + runTests(OuterTest) + self.assertEqual(ordering, [ + 'outer setup', 'start outer test', + 'inner setup', 'inner test', 'inner cleanup', + 'end outer test', 'outer cleanup']) + class TestModuleCleanUp(unittest.TestCase): def test_add_and_do_ModuleCleanup(self): @@ -747,6 +785,7 @@ def setUpModule(): unittest.addModuleCleanup(cleanup, ordering) @staticmethod def tearDownModule(): + ordering.append('tearDownModule') raise Exception('CleanUpExc') class TestableTest(unittest.TestCase): @@ -765,7 +804,8 @@ def tearDownClass(cls): self.assertEqual(result.errors[0][1].splitlines()[-1], 'Exception: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', - 'tearDownClass', 'cleanup_good']) + 'tearDownClass', 'tearDownModule', + 'cleanup_good']) self.assertEqual(unittest.case._module_cleanups, []) def test_debug_module_executes_cleanUp(self): @@ -819,6 +859,7 @@ def setUpModule(): unittest.addModuleCleanup(cleanup, ordering, blowUp=blowUp) @staticmethod def tearDownModule(): + ordering.append('tearDownModule') raise Exception('TearDownModuleExc') class TestableTest(unittest.TestCase): @@ -838,7 +879,7 @@ def tearDownClass(cls): suite.debug() self.assertEqual(str(cm.exception), 'TearDownModuleExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', - 'tearDownClass']) + 'tearDownClass', 'tearDownModule']) self.assertTrue(unittest.case._module_cleanups) unittest.case._module_cleanups.clear() @@ -849,7 +890,7 @@ def tearDownClass(cls): suite.debug() self.assertEqual(str(cm.exception), 'TearDownModuleExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', - 'tearDownClass']) + 'tearDownClass', 'tearDownModule']) self.assertTrue(unittest.case._module_cleanups) unittest.case._module_cleanups.clear() diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py index 1bab671acdef18..e05a22861d47bf 100644 --- a/Lib/unittest/test/testmock/testasync.py +++ b/Lib/unittest/test/testmock/testasync.py @@ -149,6 +149,23 @@ async def test_async(): run(test_async()) + def test_patch_dict_async_def(self): + foo = {'a': 'a'} + @patch.dict(foo, {'a': 'b'}) + async def test_async(): + self.assertEqual(foo['a'], 'b') + + self.assertTrue(iscoroutinefunction(test_async)) + run(test_async()) + + def test_patch_dict_async_def_context(self): + foo = {'a': 'a'} + async def test_async(): + with patch.dict(foo, {'a': 'b'}): + self.assertEqual(foo['a'], 'b') + + run(test_async()) + class AsyncMockTest(unittest.TestCase): def test_iscoroutinefunction_default(self): diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index d70a6943f0a739..69631cbb810b54 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -167,12 +167,11 @@ def hostname(self): def port(self): port = self._hostinfo[1] if port is not None: - try: - port = int(port, 10) - except ValueError: - message = f'Port could not be cast to integer value as {port!r}' - raise ValueError(message) from None - if not ( 0 <= port <= 65535): + if port.isdigit() and port.isascii(): + port = int(port) + else: + raise ValueError(f"Port could not be cast to integer value as {port!r}") + if not (0 <= port <= 65535): raise ValueError("Port out of range 0-65535") return port @@ -461,7 +460,7 @@ def urlsplit(url, scheme='', allow_fragments=True): allow_fragments = bool(allow_fragments) netloc = query = fragment = '' i = url.find(':') - if i > 0: + if i > 0 and url[0].isascii() and url[0].isalpha(): for c in url[:i]: if c not in scheme_chars: break @@ -1125,15 +1124,15 @@ def splitnport(host, defport=-1): def _splitnport(host, defport=-1): """Split host and port, returning numeric port. Return given default port if no ':' found; defaults to -1. - Return numerical port if a valid number are found after ':'. + Return numerical port if a valid number is found after ':'. Return None if ':' but not a valid number.""" host, delim, port = host.rpartition(':') if not delim: host = port elif port: - try: + if port.isdigit() and port.isascii(): nport = int(port) - except ValueError: + else: nport = None return host, nport return host, defport diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 6d580a434a7bea..73ad0127ecc969 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1579,7 +1579,7 @@ def ftp_open(self, req): headers = email.message_from_string(headers) return addinfourl(fp, headers, req.full_url) except ftplib.all_errors as exp: - raise URLError(f'ftp error: {exp}') from exp + raise URLError(exp) from exp def connect_ftp(self, user, passwd, host, port, dirs, timeout): return ftpwrapper(user, passwd, host, port, dirs, timeout, diff --git a/Lib/uuid.py b/Lib/uuid.py index 8fe2479f3f22cf..e863b631877f6d 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -371,7 +371,12 @@ def _get_command_stdout(command, *args): # for are actually localized, but in theory some system could do so.) env = dict(os.environ) env['LC_ALL'] = 'C' - proc = subprocess.Popen((executable,) + args, + # Empty strings will be quoted by popen so we should just ommit it + if args != ('',): + command = (executable, *args) + else: + command = (executable,) + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env) @@ -511,7 +516,7 @@ def _ifconfig_getnode(): mac = _find_mac_near_keyword('ifconfig', args, keywords, lambda i: i+1) if mac: return mac - return None + return None def _ip_getnode(): """Get the hardware address on Unix by running ip.""" diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 6032f3648e15ff..6bce3081088200 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -128,6 +128,11 @@ def create_if_needed(d): context.prompt = '(%s) ' % prompt create_if_needed(env_dir) executable = sys._base_executable + if not executable: # see gh-96861 + raise ValueError('Unable to determine path to the running ' + 'Python interpreter. Provide an explicit path or ' + 'check that your PATH environment variable is ' + 'correctly set.') dirname, exename = os.path.split(os.path.abspath(executable)) context.executable = executable context.python_dir = dirname @@ -254,7 +259,7 @@ def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): basename + ext) # Builds or venv's from builds need to remap source file # locations, as we do not put them into Lib/venv/scripts - if sysconfig.is_python_build(True) or not os.path.isfile(srcfn): + if sysconfig.is_python_build() or not os.path.isfile(srcfn): if basename.endswith('_d'): ext = '_d' + ext basename = basename[:-2] @@ -305,7 +310,7 @@ def setup_python(self, context): f for f in os.listdir(dirname) if os.path.normcase(os.path.splitext(f)[1]) in ('.exe', '.dll') ] - if sysconfig.is_python_build(True): + if sysconfig.is_python_build(): suffixes = [ f for f in suffixes if os.path.normcase(f).startswith(('python', 'vcruntime')) @@ -320,7 +325,7 @@ def setup_python(self, context): if os.path.lexists(src): copier(src, os.path.join(binpath, suffix)) - if sysconfig.is_python_build(True): + if sysconfig.is_python_build(): # copy init.tcl for root, dirs, files in os.walk(context.python_dir): if 'init.tcl' in files: @@ -333,14 +338,25 @@ def setup_python(self, context): shutil.copyfile(src, dst) break + def _call_new_python(self, context, *py_args, **kwargs): + """Executes the newly created Python using safe-ish options""" + # gh-98251: We do not want to just use '-I' because that masks + # legitimate user preferences (such as not writing bytecode). All we + # really need is to ensure that the path variables do not overrule + # normal venv handling. + args = [context.env_exec_cmd, *py_args] + kwargs['env'] = env = os.environ.copy() + env['VIRTUAL_ENV'] = context.env_dir + env.pop('PYTHONHOME', None) + env.pop('PYTHONPATH', None) + kwargs['cwd'] = context.env_dir + kwargs['executable'] = context.env_exec_cmd + subprocess.check_output(args, **kwargs) + def _setup_pip(self, context): """Installs or upgrades pip in a virtual environment""" - # We run ensurepip in isolated mode to avoid side effects from - # environment vars, the current directory and anything else - # intended for the global Python environment - cmd = [context.env_exec_cmd, '-Im', 'ensurepip', '--upgrade', - '--default-pip'] - subprocess.check_output(cmd, stderr=subprocess.STDOUT) + self._call_new_python(context, '-m', 'ensurepip', '--upgrade', + '--default-pip', stderr=subprocess.STDOUT) def setup_scripts(self, context): """ @@ -439,9 +455,8 @@ def upgrade_dependencies(self, context): logger.debug( f'Upgrading {CORE_VENV_DEPS} packages in {context.bin_path}' ) - cmd = [context.env_exec_cmd, '-m', 'pip', 'install', '--upgrade'] - cmd.extend(CORE_VENV_DEPS) - subprocess.check_call(cmd) + self._call_new_python(context, '-m', 'pip', 'install', '--upgrade', + *CORE_VENV_DEPS) def create(env_dir, system_site_packages=False, clear=False, diff --git a/Lib/venv/scripts/posix/activate.fish b/Lib/venv/scripts/posix/activate.fish index e40a1d714894fa..9aa4446005f4d8 100644 --- a/Lib/venv/scripts/posix/activate.fish +++ b/Lib/venv/scripts/posix/activate.fish @@ -13,10 +13,13 @@ function deactivate -d "Exit virtual environment and return to normal shell env end if test -n "$_OLD_FISH_PROMPT_OVERRIDE" - functions -e fish_prompt set -e _OLD_FISH_PROMPT_OVERRIDE - functions -c _old_fish_prompt fish_prompt - functions -e _old_fish_prompt + # prevents error when using nested fish instances (Issue #93858) + if functions -q _old_fish_prompt + functions -e fish_prompt + functions -c _old_fish_prompt fish_prompt + functions -e _old_fish_prompt + end end set -e VIRTUAL_ENV diff --git a/Lib/wsgiref/validate.py b/Lib/wsgiref/validate.py index 6e16578dbb648f..6044e320a474c6 100644 --- a/Lib/wsgiref/validate.py +++ b/Lib/wsgiref/validate.py @@ -1,6 +1,6 @@ # (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) -# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php -# Also licenced under the Apache License, 2.0: http://opensource.org/licenses/apache2.0.php +# Licensed under the MIT license: https://opensource.org/licenses/mit-license.php +# Also licenced under the Apache License, 2.0: https://opensource.org/licenses/apache2.0.php # Licensed to PSF under a Contributor Agreement """ Middleware to check for obedience to the WSGI specification. diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 225bd214d627c5..e9da8be5fadd42 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -405,114 +405,121 @@ def _read_directory(archive): raise ZipImportError(f"can't open Zip file: {archive!r}", path=archive) with fp: + # GH-87235: On macOS all file descriptors for /dev/fd/N share the same + # file offset, reset the file offset after scanning the zipfile diretory + # to not cause problems when some runs 'python3 /dev/fd/9 9 header_offset: - raise ZipImportError(f'bad local header offset: {archive!r}', path=archive) - file_offset += arc_offset - - try: - name = fp.read(name_size) + fp.seek(-END_CENTRAL_DIR_SIZE, 2) + header_position = fp.tell() + buffer = fp.read(END_CENTRAL_DIR_SIZE) except OSError: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) - if len(name) != name_size: + if len(buffer) != END_CENTRAL_DIR_SIZE: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) - # On Windows, calling fseek to skip over the fields we don't use is - # slower than reading the data because fseek flushes stdio's - # internal buffers. See issue #8745. + if buffer[:4] != STRING_END_ARCHIVE: + # Bad: End of Central Dir signature + # Check if there's a comment. + try: + fp.seek(0, 2) + file_size = fp.tell() + except OSError: + raise ZipImportError(f"can't read Zip file: {archive!r}", + path=archive) + max_comment_start = max(file_size - MAX_COMMENT_LEN - + END_CENTRAL_DIR_SIZE, 0) + try: + fp.seek(max_comment_start) + data = fp.read() + except OSError: + raise ZipImportError(f"can't read Zip file: {archive!r}", + path=archive) + pos = data.rfind(STRING_END_ARCHIVE) + if pos < 0: + raise ZipImportError(f'not a Zip file: {archive!r}', + path=archive) + buffer = data[pos:pos+END_CENTRAL_DIR_SIZE] + if len(buffer) != END_CENTRAL_DIR_SIZE: + raise ZipImportError(f"corrupt Zip file: {archive!r}", + path=archive) + header_position = file_size - len(data) + pos + + header_size = _unpack_uint32(buffer[12:16]) + header_offset = _unpack_uint32(buffer[16:20]) + if header_position < header_size: + raise ZipImportError(f'bad central directory size: {archive!r}', path=archive) + if header_position < header_offset: + raise ZipImportError(f'bad central directory offset: {archive!r}', path=archive) + header_position -= header_size + arc_offset = header_position - header_offset + if arc_offset < 0: + raise ZipImportError(f'bad central directory size or offset: {archive!r}', path=archive) + + files = {} + # Start of Central Directory + count = 0 try: - if len(fp.read(header_size - name_size)) != header_size - name_size: - raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + fp.seek(header_position) except OSError: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + while True: + buffer = fp.read(46) + if len(buffer) < 4: + raise EOFError('EOF read where not expected') + # Start of file header + if buffer[:4] != b'PK\x01\x02': + break # Bad: Central Dir File Header + if len(buffer) != 46: + raise EOFError('EOF read where not expected') + flags = _unpack_uint16(buffer[8:10]) + compress = _unpack_uint16(buffer[10:12]) + time = _unpack_uint16(buffer[12:14]) + date = _unpack_uint16(buffer[14:16]) + crc = _unpack_uint32(buffer[16:20]) + data_size = _unpack_uint32(buffer[20:24]) + file_size = _unpack_uint32(buffer[24:28]) + name_size = _unpack_uint16(buffer[28:30]) + extra_size = _unpack_uint16(buffer[30:32]) + comment_size = _unpack_uint16(buffer[32:34]) + file_offset = _unpack_uint32(buffer[42:46]) + header_size = name_size + extra_size + comment_size + if file_offset > header_offset: + raise ZipImportError(f'bad local header offset: {archive!r}', path=archive) + file_offset += arc_offset - if flags & 0x800: - # UTF-8 file names extension - name = name.decode() - else: - # Historical ZIP filename encoding try: - name = name.decode('ascii') - except UnicodeDecodeError: - name = name.decode('latin1').translate(cp437_table) - - name = name.replace('/', path_sep) - path = _bootstrap_external._path_join(archive, name) - t = (path, compress, data_size, file_size, file_offset, time, date, crc) - files[name] = t - count += 1 + name = fp.read(name_size) + except OSError: + raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + if len(name) != name_size: + raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + # On Windows, calling fseek to skip over the fields we don't use is + # slower than reading the data because fseek flushes stdio's + # internal buffers. See issue #8745. + try: + if len(fp.read(header_size - name_size)) != header_size - name_size: + raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + except OSError: + raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + + if flags & 0x800: + # UTF-8 file names extension + name = name.decode() + else: + # Historical ZIP filename encoding + try: + name = name.decode('ascii') + except UnicodeDecodeError: + name = name.decode('latin1').translate(cp437_table) + + name = name.replace('/', path_sep) + path = _bootstrap_external._path_join(archive, name) + t = (path, compress, data_size, file_size, file_offset, time, date, crc) + files[name] = t + count += 1 + finally: + fp.seek(start_offset) _bootstrap._verbose_message('zipimport: found {} names in {!r}', count, archive) return files diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index dd6bcf5fe7f164..86eed5ff956992 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.38.4", - url="https://sqlite.org/2022/sqlite-autoconf-3380400.tar.gz", - checksum="34c0b92a0609ed4ce78582e8dc1ed45a", + name="SQLite 3.39.4", + url="https://sqlite.org/2022/sqlite-autoconf-3390400.tar.gz", + checksum="44b7e6691b0954086f717a6c43b622a5", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' diff --git a/Mac/BuildScript/resources/License.rtf b/Mac/BuildScript/resources/License.rtf index 7fe7e66f4c0b69..1255d1ce48ed6c 100644 --- a/Mac/BuildScript/resources/License.rtf +++ b/Mac/BuildScript/resources/License.rtf @@ -12,13 +12,13 @@ HISTORY OF THE SOFTWARE\ \f1\b0 \ulnone \ -Python was created in the early 1990s by Guido van Rossum at Stichting Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands as a successor of a language called ABC. Guido remains Python's principal author, although it includes many contributions from others.\ +Python was created in the early 1990s by Guido van Rossum at Stichting Mathematisch Centrum (CWI, see https://www.cwi.nl) in the Netherlands as a successor of a language called ABC. Guido remains Python's principal author, although it includes many contributions from others.\ \ -In 1995, Guido continued his work on Python at the Corporation for National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) in Reston, Virginia where he released several versions of the software.\ +In 1995, Guido continued his work on Python at the Corporation for National Research Initiatives (CNRI, see https://www.cnri.reston.va.us) in Reston, Virginia where he released several versions of the software.\ \ -In May 2000, Guido and the Python core development team moved to BeOpen.com to form the BeOpen PythonLabs team. In October of the same year, the PythonLabs team moved to Digital Creations (now Zope Corporation, see http://www.zope.org). In 2001, the Python Software Foundation (PSF, see https://www.python.org/psf/) was formed, a non-profit organization created specifically to own Python-related Intellectual Property. Zope Corporation is a sponsoring member of the PSF.\ +In May 2000, Guido and the Python core development team moved to BeOpen.com to form the BeOpen PythonLabs team. In October of the same year, the PythonLabs team moved to Digital Creations (now Zope Corporation, see https://www.zope.dev). In 2001, the Python Software Foundation (PSF, see https://www.python.org/psf/) was formed, a non-profit organization created specifically to own Python-related Intellectual Property. Zope Corporation is a sponsoring member of the PSF.\ \ -All Python releases are Open Source (see http://www.opensource.org for the Open Source Definition). Historically, most, but not all, Python releases have also been GPL-compatible; the table below summarizes the various releases.\ +All Python releases are Open Source (see https://opensource.org for the Open Source Definition). Historically, most, but not all, Python releases have also been GPL-compatible; the table below summarizes the various releases.\ \ \f2\b Release Derived Year Owner GPL-\ diff --git a/Mac/Extras.install.py b/Mac/Extras.install.py index ab1af71dfd823f..41bfa53d616307 100644 --- a/Mac/Extras.install.py +++ b/Mac/Extras.install.py @@ -9,10 +9,9 @@ debug = 0 def isclean(name): - if name == 'CVS': return 0 - if name == '.cvsignore': return 0 - if name == '.DS_store': return 0 - if name == '.svn': return 0 + if name in ('CVS', '.cvsignore', '.svn'): + return 0 + if name.lower() == '.ds_store': return 0 if name.endswith('~'): return 0 if name.endswith('.BAK'): return 0 if name.endswith('.pyc'): return 0 diff --git a/Mac/README.rst b/Mac/README.rst index 7476639d0ff541..bc40b41f7f38ad 100644 --- a/Mac/README.rst +++ b/Mac/README.rst @@ -10,6 +10,19 @@ Python on macOS README This document provides a quick overview of some macOS specific features in the Python distribution. +Compilers for building on macOS +=============================== + +The core developers primarily test builds on macOS with Apple's compiler tools, +either Xcode or the Command Line Tools. For these we only support building with +a compiler that includes an SDK that targets the OS on the build machine, that is +the version of Xcode that shipped with the OS version or one newer. + +For example, for macOS 12 we support Xcode 13 and Xcode 14 (or the corresponding +Command Line Tools). + +Building with other compilers, such as GCC, likely works, but is not actively supported. + macOS specific arguments to configure ===================================== diff --git a/Makefile.pre.in b/Makefile.pre.in index 8fbcd7ac170afe..b356f6293e67e8 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -817,10 +817,11 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) # wasm assets directory is relative to current build dir, e.g. "./usr/local". # --preload-file turns a relative asset path into an absolute path. +.PHONY: wasm_stdlib +wasm_stdlib: $(WASM_STDLIB) $(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ $(srcdir)/Tools/wasm/wasm_assets.py \ - Makefile pybuilddir.txt Modules/Setup.local \ - python.html python.worker.js + Makefile pybuilddir.txt Modules/Setup.local $(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \ --buildroot . --prefix $(prefix) @@ -1713,6 +1714,10 @@ buildbottest: all fi $(TESTRUNNER) -j 1 -u all -W --slowest --fail-env-changed --timeout=$(TESTTIMEOUT) $(TESTOPTS) +# Like testall, but run Python tests with HOSTRUNNER directly. +hostrunnertest: all + $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test -u all $(TESTOPTS) + pythoninfo: all $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo @@ -1999,6 +2004,7 @@ TESTSUBDIRS= ctypes/test \ test/test_warnings test/test_warnings/data \ test/test_zoneinfo test/test_zoneinfo/data \ test/tracedmodules \ + test/typinganndata \ test/xmltestdata test/xmltestdata/c14n-20 \ test/ziptestdata \ tkinter/test tkinter/test/test_tkinter \ @@ -2383,8 +2389,7 @@ rmtestturds: -rm -f gb-18030-2000.xml docclean: - -rm -rf Doc/build - -rm -rf Doc/tools/sphinx Doc/tools/pygments Doc/tools/docutils + $(MAKE) -C $(srcdir)/Doc clean # like the 'clean' target but retain the profile guided optimization (PGO) # data. The PGO data is only valid if source code remains unchanged. @@ -2435,7 +2440,7 @@ clobber: clean # Make things extra clean, before making a distribution: # remove all generated files, even Makefile[.pre] # Keep configure and Python-ast.[ch], it's possible they can't be generated -distclean: clobber +distclean: clobber docclean for file in $(srcdir)/Lib/test/data/* ; do \ if test "$$file" != "$(srcdir)/Lib/test/data/README"; then rm "$$file"; fi; \ done @@ -2523,12 +2528,12 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h MODULE_CMATH_DEPS=$(srcdir)/Modules/_math.h MODULE_MATH_DEPS=$(srcdir)/Modules/_math.h -MODULE_PYEXPAT_DEPS=$(LIBEXPAT_HEADERS) @LIBEXPAT_INTERNAL@ +MODULE_PYEXPAT_DEPS=@LIBEXPAT_INTERNAL@ MODULE_UNICODEDATA_DEPS=$(srcdir)/Modules/unicodedata_db.h $(srcdir)/Modules/unicodename_db.h MODULE__BLAKE2_DEPS=$(srcdir)/Modules/_blake2/impl/blake2-config.h $(srcdir)/Modules/_blake2/impl/blake2-impl.h $(srcdir)/Modules/_blake2/impl/blake2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2b-ref.c $(srcdir)/Modules/_blake2/impl/blake2b-round.h $(srcdir)/Modules/_blake2/impl/blake2b.c $(srcdir)/Modules/_blake2/impl/blake2s-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2s-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2s-load-xop.h $(srcdir)/Modules/_blake2/impl/blake2s-ref.c $(srcdir)/Modules/_blake2/impl/blake2s-round.h $(srcdir)/Modules/_blake2/impl/blake2s.c $(srcdir)/Modules/_blake2/blake2module.h $(srcdir)/Modules/hashlib.h MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h -MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h $(LIBMPDEC_HEADERS) @LIBMPDEC_INTERNAL@ -MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c $(LIBEXPAT_HEADERS) @LIBEXPAT_INTERNAL@ +MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ +MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h diff --git a/Misc/HISTORY b/Misc/HISTORY index 570638869f92ed..0330c4891b1c36 100644 --- a/Misc/HISTORY +++ b/Misc/HISTORY @@ -19610,7 +19610,7 @@ durable way. For example, some people say they're confused by that the Open Source Initiative's entry for the Python Software Foundation License:: - http://www.opensource.org/licenses/PythonSoftFoundation.php + https://opensource.org/licenses/PythonSoftFoundation.php says "Python 2.1.1" all over it, wondering whether it applies only to Python 2.1.1. diff --git a/Misc/NEWS.d/3.11.1.rst b/Misc/NEWS.d/3.11.1.rst new file mode 100644 index 00000000000000..62cf53afe507c9 --- /dev/null +++ b/Misc/NEWS.d/3.11.1.rst @@ -0,0 +1,1545 @@ +.. date: 2022-12-05-01-39-10 +.. gh-issue: 100001 +.. nonce: uD05Fc +.. release date: 2022-12-06 +.. section: Security + +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` +:class:`BaseHTTPRequestHandler` ``.log_message`` method to replace control +characters with a ``\xHH`` hex escape before printing. + +.. + +.. date: 2022-11-11-12-50-28 +.. gh-issue: 87604 +.. nonce: OtwH5L +.. section: Security + +Avoid publishing list of active per-interpreter audit hooks via the +:mod:`gc` module + +.. + +.. date: 2022-11-04-09-29-36 +.. gh-issue: 98433 +.. nonce: l76c5G +.. section: Security + +The IDNA codec decoder used on DNS hostnames by :mod:`socket` or +:mod:`asyncio` related name resolution functions no longer involves a +quadratic algorithm. This prevents a potential CPU denial of service if an +out-of-spec excessive length hostname involving bidirectional characters +were decoded. Some protocols such as :mod:`urllib` http ``3xx`` redirects +potentially allow for an attacker to supply such a name. + +.. + +.. date: 2022-10-26-21-04-23 +.. gh-issue: 98739 +.. nonce: keBWcY +.. section: Security + +Update bundled libexpat to 2.5.0 + +.. + +.. date: 2022-09-28-12-10-57 +.. gh-issue: 97612 +.. nonce: y6NvOQ +.. section: Security + +Fix a shell code injection vulnerability in the +``get-remote-certificate.py`` example script. The script no longer uses a +shell to run ``openssl`` commands. Issue reported and initial fix by Caleb +Shortt. Patch by Victor Stinner. + +.. + +.. date: 2022-11-30-15-29-08 +.. gh-issue: 99886 +.. nonce: feJkSv +.. section: Core and Builtins + +Fix a crash when an object which does not have a dictionary frees its +instance values. + +.. + +.. date: 2022-11-30-11-09-40 +.. gh-issue: 99891 +.. nonce: 9VomwB +.. section: Core and Builtins + +Fix a bug in the tokenizer that could cause infinite recursion when showing +syntax warnings that happen in the first line of the source. Patch by Pablo +Galindo + +.. + +.. date: 2022-11-26-05-34-00 +.. gh-issue: 99729 +.. nonce: A3ovwQ +.. section: Core and Builtins + +Fix an issue that could cause frames to be visible to Python code as they +are being torn down, possibly leading to memory corruption or hard crashes +of the interpreter. + +.. + +.. date: 2022-11-21-11-27-14 +.. gh-issue: 99578 +.. nonce: DcKoBJ +.. section: Core and Builtins + +Fix a reference bug in :func:`_imp.create_builtin()` after the creation of +the first sub-interpreter for modules ``builtins`` and ``sys``. Patch by +Victor Stinner. + +.. + +.. date: 2022-11-19-22-27-52 +.. gh-issue: 99581 +.. nonce: yKYPbf +.. section: Core and Builtins + +Fixed a bug that was causing a buffer overflow if the tokenizer copies a +line missing the newline caracter from a file that is as long as the +available tokenizer buffer. Patch by Pablo galindo + +.. + +.. date: 2022-11-18-11-24-25 +.. gh-issue: 99553 +.. nonce: F64h-n +.. section: Core and Builtins + +Fix bug where an :exc:`ExceptionGroup` subclass can wrap a +:exc:`BaseException`. + +.. + +.. date: 2022-11-12-01-39-57 +.. gh-issue: 99370 +.. nonce: _cu32j +.. section: Core and Builtins + +Fix zip path for venv created from a non-installed python on POSIX +platforms. + +.. + +.. date: 2022-11-10-02-11-23 +.. gh-issue: 99298 +.. nonce: NeArAJ +.. section: Core and Builtins + +Fix an issue that could potentially cause incorrect error handling for some +bytecode instructions. + +.. + +.. date: 2022-11-08-16-35-25 +.. gh-issue: 99205 +.. nonce: 2YOoFT +.. section: Core and Builtins + +Fix an issue that prevented :c:type:`PyThreadState` and +:c:type:`PyInterpreterState` memory from being freed properly. + +.. + +.. date: 2022-11-07-10-29-41 +.. gh-issue: 99181 +.. nonce: bfG4bI +.. section: Core and Builtins + +Fix failure in :keyword:`except* ` with unhashable exceptions. + +.. + +.. date: 2022-11-07-08-17-12 +.. gh-issue: 99204 +.. nonce: Mf4hMD +.. section: Core and Builtins + +Fix calculation of :data:`sys._base_executable` when inside a POSIX virtual +environment using copies of the python binary when the base installation +does not provide the executable name used by the venv. Calculation will fall +back to alternative names ("python", "python."). + +.. + +.. date: 2022-11-06-22-59-02 +.. gh-issue: 96055 +.. nonce: TmQuJn +.. section: Core and Builtins + +Update :mod:`faulthandler` to emit an error message with the proper +unexpected signal number. Patch by Dong-hee Na. + +.. + +.. date: 2022-11-06-13-25-01 +.. gh-issue: 99153 +.. nonce: uE3CVL +.. section: Core and Builtins + +Fix location of :exc:`SyntaxError` for a :keyword:`try` block with both +:keyword:`except` and :keyword:`except* `. + +.. + +.. date: 2022-11-06-00-17-58 +.. gh-issue: 99103 +.. nonce: bFA9BX +.. section: Core and Builtins + +Fix the error reporting positions of specialized traceback anchors when the +source line contains Unicode characters. + +.. + +.. date: 2022-10-31-21-01-35 +.. gh-issue: 98852 +.. nonce: MYaRN6 +.. section: Core and Builtins + +Fix subscription of type aliases containing bare generic types or types like +:class:`~typing.TypeVar`: for example ``tuple[A, T][int]`` and +``tuple[TypeVar, T][int]``, where ``A`` is a generic type, and ``T`` is a +type variable. + +.. + +.. date: 2022-10-31-18-03-10 +.. gh-issue: 98925 +.. nonce: zpdjVd +.. section: Core and Builtins + +Lower the recursion depth for marshal on WASI to support wasmtime 2.0/main. + +.. + +.. date: 2022-10-28-14-52-55 +.. gh-issue: 98783 +.. nonce: iG0kMs +.. section: Core and Builtins + +Fix multiple crashes in debug mode when ``str`` subclasses are used instead +of ``str`` itself. + +.. + +.. date: 2022-10-21-11-28-53 +.. gh-issue: 99257 +.. nonce: nmcuf- +.. section: Core and Builtins + +Fix an issue where member descriptors (such as those for +:attr:`~object.__slots__`) could behave incorrectly or crash instead of +raising a :exc:`TypeError` when accessed via an instance of an invalid type. + +.. + +.. date: 2022-10-19-23-48-46 +.. gh-issue: 98374 +.. nonce: eOBh8M +.. section: Core and Builtins + +Suppress ImportError for invalid query for help() command. Patch by Dong-hee +Na. + +.. + +.. date: 2022-10-19-01-01-08 +.. gh-issue: 98415 +.. nonce: ZS2eWh +.. section: Core and Builtins + +Fix detection of MAC addresses for :mod:`uuid` on certain OSs. Patch by +Chaim Sanders + +.. + +.. date: 2022-10-15-23-15-14 +.. gh-issue: 92119 +.. nonce: PMSwwG +.. section: Core and Builtins + +Print exception class name instead of its string representation when raising +errors from :mod:`ctypes` calls. + +.. + +.. date: 2022-10-06-15-45-57 +.. gh-issue: 96078 +.. nonce: fS-6mU +.. section: Core and Builtins + +:func:`os.sched_yield` now release the GIL while calling sched_yield(2). +Patch by Dong-hee Na. + +.. + +.. date: 2022-10-06-05-41-01 +.. gh-issue: 93354 +.. nonce: 6BpHl2 +.. section: Core and Builtins + +Fix an issue that could delay the specialization of :opcode:`PRECALL` +instructions. + +.. + +.. date: 2022-10-05-17-02-22 +.. gh-issue: 97943 +.. nonce: LYAWlE +.. section: Core and Builtins + +Bugfix: :func:`PyFunction_GetAnnotations` should return a borrowed +reference. It was returning a new reference. + +.. + +.. date: 2022-10-04-02-00-10 +.. gh-issue: 97779 +.. nonce: f3N1hI +.. section: Core and Builtins + +Ensure that all Python frame objects are backed by "complete" frames. + +.. + +.. date: 2022-10-01-08-55-09 +.. gh-issue: 97591 +.. nonce: pw6kkH +.. section: Core and Builtins + +Fixed a missing incref/decref pair in ``Exception.__setstate__()``. Patch by +Ofey Chan. + +.. + +.. date: 2022-09-29-15-19-29 +.. gh-issue: 94526 +.. nonce: wq5m6T +.. section: Core and Builtins + +Fix the Python path configuration used to initialized :data:`sys.path` at +Python startup. Paths are no longer encoded to UTF-8/strict to avoid +encoding errors if it contains surrogate characters (bytes paths are decoded +with the surrogateescape error handler). Patch by Victor Stinner. + +.. + +.. date: 2022-09-20-11-06-45 +.. gh-issue: 95921 +.. nonce: dkcRQn +.. section: Core and Builtins + +Fix overly-broad source position information for chained comparisons used as +branching conditions. + +.. + +.. date: 2022-09-16-16-54-35 +.. gh-issue: 96387 +.. nonce: GRzewg +.. section: Core and Builtins + +At Python exit, sometimes a thread holding the GIL can wait forever for a +thread (usually a daemon thread) which requested to drop the GIL, whereas +the thread already exited. To fix the race condition, the thread which +requested the GIL drop now resets its request before exiting. Issue +discovered and analyzed by Mingliang ZHAO. Patch by Victor Stinner. + +.. + +.. date: 2022-09-16-12-36-13 +.. gh-issue: 96864 +.. nonce: PLU3i8 +.. section: Core and Builtins + +Fix a possible assertion failure, fatal error, or :exc:`SystemError` if a +line tracing event raises an exception while opcode tracing is enabled. + +.. + +.. date: 2022-09-13-12-06-46 +.. gh-issue: 96678 +.. nonce: NqGFyb +.. section: Core and Builtins + +Fix undefined behaviour in C code of null pointer arithmetic. + +.. + +.. date: 2022-09-12-16-58-22 +.. gh-issue: 96754 +.. nonce: 0GRme5 +.. section: Core and Builtins + +Make sure that all frame objects created are created from valid interpreter +frames. Prevents the possibility of invalid frames in backtraces and signal +handlers. + +.. + +.. date: 2022-08-29-13-06-58 +.. gh-issue: 95196 +.. nonce: eGRR4b +.. section: Core and Builtins + +Disable incorrect pickling of the C implemented classmethod descriptors. + +.. + +.. date: 2022-08-15-21-08-11 +.. gh-issue: 96005 +.. nonce: 6eoc8k +.. section: Core and Builtins + +On WASI :data:`~errno.ENOTCAPABLE` is now mapped to :exc:`PermissionError`. +The :mod:`errno` modules exposes the new error number. ``getpath.py`` now +ignores :exc:`PermissionError` when it cannot open landmark files +``pybuilddir.txt`` and ``pyenv.cfg``. + +.. + +.. date: 2022-06-10-16-37-44 +.. gh-issue: 93696 +.. nonce: 65BI2R +.. section: Core and Builtins + +Allow :mod:`pdb` to locate source for frozen modules in the standard +library. + +.. + +.. bpo: 31718 +.. date: 2020-02-23-23-48-15 +.. nonce: sXko5e +.. section: Core and Builtins + +Raise :exc:`ValueError` instead of :exc:`SystemError` when methods of +uninitialized :class:`io.IncrementalNewlineDecoder` objects are called. +Patch by Oren Milman. + +.. + +.. bpo: 38031 +.. date: 2019-09-04-19-09-49 +.. nonce: Yq4L72 +.. section: Core and Builtins + +Fix a possible assertion failure in :class:`io.FileIO` when the opener +returns an invalid file descriptor. + +.. + +.. date: 2022-12-05-13-40-15 +.. gh-issue: 100001 +.. nonce: 78ReYp +.. section: Library + +Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so +that it is technically possible to parse the line and reconstruct what the +original data was. Without this a \xHH is ambiguious as to if it is a hex +replacement we put in or the characters r"\x" came through in the original +request line. + +.. + +.. date: 2022-12-02-13-05-00 +.. gh-issue: 93453 +.. nonce: EFj1NN +.. section: Library + +:func:`asyncio.get_event_loop` now only emits a deprecation warning when a +new event loop was created implicitly. It no longer emits a deprecation +warning if the current event loop was set. + +.. + +.. date: 2022-11-21-17-56-18 +.. gh-issue: 51524 +.. nonce: nTykx8 +.. section: Library + +Fix bug when calling trace.CoverageResults with valid infile. + +.. + +.. date: 2022-11-21-13-49-03 +.. gh-issue: 99645 +.. nonce: 9w1QKq +.. section: Library + +Fix a bug in handling class cleanups in :class:`unittest.TestCase`. Now +``addClassCleanup()`` uses separate lists for different ``TestCase`` +subclasses, and ``doClassCleanups()`` only cleans up the particular class. + +.. + +.. date: 2022-11-15-10-55-24 +.. gh-issue: 97001 +.. nonce: KeQuVF +.. section: Library + +Release the GIL when calling termios APIs to avoid blocking threads. + +.. + +.. date: 2022-11-13-02-06-56 +.. gh-issue: 99341 +.. nonce: 8-OlwB +.. section: Library + +Fix :func:`ast.increment_lineno` to also cover :class:`ast.TypeIgnore` when +changing line numbers. + +.. + +.. date: 2022-11-12-15-45-51 +.. gh-issue: 99418 +.. nonce: FxfAXS +.. section: Library + +Fix bug in :func:`urllib.parse.urlparse` that causes URL schemes that begin +with a digit, a plus sign, or a minus sign to be parsed incorrectly. + +.. + +.. date: 2022-11-12-12-15-30 +.. gh-issue: 99382 +.. nonce: dKg_rW +.. section: Library + +Check the number of arguments in substitution in user generics containing a +:class:`~typing.TypeVarTuple` and one or more :class:`~typing.TypeVar`. + +.. + +.. date: 2022-11-12-12-10-23 +.. gh-issue: 99379 +.. nonce: bcGhxF +.. section: Library + +Fix substitution of :class:`~typing.ParamSpec` followed by +:class:`~typing.TypeVarTuple` in generic aliases. + +.. + +.. date: 2022-11-12-12-08-34 +.. gh-issue: 99344 +.. nonce: 7M_u8G +.. section: Library + +Fix substitution of :class:`~typing.TypeVarTuple` and +:class:`~typing.ParamSpec` together in user generics. + +.. + +.. date: 2022-11-09-20-48-42 +.. gh-issue: 74044 +.. nonce: zBj26K +.. section: Library + +Fixed bug where :func:`inspect.signature` reported incorrect arguments for +decorated methods. + +.. + +.. date: 2022-11-09-12-16-35 +.. gh-issue: 99275 +.. nonce: klOqoL +.. section: Library + +Fix ``SystemError`` in :mod:`ctypes` when exception was not set during +``__initsubclass__``. + +.. + +.. date: 2022-11-09-08-40-52 +.. gh-issue: 99277 +.. nonce: J1P44O +.. section: Library + +Remove older version of ``_SSLProtocolTransport.get_write_buffer_limits`` in +:mod:`!asyncio.sslproto` + +.. + +.. date: 2022-11-08-11-15-37 +.. gh-issue: 99248 +.. nonce: 1vt8xI +.. section: Library + +fix negative numbers failing in verify() + +.. + +.. date: 2022-11-06-12-44-51 +.. gh-issue: 99155 +.. nonce: vLZOzi +.. section: Library + +Fix :class:`statistics.NormalDist` pickle with ``0`` and ``1`` protocols. + +.. + +.. date: 2022-11-05-23-16-15 +.. gh-issue: 93464 +.. nonce: ucd4vP +.. section: Library + +``enum.auto()`` is now correctly activated when combined with other +assignment values. E.g. ``ONE = auto(), 'some text'`` will now evaluate as +``(1, 'some text')``. + +.. + +.. date: 2022-11-05-17-16-40 +.. gh-issue: 99134 +.. nonce: Msgspf +.. section: Library + +Update the bundled copy of pip to version 22.3.1. + +.. + +.. date: 2022-11-02-05-54-02 +.. gh-issue: 83004 +.. nonce: 0v8iyw +.. section: Library + +Clean up refleak on failed module initialisation in :mod:`_zoneinfo` + +.. + +.. date: 2022-11-02-05-53-25 +.. gh-issue: 83004 +.. nonce: qc_KHr +.. section: Library + +Clean up refleaks on failed module initialisation in in :mod:`_pickle` + +.. + +.. date: 2022-11-02-05-52-36 +.. gh-issue: 83004 +.. nonce: LBl79O +.. section: Library + +Clean up refleak on failed module initialisation in :mod:`_io`. + +.. + +.. date: 2022-10-31-12-34-03 +.. gh-issue: 98897 +.. nonce: rgNn4x +.. section: Library + +Fix memory leak in :func:`math.dist` when both points don't have the same +dimension. Patch by Kumar Aditya. + +.. + +.. date: 2022-10-30-12-22-24 +.. gh-issue: 98706 +.. nonce: v1Kuy5 +.. section: Library + +[3.11] Applied changes from importlib_metadata `4.11.4 through 4.13 +`_, +including compatibility and robustness fixes for ``Distribution`` objects +without ``_normalized_name``, disallowing invalid inputs to +``Distribution.from_name``, and refined behaviors in +``PathDistribution._name_from_stem`` and +``PathDistribution._normalized_name``. + +.. + +.. date: 2022-10-29-03-40-18 +.. gh-issue: 98793 +.. nonce: WSPB4A +.. section: Library + +Fix argument typechecks in :func:`!_overlapped.WSAConnect` and +:func:`!_overlapped.Overlapped.WSASendTo` functions. + +.. + +.. date: 2022-10-28-23-44-17 +.. gh-issue: 98744 +.. nonce: sGHDWm +.. section: Library + +Prevent crashing in :mod:`traceback` when retrieving the byte-offset for +some source files that contain certain unicode characters. + +.. + +.. date: 2022-10-27-12-56-38 +.. gh-issue: 98740 +.. nonce: ZoqqGM +.. section: Library + +Fix internal error in the :mod:`re` module which in very rare circumstances +prevented compilation of a regular expression containing a :ref:`conditional +expression ` without the "else" branch. + +.. + +.. date: 2022-10-26-07-51-55 +.. gh-issue: 98703 +.. nonce: 0hW773 +.. section: Library + +Fix :meth:`asyncio.StreamWriter.drain` to call ``protocol.connection_lost`` +callback only once on Windows. + +.. + +.. date: 2022-10-25-20-17-34 +.. gh-issue: 98624 +.. nonce: YQUPFy +.. section: Library + +Add a mutex to unittest.mock.NonCallableMock to protect concurrent access to +mock attributes. + +.. + +.. date: 2022-10-23-18-30-39 +.. gh-issue: 89237 +.. nonce: kBui30 +.. section: Library + +Fix hang on Windows in ``subprocess.wait_closed()`` in :mod:`asyncio` with +:class:`~asyncio.ProactorEventLoop`. Patch by Kumar Aditya. + +.. + +.. date: 2022-10-19-18-31-53 +.. gh-issue: 98458 +.. nonce: vwyq7O +.. section: Library + +Fix infinite loop in unittest when a self-referencing chained exception is +raised + +.. + +.. date: 2022-10-19-09-29-12 +.. gh-issue: 97928 +.. nonce: xj3im7 +.. section: Library + +:meth:`tkinter.Text.count` raises now an exception for options starting with +"-" instead of silently ignoring them. + +.. + +.. date: 2022-10-16-18-52-00 +.. gh-issue: 97966 +.. nonce: humlhz +.. section: Library + +On ``uname_result``, restored expectation that ``_fields`` and ``_asdict`` +would include all six properties including ``processor``. + +.. + +.. date: 2022-10-16-06-18-59 +.. gh-issue: 98307 +.. nonce: b2_CDu +.. section: Library + +A :meth:`~logging.handlers.SysLogHandler.createSocket` method was added to +:class:`~logging.handlers.SysLogHandler`. + +.. + +.. date: 2022-10-14-19-57-37 +.. gh-issue: 96035 +.. nonce: 0xcX-p +.. section: Library + +Fix bug in :func:`urllib.parse.urlparse` that causes certain port numbers +containing whitespace, underscores, plus and minus signs, or non-ASCII +digits to be incorrectly accepted. + +.. + +.. date: 2022-10-14-11-46-31 +.. gh-issue: 98251 +.. nonce: Uxc9al +.. section: Library + +Allow :mod:`venv` to pass along :envvar:`PYTHON*` variables to ``ensurepip`` +and ``pip`` when they do not impact path resolution + +.. + +.. date: 2022-10-12-10-00-40 +.. gh-issue: 98178 +.. nonce: hspH51 +.. section: Library + +On macOS, fix a crash in :func:`syslog.syslog` in multi-threaded +applications. On macOS, the libc ``syslog()`` function is not thread-safe, +so :func:`syslog.syslog` no longer releases the GIL to call it. Patch by +Victor Stinner. + +.. + +.. date: 2022-10-10-07-07-31 +.. gh-issue: 96151 +.. nonce: K9fwoq +.. section: Library + +Allow ``BUILTINS`` to be a valid field name for frozen dataclasses. + +.. + +.. date: 2022-10-09-12-12-38 +.. gh-issue: 87730 +.. nonce: ClgP3f +.. section: Library + +Wrap network errors consistently in urllib FTP support, so the test suite +doesn't fail when a network is available but the public internet is not +reachable. + +.. + +.. date: 2022-10-08-19-39-27 +.. gh-issue: 98086 +.. nonce: y---WC +.. section: Library + +Make sure ``patch.dict()`` can be applied on async functions. + +.. + +.. date: 2022-10-06-23-42-00 +.. gh-issue: 90985 +.. nonce: s280JY +.. section: Library + +Earlier in 3.11 we deprecated ``asyncio.Task.cancel("message")``. We +realized we were too harsh, and have undeprecated it. + +.. + +.. date: 2022-10-04-21-21-41 +.. gh-issue: 97837 +.. nonce: 19q-eg +.. section: Library + +Change deprecate warning message in :mod:`unittest` from + +``It is deprecated to return a value!=None`` + +to + +``It is deprecated to return a value that is not None from a test case`` + +.. + +.. date: 2022-10-04-07-55-19 +.. gh-issue: 97825 +.. nonce: mNdv1l +.. section: Library + +Fixes :exc:`AttributeError` when :meth:`subprocess.check_output` is used +with argument ``input=None`` and either of the arguments *encoding* or +*errors* are used. + +.. + +.. date: 2022-10-02-12-38-22 +.. gh-issue: 82836 +.. nonce: OvYLmC +.. section: Library + +Fix :attr:`~ipaddress.IPv4Address.is_private` properties in the +:mod:`ipaddress` module. Previously non-private networks (0.0.0.0/0) would +return True from this method; now they correctly return False. + +.. + +.. date: 2022-09-30-15-56-20 +.. gh-issue: 96827 +.. nonce: lzy1iw +.. section: Library + +Avoid spurious tracebacks from :mod:`asyncio` when default executor cleanup +is delayed until after the event loop is closed (e.g. as the result of a +keyboard interrupt). + +.. + +.. date: 2022-09-29-23-22-24 +.. gh-issue: 97592 +.. nonce: tpJg_J +.. section: Library + +Avoid a crash in the C version of +:meth:`asyncio.Future.remove_done_callback` when an evil argument is passed. + +.. + +.. date: 2022-09-29-08-15-55 +.. gh-issue: 97639 +.. nonce: JSjWYW +.. section: Library + +Remove ``tokenize.NL`` check from :mod:`tabnanny`. + +.. + +.. date: 2022-09-25-20-42-33 +.. gh-issue: 73588 +.. nonce: uVtjEA +.. section: Library + +Fix generation of the default name of :class:`tkinter.Checkbutton`. +Previously, checkbuttons in different parent widgets could have the same +short name and share the same state if arguments "name" and "variable" are +not specified. Now they are globally unique. + +.. + +.. date: 2022-09-22-14-35-02 +.. gh-issue: 97005 +.. nonce: yf21Q7 +.. section: Library + +Update bundled libexpat to 2.4.9 + +.. + +.. date: 2022-09-22-11-50-29 +.. gh-issue: 85760 +.. nonce: DETTPd +.. section: Library + +Fix race condition in :mod:`asyncio` where +:meth:`~asyncio.SubprocessProtocol.process_exited` called before the +:meth:`~asyncio.SubprocessProtocol.pipe_data_received` leading to +inconsistent output. Patch by Kumar Aditya. + +.. + +.. date: 2022-09-17-13-15-10 +.. gh-issue: 96819 +.. nonce: 6RfqM7 +.. section: Library + +Fixed check in :mod:`multiprocessing.resource_tracker` that guarantees that +the length of a write to a pipe is not greater than ``PIPE_BUF``. + +.. + +.. date: 2022-09-15-00-37-33 +.. gh-issue: 96741 +.. nonce: 4b6czN +.. section: Library + +Corrected type annotation for dataclass attribute +``pstats.FunctionProfile.ncalls`` to be ``str``. + +.. + +.. date: 2022-08-30-11-46-36 +.. gh-issue: 95987 +.. nonce: CV7_u4 +.. section: Library + +Fix ``repr`` of ``Any`` subclasses. + +.. + +.. date: 2022-08-29-16-54-36 +.. gh-issue: 96388 +.. nonce: dCpJcu +.. section: Library + +Work around missing socket functions in :class:`~socket.socket`'s +``__repr__``. + +.. + +.. date: 2022-08-29-12-35-28 +.. gh-issue: 96073 +.. nonce: WaGstf +.. section: Library + +In :mod:`inspect`, fix overeager replacement of "``typing.``" in formatting +annotations. + +.. + +.. date: 2022-08-23-03-13-18 +.. gh-issue: 96192 +.. nonce: TJywOF +.. section: Library + +Fix handling of ``bytes`` :term:`path-like objects ` in +:func:`os.ismount()`. + +.. + +.. date: 2022-08-20-10-31-01 +.. gh-issue: 96052 +.. nonce: a6FhaD +.. section: Library + +Fix handling compiler warnings (SyntaxWarning and DeprecationWarning) in +:func:`codeop.compile_command` when checking for incomplete input. +Previously it emitted warnings and raised a SyntaxError. Now it always +returns ``None`` for incomplete input without emitting any warnings. + +.. + +.. date: 2022-08-06-12-18-07 +.. gh-issue: 88863 +.. nonce: NnqsuJ +.. section: Library + +To avoid apparent memory leaks when :func:`asyncio.open_connection` raises, +break reference cycles generated by local exception and future instances +(which has exception instance as its member var). Patch by Dong Uk, Kang. + +.. + +.. date: 2022-07-22-09-09-08 +.. gh-issue: 91212 +.. nonce: 53O8Ab +.. section: Library + +Fixed flickering of the turtle window when the tracer is turned off. Patch +by Shin-myoung-serp. + +.. + +.. date: 2022-07-08-08-39-35 +.. gh-issue: 88050 +.. nonce: 0aOC_m +.. section: Library + +Fix :mod:`asyncio` subprocess transport to kill process cleanly when process +is blocked and avoid ``RuntimeError`` when loop is closed. Patch by Kumar +Aditya. + +.. + +.. date: 2022-06-17-12-02-30 +.. gh-issue: 93858 +.. nonce: R49ARc +.. section: Library + +Prevent error when activating venv in nested fish instances. + +.. + +.. date: 2022-04-23-03-46-37 +.. gh-issue: 91078 +.. nonce: 87-hkp +.. section: Library + +:meth:`TarFile.next` now returns ``None`` when called on an empty tarfile. + +.. + +.. bpo: 47220 +.. date: 2022-04-04-22-54-11 +.. nonce: L9jYu4 +.. section: Library + +Document the optional *callback* parameter of :class:`WeakMethod`. Patch by +Géry Ogam. + +.. + +.. bpo: 46364 +.. date: 2022-01-14-10-49-20 +.. nonce: SzhlU9 +.. section: Library + +Restrict use of sockets instead of pipes for stdin of subprocesses created +by :mod:`asyncio` to AIX platform only. + +.. + +.. bpo: 38523 +.. date: 2020-10-23-22-20-52 +.. nonce: CrkxLh +.. section: Library + +:func:`shutil.copytree` now applies the *ignore_dangling_symlinks* argument +recursively. + +.. + +.. bpo: 36267 +.. date: 2019-09-03-15-45-19 +.. nonce: z42Ex7 +.. section: Library + +Fix IndexError in :class:`argparse.ArgumentParser` when a ``store_true`` +action is given an explicit argument. + +.. + +.. date: 2022-11-16-12-52-23 +.. gh-issue: 92892 +.. nonce: TS-P0j +.. section: Documentation + +Document that calling variadic functions with ctypes requires special care +on macOS/arm64 (and possibly other platforms). + +.. + +.. date: 2022-10-16-17-34-45 +.. gh-issue: 85525 +.. nonce: DvkD0v +.. section: Documentation + +Remove extra row + +.. + +.. date: 2022-08-12-01-12-52 +.. gh-issue: 95588 +.. nonce: PA0FI7 +.. section: Documentation + +Clarified the conflicting advice given in the :mod:`ast` documentation about +:func:`ast.literal_eval` being "safe" for use on untrusted input while at +the same time warning that it can crash the process. The latter statement is +true and is deemed unfixable without a large amount of work unsuitable for a +bugfix. So we keep the warning and no longer claim that ``literal_eval`` is +safe. + +.. + +.. bpo: 41825 +.. date: 2020-09-22-12-32-16 +.. nonce: npcaCb +.. section: Documentation + +Restructured the documentation for the :func:`os.wait* ` family of +functions, and improved the docs for :func:`os.waitid` with more explanation +of the possible argument constants. + +.. + +.. date: 2022-12-05-16-12-56 +.. gh-issue: 99892 +.. nonce: sz_eW8 +.. section: Tests + +Skip test_normalization() of test_unicodedata if it fails to download +NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. + +.. + +.. date: 2022-12-01-18-55-18 +.. gh-issue: 99934 +.. nonce: Ox3Fqf +.. section: Tests + +Correct test_marsh on (32 bit) x86: test_deterministic sets was failing. + +.. + +.. date: 2022-11-21-19-21-30 +.. gh-issue: 99659 +.. nonce: 4gP0nm +.. section: Tests + +Optional big memory tests in ``test_sqlite3`` now catch the correct +:exc:`sqlite.DataError` exception type in case of too large strings and/or +blobs passed. + +.. + +.. date: 2022-10-26-15-19-20 +.. gh-issue: 98713 +.. nonce: Lnu32R +.. section: Tests + +Fix a bug in the :mod:`typing` tests where a test relying on +CPython-specific implementation details was not decorated with +``@cpython_only`` and was not skipped on other implementations. + +.. + +.. date: 2022-10-15-07-46-48 +.. gh-issue: 87390 +.. nonce: asR-Zo +.. section: Tests + +Add tests for star-unpacking with PEP 646, and some other miscellaneous PEP +646 tests. + +.. + +.. date: 2022-10-12-14-57-06 +.. gh-issue: 96853 +.. nonce: ANe-bw +.. section: Tests + +Added explicit coverage of ``Py_Initialize`` (and hence ``Py_InitializeEx``) +back to the embedding tests (all other embedding tests migrated to +``Py_InitializeFromConfig`` in Python 3.11) + +.. + +.. bpo: 34272 +.. date: 2018-07-29-15-59-51 +.. nonce: lVX2uR +.. section: Tests + +Some C API tests were moved into the new Lib/test/test_capi/ directory. + +.. + +.. date: 2022-11-24-02-58-10 +.. gh-issue: 99086 +.. nonce: DV_4Br +.. section: Build + +Fix ``-Wimplicit-int``, ``-Wstrict-prototypes``, and +``-Wimplicit-function-declaration`` compiler warnings in +:program:`configure` checks. + +.. + +.. date: 2022-11-15-08-40-22 +.. gh-issue: 99337 +.. nonce: 5LoQDE +.. section: Build + +Fix a compilation issue with GCC 12 on macOS. + +.. + +.. date: 2022-11-04-02-58-10 +.. gh-issue: 99086 +.. nonce: DV_4Br +.. section: Build + +Fix ``-Wimplicit-int`` compiler warning in :program:`configure` check for +``PTHREAD_SCOPE_SYSTEM``. + +.. + +.. date: 2022-11-03-08-10-49 +.. gh-issue: 98872 +.. nonce: gdsR8X +.. section: Build + +Fix a possible fd leak in ``Programs/_freeze_module.c`` introduced in Python +3.11. + +.. + +.. date: 2022-11-02-19-25-07 +.. gh-issue: 99016 +.. nonce: R05NkD +.. section: Build + +Fix build with ``PYTHON_FOR_REGEN=python3.8``. + +.. + +.. date: 2022-11-02-18-45-35 +.. gh-issue: 97731 +.. nonce: zKpTlj +.. section: Build + +Specify the full path to the source location for ``make docclean`` (needed +for cross-builds). + +.. + +.. date: 2022-10-26-12-37-52 +.. gh-issue: 98707 +.. nonce: eVXGEx +.. section: Build + +Don't use vendored ``libmpdec`` headers if :option:`--with-system-libmpdec` +is passed to :program:`configure`. Don't use vendored ``libexpat`` headers +if :option:`--with-system-expat` is passed to :program:`!configure`. + +.. + +.. date: 2022-09-20-12-43-44 +.. gh-issue: 96761 +.. nonce: IF29kR +.. section: Build + +Fix the build process of clang compiler for :program:`_bootstrap_python` if +LTO optimization is applied. Patch by Matthias Görgens and Dong-hee Na. + +.. + +.. date: 2022-09-17-11-19-24 +.. gh-issue: 96883 +.. nonce: p_gr62 +.. section: Build + +``wasm32-emscripten`` builds for browsers now include +:mod:`concurrent.futures` for :mod:`asyncio` and :mod:`unittest.mock`. + +.. + +.. date: 2022-08-26-11-09-11 +.. gh-issue: 84461 +.. nonce: Nsdn_R +.. section: Build + +``wasm32-emscripten`` platform no longer builds :mod:`resource` module, +:func:`~os.getresuid`, :func:`~os.getresgid`, and their setters. The APIs +are stubs and not functional. + +.. + +.. date: 2022-06-25-23-25-47 +.. gh-issue: 94280 +.. nonce: YhEyW_ +.. section: Build + +Updated pegen regeneration script on Windows to find and use Python 3.9 or +higher. Prior to this, pegen regeneration already required 3.9 or higher, +but the script may have used lower versions of Python. + +.. + +.. date: 2022-11-23-17-17-16 +.. gh-issue: 99345 +.. nonce: jOa3-f +.. section: Windows + +Use faster initialization functions to detect install location for Windows +Store package + +.. + +.. date: 2022-11-21-19-50-18 +.. gh-issue: 98629 +.. nonce: tMmB_B +.. section: Windows + +Fix initialization of :data:`sys.version` and ``sys._git`` on Windows + +.. + +.. date: 2022-11-16-19-03-21 +.. gh-issue: 99442 +.. nonce: 6Dgk3Q +.. section: Windows + +Fix handling in :ref:`launcher` when ``argv[0]`` does not include a file +extension. + +.. + +.. date: 2022-11-01-11-07-33 +.. gh-issue: 98689 +.. nonce: 0f6e_N +.. section: Windows + +Update Windows builds to zlib v1.2.13. v1.2.12 has CVE-2022-37434, but the +vulnerable ``inflateGetHeader`` API is not used by Python. + +.. + +.. date: 2022-11-01-00-37-13 +.. gh-issue: 98790 +.. nonce: fpaPAx +.. section: Windows + +Assumes that a missing ``DLLs`` directory means that standard extension +modules are in the executable's directory. + +.. + +.. date: 2022-10-27-20-30-16 +.. gh-issue: 98745 +.. nonce: v06p4r +.. section: Windows + +Update :file:`py.exe` launcher to install 3.11 by default and 3.12 on +request. + +.. + +.. date: 2022-10-26-17-43-09 +.. gh-issue: 98692 +.. nonce: bOopfZ +.. section: Windows + +Fix the :ref:`launcher` ignoring unrecognized shebang lines instead of +treating them as local paths + +.. + +.. date: 2022-10-25-10-34-17 +.. gh-issue: 94328 +.. nonce: 19NhdU +.. section: Windows + +Update Windows installer to use SQLite 3.39.4. + +.. + +.. date: 2022-10-02-11-59-23 +.. gh-issue: 97728 +.. nonce: dIdlPE +.. section: Windows + +Fix possible crashes caused by the use of uninitialized variables when pass +invalid arguments in :func:`os.system` on Windows and in Windows-specific +modules (like ``winreg``). + +.. + +.. date: 2022-09-23-15-40-04 +.. gh-issue: 96965 +.. nonce: CsnEGs +.. section: Windows + +Update libffi to 3.4.3 + +.. + +.. date: 2022-08-30-12-01-51 +.. gh-issue: 94781 +.. nonce: OxO-Gr +.. section: Windows + +Fix :file:`pcbuild.proj` to clean previous instances of ouput files in +``Python\deepfreeze`` and ``Python\frozen_modules`` directories on Windows. +Patch by Charlie Zhao. + +.. + +.. bpo: 40882 +.. date: 2020-06-06-15-10-37 +.. nonce: UvNbdj +.. section: Windows + +Fix a memory leak in :class:`multiprocessing.shared_memory.SharedMemory` on +Windows. + +.. + +.. date: 2022-11-25-09-23-20 +.. gh-issue: 87235 +.. nonce: SifjCD +.. section: macOS + +On macOS ``python3 /dev/fd/9 9`_ +:mod:`inspect` docs' :ref:`inspect-types` table. .. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 50dbb0c7bcffce..ebae4e44005cc0 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2271,6 +2271,50 @@ [function.PyMemoryView_FromBuffer] added = '3.11' +# Constants for Py_buffer API added to this list in Python 3.11.1 (https://github.com/python/cpython/issues/98680) +# (they were available with 3.11.0) +[const.PyBUF_MAX_NDIM] + added = '3.11' +[const.PyBUF_SIMPLE] + added = '3.11' +[const.PyBUF_WRITABLE] + added = '3.11' +[const.PyBUF_FORMAT] + added = '3.11' +[const.PyBUF_ND] + added = '3.11' +[const.PyBUF_STRIDES] + added = '3.11' +[const.PyBUF_C_CONTIGUOUS] + added = '3.11' +[const.PyBUF_F_CONTIGUOUS] + added = '3.11' +[const.PyBUF_ANY_CONTIGUOUS] + added = '3.11' +[const.PyBUF_INDIRECT] + added = '3.11' +[const.PyBUF_CONTIG] + added = '3.11' +[const.PyBUF_CONTIG_RO] + added = '3.11' +[const.PyBUF_STRIDED] + added = '3.11' +[const.PyBUF_STRIDED_RO] + added = '3.11' +[const.PyBUF_RECORDS] + added = '3.11' +[const.PyBUF_RECORDS_RO] + added = '3.11' +[const.PyBUF_FULL] + added = '3.11' +[const.PyBUF_FULL_RO] + added = '3.11' +[const.PyBUF_READ] + added = '3.11' +[const.PyBUF_WRITE] + added = '3.11' + + # (Detailed comments aren't really needed for further entries: from here on # we can use version control logs.) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index e51f97f4659076..40f1f80be4473a 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -339,13 +339,6 @@ get_event_loop(int stacklevel) return loop; } - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "There is no current event loop", - stacklevel)) - { - return NULL; - } - policy = PyObject_CallNoArgs(asyncio_get_event_loop_policy); if (policy == NULL) { return NULL; @@ -1052,7 +1045,11 @@ _asyncio_Future_remove_done_callback(FutureObj *self, PyObject *fn) return NULL; } - for (i = 0; i < PyList_GET_SIZE(self->fut_callbacks); i++) { + // Beware: PyObject_RichCompareBool below may change fut_callbacks. + // See GH-97592. + for (i = 0; + self->fut_callbacks != NULL && i < PyList_GET_SIZE(self->fut_callbacks); + i++) { int ret; PyObject *item = PyList_GET_ITEM(self->fut_callbacks, i); Py_INCREF(item); @@ -1071,7 +1068,8 @@ _asyncio_Future_remove_done_callback(FutureObj *self, PyObject *fn) } } - if (j == 0) { + // Note: fut_callbacks may have been cleared. + if (j == 0 || self->fut_callbacks == NULL) { Py_CLEAR(self->fut_callbacks); Py_DECREF(newlist); return PyLong_FromSsize_t(len + cleared_callback0); @@ -3124,6 +3122,11 @@ _asyncio_get_event_loop_impl(PyObject *module) return get_event_loop(1); } +// This internal method is going away in Python 3.12, left here only for +// backwards compatibility with 3.10.0 - 3.10.8 and 3.11.0. +// Similarly, this method's Python equivalent in asyncio.events is going +// away as well. +// See GH-99949 for more details. /*[clinic input] _asyncio._get_event_loop stacklevel: int = 3 diff --git a/Modules/_csv.c b/Modules/_csv.c index cbf4c5de51968e..7519da6807fe3e 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -704,7 +704,7 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) self->state = ESCAPED_CHAR; } else if (c == ' ' && dialect->skipinitialspace) - /* ignore space at start of field */ + /* ignore spaces at start of field */ ; else if (c == dialect->delimiter) { /* save empty field */ @@ -1647,9 +1647,9 @@ PyDoc_STRVAR(csv_module_doc, " quoting character. It defaults to '\"'.\n" " * delimiter - specifies a one-character string to use as the\n" " field separator. It defaults to ','.\n" -" * skipinitialspace - specifies how to interpret whitespace which\n" -" immediately follows a delimiter. It defaults to False, which\n" -" means that whitespace immediately following a delimiter is part\n" +" * skipinitialspace - specifies how to interpret spaces which\n" +" immediately follow a delimiter. It defaults to False, which\n" +" means that spaces immediately following a delimiter is part\n" " of the following field.\n" " * lineterminator - specifies the character sequence which should\n" " terminate rows.\n" diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 95b0912aecd75c..c28762d49ba497 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -405,9 +405,15 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, "ffi_prep_cif failed with %d", result); goto error; } + + #if HAVE_FFI_PREP_CLOSURE_LOC # ifdef USING_APPLE_OS_LIBFFI +# ifdef HAVE_BUILTIN_AVAILABLE # define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# else +# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME (ffi_prep_closure_loc != NULL) +# endif # else # define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME 1 # endif diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 3fab9ad0c1e7bd..f42ff08f58bbcb 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -96,6 +96,7 @@ #define CTYPES_CAPSULE_NAME_PYMEM "_ctypes pymem" + static void pymem_destructor(PyObject *ptr) { void *p = PyCapsule_GetPointer(ptr, CTYPES_CAPSULE_NAME_PYMEM); @@ -829,7 +830,11 @@ static int _call_function_pointer(int flags, #endif # ifdef USING_APPLE_OS_LIBFFI +# ifdef HAVE_BUILTIN_AVAILABLE # define HAVE_FFI_PREP_CIF_VAR_RUNTIME __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# else +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME (ffi_prep_cif_var != NULL) +# endif # elif HAVE_FFI_PREP_CIF_VAR # define HAVE_FFI_PREP_CIF_VAR_RUNTIME true # else @@ -1016,7 +1021,10 @@ void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...) PyErr_Fetch(&tp, &v, &tb); PyErr_NormalizeException(&tp, &v, &tb); - cls_str = PyObject_Str(tp); + if (PyType_Check(tp)) + cls_str = PyType_GetName((PyTypeObject *)tp); + else + cls_str = PyObject_Str(tp); if (cls_str) { PyUnicode_AppendAndDel(&s, cls_str); PyUnicode_AppendAndDel(&s, PyUnicode_FromString(": ")); @@ -1439,8 +1447,13 @@ copy_com_pointer(PyObject *self, PyObject *args) #else #ifdef __APPLE__ #ifdef HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH -#define HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH_RUNTIME \ - __builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) +# ifdef HAVE_BUILTIN_AVAILABLE +# define HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH_RUNTIME \ + __builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) +# else +# define HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH_RUNTIME \ + (_dyld_shared_cache_contains_path != NULL) +# endif #else // Support the deprecated case of compiling on an older macOS version static void *libsystem_b_handle; diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 88eb9f59922a04..a7029b6e6da2b8 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -26,6 +26,12 @@ #endif #endif +#if defined(__has_builtin) +#if __has_builtin(__builtin_available) +#define HAVE_BUILTIN_AVAILABLE 1 +#endif +#endif + typedef struct tagPyCArgObject PyCArgObject; typedef struct tagCDataObject CDataObject; typedef PyObject *(* GETFUNC)(void *, Py_ssize_t size); diff --git a/Modules/_ctypes/malloc_closure.c b/Modules/_ctypes/malloc_closure.c index 38edc90e707639..108660c967b8ed 100644 --- a/Modules/_ctypes/malloc_closure.c +++ b/Modules/_ctypes/malloc_closure.c @@ -20,6 +20,7 @@ /* #define MALLOC_CLOSURE_DEBUG */ /* enable for some debugging output */ + /******************************************************************/ typedef union _tagITEM { @@ -93,7 +94,11 @@ void Py_ffi_closure_free(void *p) { #ifdef HAVE_FFI_CLOSURE_ALLOC #ifdef USING_APPLE_OS_LIBFFI +# ifdef HAVE_BUILTIN_AVAILABLE if (__builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)) { +# else + if (ffi_closure_free != NULL) { +# endif #endif ffi_closure_free(p); return; @@ -111,7 +116,11 @@ void *Py_ffi_closure_alloc(size_t size, void** codeloc) { #ifdef HAVE_FFI_CLOSURE_ALLOC #ifdef USING_APPLE_OS_LIBFFI +# ifdef HAVE_BUILTIN_AVAILABLE if (__builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)) { +# else + if (ffi_closure_alloc != NULL) { +# endif #endif return ffi_closure_alloc(size, codeloc); #ifdef USING_APPLE_OS_LIBFFI diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index a819ce910d4b52..88999b8dff1838 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -430,8 +430,11 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct } stgdict = PyType_stgdict(type); - if (!stgdict) + if (!stgdict) { + PyErr_SetString(PyExc_TypeError, + "ctypes state is not initialized"); return -1; + } /* If this structure/union is already marked final we cannot assign _fields_ anymore. */ diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 38ef24637b7318..a7b2e984310d18 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -703,10 +703,10 @@ PyInit__io(void) goto fail; /* BlockingIOError, for compatibility */ - Py_INCREF(PyExc_BlockingIOError); - if (PyModule_AddObject(m, "BlockingIOError", - (PyObject *) PyExc_BlockingIOError) < 0) + if (PyModule_AddObjectRef(m, "BlockingIOError", + (PyObject *) PyExc_BlockingIOError) < 0) { goto fail; + } // Set type base classes PyFileIO_Type.tp_base = &PyRawIOBase_Type; diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 8b1cff56d75fa3..4496609afcbc26 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -492,8 +492,12 @@ _Py_COMP_DIAG_POP ret = -1; if (!fd_is_own) self->fd = -1; - if (self->fd >= 0) + if (self->fd >= 0) { + PyObject *exc, *val, *tb; + PyErr_Fetch(&exc, &val, &tb); internal_close(self); + _PyErr_ChainExceptions(exc, val, tb); + } done: #ifdef MS_WINDOWS diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 3369694653fd96..6cb5a6861a158c 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -231,17 +231,16 @@ _io_IncrementalNewlineDecoder___init___impl(nldecoder_object *self, PyObject *errors) /*[clinic end generated code: output=fbd04d443e764ec2 input=89db6b19c6b126bf]*/ { - self->decoder = decoder; - Py_INCREF(decoder); if (errors == NULL) { - self->errors = &_Py_ID(strict); + errors = Py_NewRef(&_Py_ID(strict)); } else { - self->errors = errors; + errors = Py_NewRef(errors); } - Py_INCREF(self->errors); + Py_XSETREF(self->errors, errors); + Py_XSETREF(self->decoder, Py_NewRef(decoder)); self->translate = translate ? 1 : 0; self->seennl = 0; self->pendingcr = 0; @@ -276,6 +275,13 @@ check_decoded(PyObject *decoded) return 0; } +#define CHECK_INITIALIZED_DECODER(self) \ + if (self->errors == NULL) { \ + PyErr_SetString(PyExc_ValueError, \ + "IncrementalNewlineDecoder.__init__() not called"); \ + return NULL; \ + } + #define SEEN_CR 1 #define SEEN_LF 2 #define SEEN_CRLF 4 @@ -289,11 +295,7 @@ _PyIncrementalNewlineDecoder_decode(PyObject *myself, Py_ssize_t output_len; nldecoder_object *self = (nldecoder_object *) myself; - if (self->decoder == NULL) { - PyErr_SetString(PyExc_ValueError, - "IncrementalNewlineDecoder.__init__ not called"); - return NULL; - } + CHECK_INITIALIZED_DECODER(self); /* decode input (with the eventual \r from a previous pass) */ if (self->decoder != Py_None) { @@ -507,6 +509,8 @@ _io_IncrementalNewlineDecoder_getstate_impl(nldecoder_object *self) PyObject *buffer; unsigned long long flag; + CHECK_INITIALIZED_DECODER(self); + if (self->decoder != Py_None) { PyObject *state = PyObject_CallMethodNoArgs(self->decoder, &_Py_ID(getstate)); @@ -551,6 +555,8 @@ _io_IncrementalNewlineDecoder_setstate(nldecoder_object *self, PyObject *buffer; unsigned long long flag; + CHECK_INITIALIZED_DECODER(self); + if (!PyTuple_Check(state)) { PyErr_SetString(PyExc_TypeError, "state argument must be a tuple"); return NULL; @@ -581,6 +587,8 @@ static PyObject * _io_IncrementalNewlineDecoder_reset_impl(nldecoder_object *self) /*[clinic end generated code: output=32fa40c7462aa8ff input=728678ddaea776df]*/ { + CHECK_INITIALIZED_DECODER(self); + self->seennl = 0; self->pendingcr = 0; if (self->decoder != Py_None) @@ -592,6 +600,8 @@ _io_IncrementalNewlineDecoder_reset_impl(nldecoder_object *self) static PyObject * incrementalnewlinedecoder_newlines_get(nldecoder_object *self, void *context) { + CHECK_INITIALIZED_DECODER(self); + switch (self->seennl) { case SEEN_CR: return PyUnicode_FromString("\r"); diff --git a/Modules/_pickle.c b/Modules/_pickle.c index b4b1dda199c23a..721079c91f5f2c 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -7998,16 +7998,15 @@ PyInit__pickle(void) if (st->UnpicklingError == NULL) return NULL; - Py_INCREF(st->PickleError); - if (PyModule_AddObject(m, "PickleError", st->PickleError) < 0) + if (PyModule_AddObjectRef(m, "PickleError", st->PickleError) < 0) { return NULL; - Py_INCREF(st->PicklingError); - if (PyModule_AddObject(m, "PicklingError", st->PicklingError) < 0) + } + if (PyModule_AddObjectRef(m, "PicklingError", st->PicklingError) < 0) { return NULL; - Py_INCREF(st->UnpicklingError); - if (PyModule_AddObject(m, "UnpicklingError", st->UnpicklingError) < 0) + } + if (PyModule_AddObjectRef(m, "UnpicklingError", st->UnpicklingError) < 0) { return NULL; - + } if (_Pickle_InitState(st) < 0) return NULL; diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 0a7019a085b97d..448e761c988ca8 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -1528,7 +1528,7 @@ _sre_compile_impl(PyObject *module, PyObject *pattern, int flags, #endif /* Report failure */ -#define FAIL do { VTRACE(("FAIL: %d\n", __LINE__)); return 0; } while (0) +#define FAIL do { VTRACE(("FAIL: %d\n", __LINE__)); return -1; } while (0) /* Extract opcode, argument, or skip count from code array */ #define GET_OP \ @@ -1552,7 +1552,7 @@ _sre_compile_impl(PyObject *module, PyObject *pattern, int flags, skip = *code; \ VTRACE(("%lu (skip to %p)\n", \ (unsigned long)skip, code+skip)); \ - if (skip-adj > (uintptr_t)(end - code)) \ + if (skip-adj > (uintptr_t)(end - code)) \ FAIL; \ code++; \ } while (0) @@ -1641,9 +1641,10 @@ _validate_charset(SRE_CODE *code, SRE_CODE *end) } } - return 1; + return 0; } +/* Returns 0 on success, -1 on failure, and 1 if the last op is JUMP. */ static int _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) { @@ -1721,7 +1722,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) case SRE_OP_IN_LOC_IGNORE: GET_SKIP; /* Stop 1 before the end; we check the FAILURE below */ - if (!_validate_charset(code, code+skip-2)) + if (_validate_charset(code, code+skip-2)) FAIL; if (code[skip-2] != SRE_OP_FAILURE) FAIL; @@ -1775,7 +1776,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) } /* Validate the charset */ if (flags & SRE_INFO_CHARSET) { - if (!_validate_charset(code, newcode-1)) + if (_validate_charset(code, newcode-1)) FAIL; if (newcode[-1] != SRE_OP_FAILURE) FAIL; @@ -1796,7 +1797,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) if (skip == 0) break; /* Stop 2 before the end; we check the JUMP below */ - if (!_validate_inner(code, code+skip-3, groups)) + if (_validate_inner(code, code+skip-3, groups)) FAIL; code += skip-3; /* Check that it ends with a JUMP, and that each JUMP @@ -1810,6 +1811,8 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) else if (code+skip-1 != target) FAIL; } + if (code != target) + FAIL; } break; @@ -1825,7 +1828,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) FAIL; if (max > SRE_MAXREPEAT) FAIL; - if (!_validate_inner(code, code+skip-4, groups)) + if (_validate_inner(code, code+skip-4, groups)) FAIL; code += skip-4; GET_OP; @@ -1845,7 +1848,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) FAIL; if (max > SRE_MAXREPEAT) FAIL; - if (!_validate_inner(code, code+skip-3, groups)) + if (_validate_inner(code, code+skip-3, groups)) FAIL; code += skip-3; GET_OP; @@ -1863,7 +1866,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) case SRE_OP_ATOMIC_GROUP: { GET_SKIP; - if (!_validate_inner(code, code+skip-2, groups)) + if (_validate_inner(code, code+skip-2, groups)) FAIL; code += skip-2; GET_OP; @@ -1915,24 +1918,17 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) to allow arbitrary jumps anywhere in the code; so we just look for a JUMP opcode preceding our skip target. */ - if (skip >= 3 && skip-3 < (uintptr_t)(end - code) && - code[skip-3] == SRE_OP_JUMP) - { - VTRACE(("both then and else parts present\n")); - if (!_validate_inner(code+1, code+skip-3, groups)) - FAIL; + VTRACE(("then part:\n")); + int rc = _validate_inner(code+1, code+skip-1, groups); + if (rc == 1) { + VTRACE(("else part:\n")); code += skip-2; /* Position after JUMP, at */ GET_SKIP; - if (!_validate_inner(code, code+skip-1, groups)) - FAIL; - code += skip-1; - } - else { - VTRACE(("only a then part present\n")); - if (!_validate_inner(code+1, code+skip-1, groups)) - FAIL; - code += skip-1; + rc = _validate_inner(code, code+skip-1, groups); } + if (rc) + FAIL; + code += skip-1; break; case SRE_OP_ASSERT: @@ -1943,7 +1939,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) if (arg & 0x80000000) FAIL; /* Width too large */ /* Stop 1 before the end; we check the SUCCESS below */ - if (!_validate_inner(code+1, code+skip-2, groups)) + if (_validate_inner(code+1, code+skip-2, groups)) FAIL; code += skip-2; GET_OP; @@ -1951,6 +1947,12 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) FAIL; break; + case SRE_OP_JUMP: + if (code + 1 != end) + FAIL; + VTRACE(("JUMP: %d\n", __LINE__)); + return 1; + default: FAIL; @@ -1958,7 +1960,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) } VTRACE(("okay\n")); - return 1; + return 0; } static int @@ -1973,7 +1975,7 @@ _validate_outer(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) static int _validate(PatternObject *self) { - if (!_validate_outer(self->code, self->code+self->codesize, self->groups)) + if (_validate_outer(self->code, self->code+self->codesize, self->groups)) { PyErr_SetString(PyExc_RuntimeError, "invalid SRE code"); return 0; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 43fec8138a0ebc..1291eff481cbb8 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5348,6 +5348,71 @@ get_mapping_items(PyObject* self, PyObject *obj) return PyMapping_Items(obj); } +static PyObject * +test_mapping_has_key_string(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *context = PyDict_New(); + PyObject *val = PyLong_FromLong(1); + + // Since this uses `const char*` it is easier to test this in C: + PyDict_SetItemString(context, "a", val); + if (!PyMapping_HasKeyString(context, "a")) { + PyErr_SetString(PyExc_RuntimeError, + "Existing mapping key does not exist"); + return NULL; + } + if (PyMapping_HasKeyString(context, "b")) { + PyErr_SetString(PyExc_RuntimeError, + "Missing mapping key exists"); + return NULL; + } + + Py_DECREF(val); + Py_DECREF(context); + Py_RETURN_NONE; +} + +static PyObject * +mapping_has_key(PyObject* self, PyObject *args) +{ + PyObject *context, *key; + if (!PyArg_ParseTuple(args, "OO", &context, &key)) { + return NULL; + } + return PyLong_FromLong(PyMapping_HasKey(context, key)); +} + +static PyObject * +sequence_set_slice(PyObject* self, PyObject *args) +{ + PyObject *sequence, *obj; + Py_ssize_t i1, i2; + if (!PyArg_ParseTuple(args, "OnnO", &sequence, &i1, &i2, &obj)) { + return NULL; + } + + int res = PySequence_SetSlice(sequence, i1, i2, obj); + if (res == -1) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +sequence_del_slice(PyObject* self, PyObject *args) +{ + PyObject *sequence; + Py_ssize_t i1, i2; + if (!PyArg_ParseTuple(args, "Onn", &sequence, &i1, &i2)) { + return NULL; + } + + int res = PySequence_DelSlice(sequence, i1, i2); + if (res == -1) { + return NULL; + } + Py_RETURN_NONE; +} static PyObject * test_pythread_tss_key_state(PyObject *self, PyObject *args) @@ -5935,6 +6000,18 @@ frame_getlasti(PyObject *self, PyObject *frame) return PyLong_FromLong(lasti); } +static PyObject * +eval_get_func_name(PyObject *self, PyObject *func) +{ + return PyUnicode_FromString(PyEval_GetFuncName(func)); +} + +static PyObject * +eval_get_func_desc(PyObject *self, PyObject *func) +{ + return PyUnicode_FromString(PyEval_GetFuncDesc(func)); +} + static PyObject * get_feature_macros(PyObject *self, PyObject *Py_UNUSED(args)) { @@ -6075,6 +6152,43 @@ settrace_to_record(PyObject *self, PyObject *list) } static PyObject *negative_dictoffset(PyObject *, PyObject *); + +static PyObject * +function_get_code(PyObject *self, PyObject *func) +{ + PyObject *code = PyFunction_GetCode(func); + if (code != NULL) { + Py_INCREF(code); + return code; + } else { + return NULL; + } +} + +static PyObject * +function_get_globals(PyObject *self, PyObject *func) +{ + PyObject *globals = PyFunction_GetGlobals(func); + if (globals != NULL) { + Py_INCREF(globals); + return globals; + } else { + return NULL; + } +} + +static PyObject * +function_get_module(PyObject *self, PyObject *func) +{ + PyObject *module = PyFunction_GetModule(func); + if (module != NULL) { + Py_INCREF(module); + return module; + } else { + return NULL; + } +} + static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyObject *getargs_s_hash_int(PyObject *, PyObject *, PyObject*); static PyObject *getargs_s_hash_int2(PyObject *, PyObject *, PyObject*); @@ -6339,6 +6453,10 @@ static PyMethodDef TestMethods[] = { {"get_mapping_keys", get_mapping_keys, METH_O}, {"get_mapping_values", get_mapping_values, METH_O}, {"get_mapping_items", get_mapping_items, METH_O}, + {"test_mapping_has_key_string", test_mapping_has_key_string, METH_NOARGS}, + {"mapping_has_key", mapping_has_key, METH_VARARGS}, + {"sequence_set_slice", sequence_set_slice, METH_VARARGS}, + {"sequence_del_slice", sequence_del_slice, METH_VARARGS}, {"test_pythread_tss_key_state", test_pythread_tss_key_state, METH_VARARGS}, {"hamt", new_hamt, METH_NOARGS}, {"bad_get", _PyCFunction_CAST(bad_get), METH_FASTCALL}, @@ -6372,9 +6490,14 @@ static PyMethodDef TestMethods[] = { {"frame_getgenerator", frame_getgenerator, METH_O, NULL}, {"frame_getbuiltins", frame_getbuiltins, METH_O, NULL}, {"frame_getlasti", frame_getlasti, METH_O, NULL}, + {"eval_get_func_name", eval_get_func_name, METH_O, NULL}, + {"eval_get_func_desc", eval_get_func_desc, METH_O, NULL}, {"get_feature_macros", get_feature_macros, METH_NOARGS, NULL}, {"test_code_api", test_code_api, METH_NOARGS, NULL}, {"settrace_to_record", settrace_to_record, METH_O, NULL}, + {"function_get_code", function_get_code, METH_O, NULL}, + {"function_get_globals", function_get_globals, METH_O, NULL}, + {"function_get_module", function_get_module, METH_O, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 9b30a900326192..f6bb07fd8b06ef 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1402,6 +1402,30 @@ _winapi_MapViewOfFile_impl(PyObject *module, HANDLE file_map, return address; } +/*[clinic input] +_winapi.UnmapViewOfFile + + address: LPCVOID + / +[clinic start generated code]*/ + +static PyObject * +_winapi_UnmapViewOfFile_impl(PyObject *module, LPCVOID address) +/*[clinic end generated code: output=4f7e18ac75d19744 input=8c4b6119ad9288a3]*/ +{ + BOOL success; + + Py_BEGIN_ALLOW_THREADS + success = UnmapViewOfFile(address); + Py_END_ALLOW_THREADS + + if (!success) { + return PyErr_SetFromWindowsErr(0); + } + + Py_RETURN_NONE; +} + /*[clinic input] _winapi.OpenFileMapping -> HANDLE @@ -2095,6 +2119,7 @@ static PyMethodDef winapi_functions[] = { _WINAPI_READFILE_METHODDEF _WINAPI_SETNAMEDPIPEHANDLESTATE_METHODDEF _WINAPI_TERMINATEPROCESS_METHODDEF + _WINAPI_UNMAPVIEWOFFILE_METHODDEF _WINAPI_VIRTUALQUERYSIZE_METHODDEF _WINAPI_WAITNAMEDPIPE_METHODDEF _WINAPI_WAITFORMULTIPLEOBJECTS_METHODDEF diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 1535721b026d1f..55975f4e269109 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -2652,8 +2652,9 @@ zoneinfomodule_exec(PyObject *m) goto error; } - Py_INCREF(&PyZoneInfo_ZoneInfoType); - PyModule_AddObject(m, "ZoneInfo", (PyObject *)&PyZoneInfo_ZoneInfoType); + if (PyModule_AddObjectRef(m, "ZoneInfo", (PyObject *)&PyZoneInfo_ZoneInfoType) < 0) { + goto error; + } /* Populate imports */ PyObject *_tzpath_module = PyImport_ImportModule("zoneinfo._tzpath"); diff --git a/Modules/audioop.c b/Modules/audioop.c index d74e634ac44889..b764dd97d535a8 100644 --- a/Modules/audioop.c +++ b/Modules/audioop.c @@ -1,3 +1,33 @@ +/* The audioop module uses the code base in g777.c file of the Sox project. + * Source: https://web.archive.org/web/19970716121258/http://www.spies.com/Sox/Archive/soxgamma.tar.gz + * Programming the AdLib/Sound Blaster + * FM Music Chips + * Version 2.0 (24 Feb 1992) + * + * Copyright (c) 1991, 1992 by Jeffrey S. Lee + * + * jlee@smylex.uucp + * + * + * + * Warranty and Copyright Policy + * + * This document is provided on an "as-is" basis, and its author makes + * no warranty or representation, express or implied, with respect to + * its quality performance or fitness for a particular purpose. In no + * event will the author of this document be liable for direct, indirect, + * special, incidental, or consequential damages arising out of the use + * or inability to use the information contained within. Use of this + * document is at your own risk. + * + * This file may be used and copied freely so long as the applicable + * copyright notices are retained, and no modifications are made to the + * text of the document. No money shall be charged for its distribution + * beyond reasonable shipping, handling and duplication costs, nor shall + * proprietary changes be made to this document so that it cannot be + * distributed freely. This document may not be included in published + * material or commercial packages without the written consent of its + * author. */ /* audioopmodule - Module to detect peak values in arrays */ @@ -28,20 +58,6 @@ fbound(double val, double minval, double maxval) } -/* Code shamelessly stolen from sox, 12.17.7, g711.c -** (c) Craig Reese, Joe Campbell and Jeff Poskanzer 1989 */ - -/* From g711.c: - * - * December 30, 1994: - * Functions linear2alaw, linear2ulaw have been updated to correctly - * convert unquantized 16 bit values. - * Tables for direct u- to A-law and A- to u-law conversions have been - * corrected. - * Borge Lindberg, Center for PersonKommunikation, Aalborg University. - * bli@cpk.auc.dk - * - */ #define BIAS 0x84 /* define the add-in bias for 16 bit samples */ #define CLIP 32635 #define SIGN_BIT (0x80) /* Sign bit for an A-law byte. */ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 87f624f9816de3..5364d9a2d6ef78 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -192,7 +192,7 @@ _winapi_CreateFileMapping(PyObject *module, PyObject *const *args, Py_ssize_t na DWORD protect; DWORD max_size_high; DWORD max_size_low; - LPCWSTR name; + LPCWSTR name = NULL; HANDLE _return_value; if (!_PyArg_ParseStack(args, nargs, "" F_HANDLE "" F_POINTER "kkkO&:CreateFileMapping", @@ -233,8 +233,8 @@ static PyObject * _winapi_CreateJunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - LPCWSTR src_path; - LPCWSTR dst_path; + LPCWSTR src_path = NULL; + LPCWSTR dst_path = NULL; if (!_PyArg_CheckPositional("CreateJunction", nargs, 2, 2)) { goto exit; @@ -394,14 +394,14 @@ static PyObject * _winapi_CreateProcess(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - const Py_UNICODE *application_name; + const Py_UNICODE *application_name = NULL; PyObject *command_line; PyObject *proc_attrs; PyObject *thread_attrs; BOOL inherit_handles; DWORD creation_flags; PyObject *env_mapping; - const Py_UNICODE *current_directory; + const Py_UNICODE *current_directory = NULL; PyObject *startup_info; if (!_PyArg_ParseStack(args, nargs, "O&OOOikOO&O:CreateProcess", @@ -731,6 +731,32 @@ _winapi_MapViewOfFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(_winapi_UnmapViewOfFile__doc__, +"UnmapViewOfFile($module, address, /)\n" +"--\n" +"\n"); + +#define _WINAPI_UNMAPVIEWOFFILE_METHODDEF \ + {"UnmapViewOfFile", (PyCFunction)_winapi_UnmapViewOfFile, METH_O, _winapi_UnmapViewOfFile__doc__}, + +static PyObject * +_winapi_UnmapViewOfFile_impl(PyObject *module, LPCVOID address); + +static PyObject * +_winapi_UnmapViewOfFile(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + LPCVOID address; + + if (!PyArg_Parse(arg, "" F_POINTER ":UnmapViewOfFile", &address)) { + goto exit; + } + return_value = _winapi_UnmapViewOfFile_impl(module, address); + +exit: + return return_value; +} + PyDoc_STRVAR(_winapi_OpenFileMapping__doc__, "OpenFileMapping($module, desired_access, inherit_handle, name, /)\n" "--\n" @@ -749,7 +775,7 @@ _winapi_OpenFileMapping(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *return_value = NULL; DWORD desired_access; BOOL inherit_handle; - LPCWSTR name; + LPCWSTR name = NULL; HANDLE _return_value; if (!_PyArg_ParseStack(args, nargs, "kiO&:OpenFileMapping", @@ -1216,4 +1242,4 @@ _winapi__mimetypes_read_windows_registry(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=dfbccec8f11b7433 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9c08a7371fcf5dd4 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/overlapped.c.h b/Modules/clinic/overlapped.c.h index 2861338317497c..6673cbfe462379 100644 --- a/Modules/clinic/overlapped.c.h +++ b/Modules/clinic/overlapped.c.h @@ -220,7 +220,7 @@ _overlapped_CreateEvent(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *EventAttributes; BOOL ManualReset; BOOL InitialState; - const Py_UNICODE *Name; + const Py_UNICODE *Name = NULL; if (!_PyArg_ParseStack(args, nargs, "OiiO&:CreateEvent", &EventAttributes, &ManualReset, &InitialState, _PyUnicode_WideCharString_Opt_Converter, &Name)) { @@ -806,7 +806,7 @@ static PyObject * _overlapped_Overlapped_ConnectPipe(OverlappedObject *self, PyObject *arg) { PyObject *return_value = NULL; - const Py_UNICODE *Address; + const Py_UNICODE *Address = NULL; if (!PyUnicode_Check(arg)) { _PyArg_BadArgument("ConnectPipe", "argument", "str", arg); @@ -851,8 +851,8 @@ _overlapped_WSAConnect(PyObject *module, PyObject *const *args, Py_ssize_t nargs HANDLE ConnectSocket; PyObject *AddressObj; - if (!_PyArg_ParseStack(args, nargs, ""F_HANDLE"O:WSAConnect", - &ConnectSocket, &AddressObj)) { + if (!_PyArg_ParseStack(args, nargs, ""F_HANDLE"O!:WSAConnect", + &ConnectSocket, &PyTuple_Type, &AddressObj)) { goto exit; } return_value = _overlapped_WSAConnect_impl(module, ConnectSocket, AddressObj); @@ -884,8 +884,8 @@ _overlapped_Overlapped_WSASendTo(OverlappedObject *self, PyObject *const *args, DWORD flags; PyObject *AddressObj; - if (!_PyArg_ParseStack(args, nargs, ""F_HANDLE"y*kO:WSASendTo", - &handle, &bufobj, &flags, &AddressObj)) { + if (!_PyArg_ParseStack(args, nargs, ""F_HANDLE"y*kO!:WSASendTo", + &handle, &bufobj, &flags, &PyTuple_Type, &AddressObj)) { goto exit; } return_value = _overlapped_Overlapped_WSASendTo_impl(self, handle, &bufobj, flags, AddressObj); @@ -968,4 +968,4 @@ _overlapped_Overlapped_WSARecvFromInto(OverlappedObject *self, PyObject *const * return return_value; } -/*[clinic end generated code: output=b0f15f5c09f1147e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5023f7748f0e073e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index ca2699b4e366a3..b66cd857e44963 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -1360,7 +1360,8 @@ PyDoc_STRVAR(os_mkdir__doc__, "dir_fd may not be implemented on your platform.\n" " If it is unavailable, using it will raise a NotImplementedError.\n" "\n" -"The mode argument is ignored on Windows."); +"The mode argument is ignored on Windows. Where it is used, the current umask\n" +"value is first masked out."); #define OS_MKDIR_METHODDEF \ {"mkdir", _PyCFunction_CAST(os_mkdir), METH_FASTCALL|METH_KEYWORDS, os_mkdir__doc__}, @@ -1749,7 +1750,7 @@ os_system(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k static const char * const _keywords[] = {"command", NULL}; static _PyArg_Parser _parser = {NULL, _keywords, "system", 0}; PyObject *argsbuf[1]; - const Py_UNICODE *command; + const Py_UNICODE *command = NULL; long _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -9378,4 +9379,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=3032d9c5c3aaa165 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8dd784bf1e41b881 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index 9e4a8eb0b998a6..8b1f316d6d7276 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -205,8 +205,9 @@ PyDoc_STRVAR(signal_strsignal__doc__, "\n" "Return the system description of the given signal.\n" "\n" -"The return values can be such as \"Interrupt\", \"Segmentation fault\", etc.\n" -"Returns None if the signal is not recognized."); +"Returns the description of signal *signalnum*, such as \"Interrupt\"\n" +"for :const:`SIGINT`. Returns :const:`None` if *signalnum* has no\n" +"description. Raises :exc:`ValueError` if *signalnum* is invalid."); #define SIGNAL_STRSIGNAL_METHODDEF \ {"strsignal", (PyCFunction)signal_strsignal, METH_O, signal_strsignal__doc__}, @@ -698,4 +699,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=6ca1b70310eecdba input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9b3f9f1ae2ac2b94 input=a9049054013a1b77]*/ diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 0516e7367050c2..4de4144520aa48 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -927,6 +927,10 @@ errno_exec(PyObject *module) #ifdef EQFULL add_errcode("EQFULL", EQFULL, "Interface output queue is full"); #endif +#ifdef ENOTCAPABLE + // WASI extension + add_errcode("ENOTCAPABLE", ENOTCAPABLE, "Capabilities insufficient"); +#endif Py_DECREF(error_dict); return 0; diff --git a/Modules/expat/COPYING b/Modules/expat/COPYING index 3c0142e71c8d4f..ce9e5939291e45 100644 --- a/Modules/expat/COPYING +++ b/Modules/expat/COPYING @@ -1,5 +1,5 @@ Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper -Copyright (c) 2001-2019 Expat maintainers +Copyright (c) 2001-2022 Expat maintainers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index c9214f64070a82..1c83563cbf68e7 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -1054,8 +1054,8 @@ XML_SetBillionLaughsAttackProtectionActivationThreshold( See http://semver.org. */ #define XML_MAJOR_VERSION 2 -#define XML_MINOR_VERSION 4 -#define XML_MICRO_VERSION 7 +#define XML_MINOR_VERSION 5 +#define XML_MICRO_VERSION 0 #ifdef __cplusplus } diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index 444eba0fb03170..e09f533b23c9df 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -28,7 +28,7 @@ Copyright (c) 2002-2003 Fred L. Drake, Jr. Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2003 Greg Stein - Copyright (c) 2016-2021 Sebastian Pipping + Copyright (c) 2016-2022 Sebastian Pipping Copyright (c) 2018 Yury Gribov Copyright (c) 2019 David Loffredo Licensed under the MIT license: @@ -107,7 +107,9 @@ #include // ULONG_MAX -#if defined(_WIN32) && ! defined(__USE_MINGW_ANSI_STDIO) +#if defined(_WIN32) \ + && (! defined(__USE_MINGW_ANSI_STDIO) \ + || (1 - __USE_MINGW_ANSI_STDIO - 1 == 0)) # define EXPAT_FMT_ULL(midpart) "%" midpart "I64u" # if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW # define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d" diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h index e5406d7ee9eb54..303283ad2de98d 100644 --- a/Modules/expat/siphash.h +++ b/Modules/expat/siphash.h @@ -106,7 +106,7 @@ * if this code is included and compiled as C++; related GCC warning is: * warning: use of C++11 long long integer constant [-Wlong-long] */ -#define _SIP_ULL(high, low) (((uint64_t)high << 32) | low) +#define _SIP_ULL(high, low) ((((uint64_t)high) << 32) | (low)) #define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 05216d997b07f7..b6c2eca97567ba 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* fcb1a62fefa945567301146eb98e3ad3413e823a41c4378e84e8b6b6f308d824 (2.4.7+) +/* 5ab094ffadd6edfc94c3eee53af44a86951f9f1f0933ada3114bbce2bfb02c99 (2.5.0+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -19,7 +19,7 @@ Copyright (c) 2016 Gustavo Grieco Copyright (c) 2016 Pascal Cuoq Copyright (c) 2016 Ed Schouten - Copyright (c) 2017-2018 Rhodri James + Copyright (c) 2017-2022 Rhodri James Copyright (c) 2017 Václav Slavík Copyright (c) 2017 Viktor Szakats Copyright (c) 2017 Chanho Park @@ -35,6 +35,7 @@ Copyright (c) 2021 Dong-hee Na Copyright (c) 2022 Samanta Navarro Copyright (c) 2022 Jeffrey Walton + Copyright (c) 2022 Jann Horn Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -1068,6 +1069,14 @@ parserCreate(const XML_Char *encodingName, parserInit(parser, encodingName); if (encodingName && ! parser->m_protocolEncodingName) { + if (dtd) { + // We need to stop the upcoming call to XML_ParserFree from happily + // destroying parser->m_dtd because the DTD is shared with the parent + // parser and the only guard that keeps XML_ParserFree from destroying + // parser->m_dtd is parser->m_isParamEntity but it will be set to + // XML_TRUE only later in XML_ExternalEntityParserCreate (or not at all). + parser->m_dtd = NULL; + } XML_ParserFree(parser); return NULL; } @@ -3011,9 +3020,6 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, int len; const char *rawName; TAG *tag = parser->m_tagStack; - parser->m_tagStack = tag->parent; - tag->parent = parser->m_freeTagList; - parser->m_freeTagList = tag; rawName = s + enc->minBytesPerChar * 2; len = XmlNameLength(enc, rawName); if (len != tag->rawNameLength @@ -3021,6 +3027,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, *eventPP = rawName; return XML_ERROR_TAG_MISMATCH; } + parser->m_tagStack = tag->parent; + tag->parent = parser->m_freeTagList; + parser->m_freeTagList = tag; --parser->m_tagLevel; if (parser->m_endElementHandler) { const XML_Char *localPart; @@ -4271,7 +4280,7 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s, const XML_Char *storedEncName = NULL; const ENCODING *newEncoding = NULL; const char *version = NULL; - const char *versionend; + const char *versionend = NULL; const XML_Char *storedversion = NULL; int standalone = -1; @@ -4975,10 +4984,10 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser->m_handlerArg, parser->m_declElementType->name, parser->m_declAttributeId->name, parser->m_declAttributeType, 0, role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE); - poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; } } + poolClear(&parser->m_tempPool); break; case XML_ROLE_DEFAULT_ATTRIBUTE_VALUE: case XML_ROLE_FIXED_ATTRIBUTE_VALUE: @@ -5386,7 +5395,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, * * If 'standalone' is false, the DTD must have no * parameter entities or we wouldn't have passed the outer - * 'if' statement. That measn the only entity in the hash + * 'if' statement. That means the only entity in the hash * table is the external subset name "#" which cannot be * given as a parameter entity name in XML syntax, so the * lookup must have returned NULL and we don't even reach @@ -5798,19 +5807,27 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, if (result != XML_ERROR_NONE) return result; - else if (textEnd != next - && parser->m_parsingStatus.parsing == XML_SUSPENDED) { + + if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) { entity->processed = (int)(next - (const char *)entity->textPtr); return result; - } else { + } + #ifdef XML_DTD - entityTrackingOnClose(parser, entity, __LINE__); + entityTrackingOnClose(parser, entity, __LINE__); #endif - entity->open = XML_FALSE; - parser->m_openInternalEntities = openEntity->next; - /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity; + entity->open = XML_FALSE; + parser->m_openInternalEntities = openEntity->next; + /* put openEntity back in list of free instances */ + openEntity->next = parser->m_freeInternalEntities; + parser->m_freeInternalEntities = openEntity; + + // If there are more open entities we want to stop right here and have the + // upcoming call to XML_ResumeParser continue with entity content, or it would + // be ignored altogether. + if (parser->m_openInternalEntities != NULL + && parser->m_parsingStatus.parsing == XML_SUSPENDED) { + return XML_ERROR_NONE; } #ifdef XML_DTD @@ -5826,10 +5843,15 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, { parser->m_processor = contentProcessor; /* see externalEntityContentProcessor vs contentProcessor */ - return doContent(parser, parser->m_parentParser ? 1 : 0, parser->m_encoding, - s, end, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer, - XML_ACCOUNT_DIRECT); + result = doContent(parser, parser->m_parentParser ? 1 : 0, + parser->m_encoding, s, end, nextPtr, + (XML_Bool)! parser->m_parsingStatus.finalBuffer, + XML_ACCOUNT_DIRECT); + if (result == XML_ERROR_NONE) { + if (! storeRawNames(parser)) + return XML_ERROR_NO_MEMORY; + } + return result; } } diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index c659983b4008bd..2b7012a58be419 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -21,6 +21,7 @@ Copyright (c) 2017 José Gutiérrez de la Concha Copyright (c) 2019 David Loffredo Copyright (c) 2021 Dong-hee Na + Copyright (c) 2022 Martin Ettl Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -296,7 +297,7 @@ sb_charMatches(const ENCODING *enc, const char *p, int c) { } #else /* c is an ASCII character */ -# define CHAR_MATCHES(enc, p, c) (*(p) == c) +# define CHAR_MATCHES(enc, p, c) (*(p) == (c)) #endif #define PREFIX(ident) normal_##ident @@ -740,7 +741,7 @@ DEFINE_UTF16_TO_UTF16(big2_) ((p)[1] == 0 ? ((struct normal_encoding *)(enc))->type[(unsigned char)*(p)] \ : unicode_byte_type((p)[1], (p)[0])) #define LITTLE2_BYTE_TO_ASCII(p) ((p)[1] == 0 ? (p)[0] : -1) -#define LITTLE2_CHAR_MATCHES(p, c) ((p)[1] == 0 && (p)[0] == c) +#define LITTLE2_CHAR_MATCHES(p, c) ((p)[1] == 0 && (p)[0] == (c)) #define LITTLE2_IS_NAME_CHAR_MINBPC(p) \ UCS2_GET_NAMING(namePages, (unsigned char)p[1], (unsigned char)p[0]) #define LITTLE2_IS_NMSTRT_CHAR_MINBPC(p) \ @@ -875,7 +876,7 @@ static const struct normal_encoding internal_little2_encoding ? ((struct normal_encoding *)(enc))->type[(unsigned char)(p)[1]] \ : unicode_byte_type((p)[0], (p)[1])) #define BIG2_BYTE_TO_ASCII(p) ((p)[0] == 0 ? (p)[1] : -1) -#define BIG2_CHAR_MATCHES(p, c) ((p)[0] == 0 && (p)[1] == c) +#define BIG2_CHAR_MATCHES(p, c) ((p)[0] == 0 && (p)[1] == (c)) #define BIG2_IS_NAME_CHAR_MINBPC(p) \ UCS2_GET_NAMING(namePages, (unsigned char)p[0], (unsigned char)p[1]) #define BIG2_IS_NMSTRT_CHAR_MINBPC(p) \ diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index 4072b06497d1c2..1971d74bf8c91f 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -16,6 +16,7 @@ Copyright (c) 2018 Anton Maklakov Copyright (c) 2019 David Loffredo Copyright (c) 2020 Boris Kolpackov + Copyright (c) 2022 Martin Ettl Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -96,7 +97,7 @@ # define CHECK_NMSTRT_CASE(n, enc, ptr, end, nextTokPtr) \ case BT_LEAD##n: \ - if (end - ptr < n) \ + if ((end) - (ptr) < (n)) \ return XML_TOK_PARTIAL_CHAR; \ if (IS_INVALID_CHAR(enc, ptr, n) || ! IS_NMSTRT_CHAR(enc, ptr, n)) { \ *nextTokPtr = ptr; \ @@ -124,7 +125,8 @@ # define PREFIX(ident) ident # endif -# define HAS_CHARS(enc, ptr, end, count) (end - ptr >= count * MINBPC(enc)) +# define HAS_CHARS(enc, ptr, end, count) \ + ((end) - (ptr) >= ((count)*MINBPC(enc))) # define HAS_CHAR(enc, ptr, end) HAS_CHARS(enc, ptr, end, 1) diff --git a/Modules/expat/xmltok_impl.h b/Modules/expat/xmltok_impl.h index c518aada013df2..3469c4ae138c95 100644 --- a/Modules/expat/xmltok_impl.h +++ b/Modules/expat/xmltok_impl.h @@ -45,7 +45,7 @@ enum { BT_LF, /* line feed = "\n" */ BT_GT, /* greater than = ">" */ BT_QUOT, /* quotation character = "\"" */ - BT_APOS, /* aposthrophe = "'" */ + BT_APOS, /* apostrophe = "'" */ BT_EQUALS, /* equal sign = "=" */ BT_QUEST, /* question mark = "?" */ BT_EXCL, /* exclamation mark = "!" */ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 4847c1cb87bc8e..04995d2e745792 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -334,14 +334,17 @@ faulthandler_fatal_error(int signum) size_t i; fault_handler_t *handler = NULL; int save_errno = errno; + int found = 0; if (!fatal_error.enabled) return; for (i=0; i < faulthandler_nsignals; i++) { handler = &faulthandler_handlers[i]; - if (handler->signum == signum) + if (handler->signum == signum) { + found = 1; break; + } } if (handler == NULL) { /* faulthandler_nsignals == 0 (unlikely) */ @@ -351,9 +354,18 @@ faulthandler_fatal_error(int signum) /* restore the previous handler */ faulthandler_disable_fatal_handler(handler); - PUTS(fd, "Fatal Python error: "); - PUTS(fd, handler->name); - PUTS(fd, "\n\n"); + if (found) { + PUTS(fd, "Fatal Python error: "); + PUTS(fd, handler->name); + PUTS(fd, "\n\n"); + } + else { + char unknown_signum[23] = {0,}; + snprintf(unknown_signum, 23, "%d", signum); + PUTS(fd, "Fatal Python error from unexpected signum: "); + PUTS(fd, unknown_signum); + PUTS(fd, "\n\n"); + } faulthandler_dump_traceback(fd, fatal_error.all_threads, fatal_error.interp); diff --git a/Modules/getpath.c b/Modules/getpath.c index 94479887cf850a..ceacf36d8968a1 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -82,27 +82,32 @@ getpath_abspath(PyObject *Py_UNUSED(self), PyObject *args) static PyObject * getpath_basename(PyObject *Py_UNUSED(self), PyObject *args) { - const char *path; - if (!PyArg_ParseTuple(args, "s", &path)) { + PyObject *path; + if (!PyArg_ParseTuple(args, "U", &path)) { return NULL; } - const char *name = strrchr(path, SEP); - return PyUnicode_FromString(name ? name + 1 : path); + Py_ssize_t end = PyUnicode_GET_LENGTH(path); + Py_ssize_t pos = PyUnicode_FindChar(path, SEP, 0, end, -1); + if (pos < 0) { + return Py_NewRef(path); + } + return PyUnicode_Substring(path, pos + 1, end); } static PyObject * getpath_dirname(PyObject *Py_UNUSED(self), PyObject *args) { - const char *path; - if (!PyArg_ParseTuple(args, "s", &path)) { + PyObject *path; + if (!PyArg_ParseTuple(args, "U", &path)) { return NULL; } - const char *name = strrchr(path, SEP); - if (!name) { + Py_ssize_t end = PyUnicode_GET_LENGTH(path); + Py_ssize_t pos = PyUnicode_FindChar(path, SEP, 0, end, -1); + if (pos < 0) { return PyUnicode_FromStringAndSize(NULL, 0); } - return PyUnicode_FromStringAndSize(path, (name - path)); + return PyUnicode_Substring(path, 0, pos); } @@ -256,7 +261,7 @@ getpath_joinpath(PyObject *Py_UNUSED(self), PyObject *args) } Py_ssize_t n = PyTuple_GET_SIZE(args); if (n == 0) { - return PyUnicode_FromString(NULL); + return PyUnicode_FromStringAndSize(NULL, 0); } /* Convert all parts to wchar and accumulate max final length */ wchar_t **parts = (wchar_t **)PyMem_Malloc(n * sizeof(wchar_t *)); diff --git a/Modules/getpath.py b/Modules/getpath.py index dceeed7702c0be..fc533a8f5f5eb4 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -351,11 +351,11 @@ def search_up(prefix, *landmarks, test=isfile): try: # Read pyvenv.cfg from one level above executable pyvenvcfg = readlines(joinpath(venv_prefix, VENV_LANDMARK)) - except FileNotFoundError: + except (FileNotFoundError, PermissionError): # Try the same directory as executable pyvenvcfg = readlines(joinpath(venv_prefix2, VENV_LANDMARK)) venv_prefix = venv_prefix2 - except FileNotFoundError: + except (FileNotFoundError, PermissionError): venv_prefix = None pyvenvcfg = [] @@ -375,6 +375,25 @@ def search_up(prefix, *landmarks, test=isfile): pass if not base_executable: base_executable = joinpath(executable_dir, basename(executable)) + # It's possible "python" is executed from within a posix venv but that + # "python" is not available in the "home" directory as the standard + # `make install` does not create it and distros often do not provide it. + # + # In this case, try to fall back to known alternatives + if os_name != 'nt' and not isfile(base_executable): + base_exe = basename(executable) + for candidate in (DEFAULT_PROGRAM_NAME, f'python{VERSION_MAJOR}.{VERSION_MINOR}'): + candidate += EXE_SUFFIX if EXE_SUFFIX else '' + if base_exe == candidate: + continue + candidate = joinpath(executable_dir, candidate) + # Only set base_executable if the candidate exists. + # If no candidate succeeds, subsequent errors related to + # base_executable (like FileNotFoundError) remain in the + # context of the original executable name + if isfile(candidate): + base_executable = candidate + break break else: venv_prefix = None @@ -475,7 +494,7 @@ def search_up(prefix, *landmarks, test=isfile): # File exists but is empty platstdlib_dir = real_executable_dir build_prefix = joinpath(real_executable_dir, VPATH) - except FileNotFoundError: + except (FileNotFoundError, PermissionError): if isfile(joinpath(real_executable_dir, BUILD_LANDMARK)): build_prefix = joinpath(real_executable_dir, VPATH) if os_name == 'nt': @@ -579,15 +598,28 @@ def search_up(prefix, *landmarks, test=isfile): # Detect exec_prefix by searching from executable for the platstdlib_dir if PLATSTDLIB_LANDMARK and not exec_prefix: if executable_dir: - exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir) - if not exec_prefix: - if EXEC_PREFIX: - exec_prefix = EXEC_PREFIX - if not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)): - warn('Could not find platform dependent libraries ') + if os_name == 'nt': + # QUIRK: For compatibility and security, do not search for DLLs + # directory. The fallback below will cover it + exec_prefix = executable_dir + else: + exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir) + if not exec_prefix and EXEC_PREFIX: + exec_prefix = EXEC_PREFIX + if not exec_prefix or not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)): + if os_name == 'nt': + # QUIRK: If DLLs is missing on Windows, don't warn, just assume + # that it's all the same as prefix. + # gh-98790: We set platstdlib_dir here to avoid adding "DLLs" into + # sys.path when it doesn't exist, which would give site-packages + # precedence over executable_dir, which is *probably* where our PYDs + # live. Ideally, whoever changes our layout will tell us what the + # layout is, but in the past this worked, so it should keep working. + platstdlib_dir = exec_prefix = prefix else: warn('Could not find platform dependent libraries ') + # Fallback: assume exec_prefix == prefix if not exec_prefix: exec_prefix = prefix @@ -647,9 +679,8 @@ def search_up(prefix, *landmarks, test=isfile): else: library_dir = executable_dir pythonpath.append(joinpath(library_dir, ZIP_LANDMARK)) - elif build_prefix or venv_prefix: + elif build_prefix: # QUIRK: POSIX uses the default prefix when in the build directory - # or a venv pythonpath.append(joinpath(PREFIX, ZIP_LANDMARK)) else: pythonpath.append(joinpath(prefix, ZIP_LANDMARK)) @@ -689,7 +720,8 @@ def search_up(prefix, *landmarks, test=isfile): pythonpath.append(platstdlib_dir) if stdlib_dir: pythonpath.append(stdlib_dir) - pythonpath.append(executable_dir) + if executable_dir not in pythonpath: + pythonpath.append(executable_dir) else: if stdlib_dir: pythonpath.append(stdlib_dir) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 4a7a95730395e6..d5cdfc591e26f4 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1200,6 +1200,7 @@ cycle_setstate(cycleobject *lz, PyObject *state) PyErr_SetString(PyExc_TypeError, "state is not a tuple"); return NULL; } + // The second item can be 1/0 in old pickles and True/False in new pickles if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) { return NULL; } diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index aa93e756c606ba..0a907a0c04ba1e 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -2703,13 +2703,13 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) if (m != n) { PyErr_SetString(PyExc_ValueError, "both points must have the same number of dimensions"); - return NULL; - + goto error_exit; } if (n > NUM_STACK_ELEMS) { diffs = (double *) PyObject_Malloc(n * sizeof(double)); if (diffs == NULL) { - return PyErr_NoMemory(); + PyErr_NoMemory(); + goto error_exit; } } for (i=0 ; i # include +# include // UNLEN +# include "osdefs.h" // SEP +# define HAVE_SYMLINK #endif #ifdef __VXWORKS__ @@ -154,6 +157,18 @@ # define HAVE_SYMLINKAT_RUNTIME (symlinkat != NULL) # endif +# ifdef HAVE_UTIMENSAT +# define HAVE_UTIMENSAT_RUNTIME (utimensat != NULL) +# endif + +# ifdef HAVE_FUTIMENS +# define HAVE_FUTIMENS_RUNTIME (futimens != NULL) +# endif + +# ifdef HAVE_PWRITEV +# define HAVE_PWRITEV_RUNTIME (pwritev != NULL) +# endif + #endif #ifdef HAVE_FUTIMESAT @@ -414,18 +429,7 @@ extern char *ctermid_r(char *); # ifdef HAVE_PROCESS_H # include # endif -# ifndef IO_REPARSE_TAG_SYMLINK -# define IO_REPARSE_TAG_SYMLINK (0xA000000CL) -# endif -# ifndef IO_REPARSE_TAG_MOUNT_POINT -# define IO_REPARSE_TAG_MOUNT_POINT (0xA0000003L) -# endif -# include "osdefs.h" // SEP # include -# include -# include // ShellExecute() -# include // UNLEN -# define HAVE_SYMLINK #endif /* _MSC_VER */ #ifndef MAXPATHLEN @@ -4560,12 +4564,13 @@ If dir_fd is not None, it should be a file descriptor open to a directory, dir_fd may not be implemented on your platform. If it is unavailable, using it will raise a NotImplementedError. -The mode argument is ignored on Windows. +The mode argument is ignored on Windows. Where it is used, the current umask +value is first masked out. [clinic start generated code]*/ static PyObject * os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) -/*[clinic end generated code: output=a70446903abe821f input=e965f68377e9b1ce]*/ +/*[clinic end generated code: output=a70446903abe821f input=a61722e1576fab03]*/ { int result; #ifdef HAVE_MKDIRAT @@ -7096,8 +7101,13 @@ static PyObject * os_sched_yield_impl(PyObject *module) /*[clinic end generated code: output=902323500f222cac input=e54d6f98189391d4]*/ { - if (sched_yield()) + int result; + Py_BEGIN_ALLOW_THREADS + result = sched_yield(); + Py_END_ALLOW_THREADS + if (result < 0) { return posix_error(); + } Py_RETURN_NONE; } @@ -9811,7 +9821,7 @@ os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); #else do { -#ifdef __APPLE__ +#if defined(__APPLE__) && defined(__clang__) /* This entire function will be removed from the module dict when the API * is not available. */ @@ -9826,7 +9836,7 @@ os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, Py_END_ALLOW_THREADS } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); -#ifdef __APPLE__ +#if defined(__APPLE__) && defined(__clang__) #pragma clang diagnostic pop #endif @@ -10453,7 +10463,7 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, } while (result < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); #else -#ifdef __APPLE__ +#if defined(__APPLE__) && defined(__clang__) /* This entire function will be removed from the module dict when the API * is not available. */ @@ -10469,7 +10479,7 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, Py_END_ALLOW_THREADS } while (result < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); -#ifdef __APPLE__ +#if defined(__APPLE__) && defined(__clang__) #pragma clang diagnostic pop #endif diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 12319ee675045d..a9713422220afe 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -775,7 +775,7 @@ readinst(char *buf, int buf_size, PyObject *meth) Py_ssize_t len; const char *ptr; - str = PyObject_CallFunction(meth, "n", buf_size); + str = PyObject_CallFunction(meth, "i", buf_size); if (str == NULL) goto error; diff --git a/Modules/readline.c b/Modules/readline.c index 1b616fc4f3b4e1..27b89de7279464 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -1258,9 +1258,9 @@ setup_readline(readlinestate *mod_state) rl_attempted_completion_function = flex_complete; /* Set Python word break characters */ completer_word_break_characters = - rl_completer_word_break_characters = strdup(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?"); /* All nonalphanums except '.' */ + rl_completer_word_break_characters = completer_word_break_characters; mod_state->begidx = PyLong_FromLong(0L); mod_state->endidx = PyLong_FromLong(0L); diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index e3b37f179312d4..60a8067fc8c8ae 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -627,13 +627,14 @@ signal.strsignal Return the system description of the given signal. -The return values can be such as "Interrupt", "Segmentation fault", etc. -Returns None if the signal is not recognized. +Returns the description of signal *signalnum*, such as "Interrupt" +for :const:`SIGINT`. Returns :const:`None` if *signalnum* has no +description. Raises :exc:`ValueError` if *signalnum* is invalid. [clinic start generated code]*/ static PyObject * signal_strsignal_impl(PyObject *module, int signalnum) -/*[clinic end generated code: output=44e12e1e3b666261 input=b77914b03f856c74]*/ +/*[clinic end generated code: output=44e12e1e3b666261 input=238b335847778bc0]*/ { const char *res; @@ -1832,6 +1833,9 @@ _PyErr_CheckSignalsTstate(PyThreadState *tstate) _Py_atomic_store(&is_tripped, 0); _PyInterpreterFrame *frame = tstate->cframe->current_frame; + while (frame && _PyFrame_IsIncomplete(frame)) { + frame = frame->previous; + } signal_state_t *state = &signal_global_state; for (int i = 1; i < Py_NSIG; i++) { if (!_Py_atomic_load_relaxed(&Handlers[i].tripped)) { diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index c409fe968f8894..8416b6344ce4e0 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -207,9 +207,14 @@ syslog_syslog(PyObject * self, PyObject * args) */ PyObject *ident = S_ident_o; Py_XINCREF(ident); +#ifdef __APPLE__ + // gh-98178: On macOS, libc syslog() is not thread-safe + syslog(priority, "%s", message); +#else Py_BEGIN_ALLOW_THREADS; syslog(priority, "%s", message); Py_END_ALLOW_THREADS; +#endif Py_XDECREF(ident); Py_RETURN_NONE; } @@ -235,7 +240,7 @@ syslog_setlogmask(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "l;mask for priority", &maskpri)) return NULL; - if (PySys_Audit("syslog.setlogmask", "(O)", args ? args : Py_None) < 0) { + if (PySys_Audit("syslog.setlogmask", "l", maskpri) < 0) { return NULL; } omaskpri = setlogmask(maskpri); diff --git a/Modules/termios.c b/Modules/termios.c index 354e5ca18d04d8..fcc8f042679870 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -82,7 +82,12 @@ termios_tcgetattr_impl(PyObject *module, int fd) { termiosmodulestate *state = PyModule_GetState(module); struct termios mode; - if (tcgetattr(fd, &mode) == -1) { + int r; + + Py_BEGIN_ALLOW_THREADS + r = tcgetattr(fd, &mode); + Py_END_ALLOW_THREADS + if (r == -1) { return PyErr_SetFromErrno(state->TermiosError); } @@ -169,7 +174,12 @@ termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term) /* Get the old mode, in case there are any hidden fields... */ termiosmodulestate *state = PyModule_GetState(module); struct termios mode; - if (tcgetattr(fd, &mode) == -1) { + int r; + + Py_BEGIN_ALLOW_THREADS + r = tcgetattr(fd, &mode); + Py_END_ALLOW_THREADS + if (r == -1) { return PyErr_SetFromErrno(state->TermiosError); } @@ -211,7 +221,12 @@ termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term) return PyErr_SetFromErrno(state->TermiosError); if (cfsetospeed(&mode, (speed_t) ospeed) == -1) return PyErr_SetFromErrno(state->TermiosError); - if (tcsetattr(fd, when, &mode) == -1) + + Py_BEGIN_ALLOW_THREADS + r = tcsetattr(fd, when, &mode); + Py_END_ALLOW_THREADS + + if (r == -1) return PyErr_SetFromErrno(state->TermiosError); Py_RETURN_NONE; @@ -235,7 +250,13 @@ termios_tcsendbreak_impl(PyObject *module, int fd, int duration) /*[clinic end generated code: output=5945f589b5d3ac66 input=dc2f32417691f8ed]*/ { termiosmodulestate *state = PyModule_GetState(module); - if (tcsendbreak(fd, duration) == -1) { + int r; + + Py_BEGIN_ALLOW_THREADS + r = tcsendbreak(fd, duration); + Py_END_ALLOW_THREADS + + if (r == -1) { return PyErr_SetFromErrno(state->TermiosError); } @@ -256,7 +277,13 @@ termios_tcdrain_impl(PyObject *module, int fd) /*[clinic end generated code: output=5fd86944c6255955 input=c99241b140b32447]*/ { termiosmodulestate *state = PyModule_GetState(module); - if (tcdrain(fd) == -1) { + int r; + + Py_BEGIN_ALLOW_THREADS + r = tcdrain(fd); + Py_END_ALLOW_THREADS + + if (r == -1) { return PyErr_SetFromErrno(state->TermiosError); } @@ -282,7 +309,13 @@ termios_tcflush_impl(PyObject *module, int fd, int queue) /*[clinic end generated code: output=2424f80312ec2f21 input=0f7d08122ddc07b5]*/ { termiosmodulestate *state = PyModule_GetState(module); - if (tcflush(fd, queue) == -1) { + int r; + + Py_BEGIN_ALLOW_THREADS + r = tcflush(fd, queue); + Py_END_ALLOW_THREADS + + if (r == -1) { return PyErr_SetFromErrno(state->TermiosError); } @@ -308,7 +341,13 @@ termios_tcflow_impl(PyObject *module, int fd, int action) /*[clinic end generated code: output=afd10928e6ea66eb input=c6aff0640b6efd9c]*/ { termiosmodulestate *state = PyModule_GetState(module); - if (tcflow(fd, action) == -1) { + int r; + + Py_BEGIN_ALLOW_THREADS + r = tcflow(fd, action); + Py_END_ALLOW_THREADS + + if (r == -1) { return PyErr_SetFromErrno(state->TermiosError); } @@ -333,7 +372,13 @@ termios_tcgetwinsize_impl(PyObject *module, int fd) #if defined(TIOCGWINSZ) termiosmodulestate *state = PyModule_GetState(module); struct winsize w; - if (ioctl(fd, TIOCGWINSZ, &w) == -1) { + int r; + + Py_BEGIN_ALLOW_THREADS + r = ioctl(fd, TIOCGWINSZ, &w); + Py_END_ALLOW_THREADS + + if (r == -1) { return PyErr_SetFromErrno(state->TermiosError); } @@ -352,7 +397,12 @@ termios_tcgetwinsize_impl(PyObject *module, int fd) #elif defined(TIOCGSIZE) termiosmodulestate *state = PyModule_GetState(module); struct ttysize s; - if (ioctl(fd, TIOCGSIZE, &s) == -1) { + int r; + + Py_BEGIN_ALLOW_THREADS + r = ioctl(fd, TIOCGSIZE, &s); + Py_END_ALLOW_THREADS + if (r == -1) { return PyErr_SetFromErrno(state->TermiosError); } @@ -433,15 +483,25 @@ termios_tcsetwinsize_impl(PyObject *module, int fd, PyObject *winsz) return NULL; } - if (ioctl(fd, TIOCSWINSZ, &w) == -1) { + int r; + Py_BEGIN_ALLOW_THREADS + r = ioctl(fd, TIOCSWINSZ, &w); + Py_END_ALLOW_THREADS + + if (r == -1) { return PyErr_SetFromErrno(state->TermiosError); } Py_RETURN_NONE; #elif defined(TIOCGSIZE) && defined(TIOCSSIZE) struct ttysize s; + int r; /* Get the old ttysize because it might have more fields. */ - if (ioctl(fd, TIOCGSIZE, &s) == -1) { + Py_BEGIN_ALLOW_THREADS + r = ioctl(fd, TIOCGSIZE, &s); + Py_END_ALLOW_THREADS + + if (r == -1) { return PyErr_SetFromErrno(state->TermiosError); } @@ -453,7 +513,11 @@ termios_tcsetwinsize_impl(PyObject *module, int fd, PyObject *winsz) return NULL; } - if (ioctl(fd, TIOCSSIZE, &s) == -1) { + Py_BEGIN_ALLOW_THREADS + r = ioctl(fd, TIOCSSIZE, &s); + Py_END_ALLOW_THREADS + + if (r == -1) { return PyErr_SetFromErrno(state->TermiosError); } diff --git a/Objects/codeobject.c b/Objects/codeobject.c index d7434ddefbc026..32938b52ab40f2 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -636,12 +636,22 @@ PyCode_New(int argcount, int kwonlyargcount, exceptiontable); } -static const char assert0[6] = { +// NOTE: When modifying the construction of PyCode_NewEmpty, please also change +// test.test_code.CodeLocationTest.test_code_new_empty to keep it in sync! + +static const uint8_t assert0[6] = { RESUME, 0, LOAD_ASSERTION_ERROR, 0, RAISE_VARARGS, 1 }; +static const uint8_t linetable[2] = { + (1 << 7) // New entry. + | (PY_CODE_LOCATION_INFO_NO_COLUMNS << 3) + | (3 - 1), // Three code units. + 0, // Offset from co_firstlineno. +}; + PyCodeObject * PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) { @@ -649,6 +659,7 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) PyObject *filename_ob = NULL; PyObject *funcname_ob = NULL; PyObject *code_ob = NULL; + PyObject *linetable_ob = NULL; PyCodeObject *result = NULL; nulltuple = PyTuple_New(0); @@ -663,10 +674,14 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) if (filename_ob == NULL) { goto failed; } - code_ob = PyBytes_FromStringAndSize(assert0, 6); + code_ob = PyBytes_FromStringAndSize((const char *)assert0, 6); if (code_ob == NULL) { goto failed; } + linetable_ob = PyBytes_FromStringAndSize((const char *)linetable, 2); + if (linetable_ob == NULL) { + goto failed; + } #define emptystring (PyObject *)&_Py_SINGLETON(bytes_empty) struct _PyCodeConstructor con = { @@ -675,7 +690,7 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) .qualname = funcname_ob, .code = code_ob, .firstlineno = firstlineno, - .linetable = emptystring, + .linetable = linetable_ob, .consts = nulltuple, .names = nulltuple, .localsplusnames = nulltuple, @@ -690,6 +705,7 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) Py_XDECREF(funcname_ob); Py_XDECREF(filename_ob); Py_XDECREF(code_ob); + Py_XDECREF(linetable_ob); return result; } diff --git a/Objects/descrobject.c b/Objects/descrobject.c index c3c541bf3c3212..6a5c2a4cf99981 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -16,7 +16,7 @@ class property "propertyobject *" "&PyProperty_Type" // see pycore_object.h #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) #include -EM_JS(PyObject*, descr_set_trampoline_call, (setter set, PyObject *obj, PyObject *value, void *closure), { +EM_JS(int, descr_set_trampoline_call, (setter set, PyObject *obj, PyObject *value, void *closure), { return wasmTable.get(set)(obj, value, closure); }); @@ -775,7 +775,7 @@ PyTypeObject PyClassMethodDescr_Type = { 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ - descr_methods, /* tp_methods */ + 0, /* tp_methods */ descr_members, /* tp_members */ method_getset, /* tp_getset */ 0, /* tp_base */ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index ebbd22ee7c145e..4a214f8cf5b751 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5573,14 +5573,16 @@ _PyObject_FreeInstanceAttributes(PyObject *self) PyTypeObject *tp = Py_TYPE(self); assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictValues **values_ptr = _PyObject_ValuesPointer(self); - if (*values_ptr == NULL) { + PyDictValues *values = *values_ptr; + if (values == NULL) { return; } + *values_ptr = NULL; PyDictKeysObject *keys = CACHED_KEYS(tp); for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_XDECREF((*values_ptr)->values[i]); + Py_XDECREF(values->values[i]); } - free_values(*values_ptr); + free_values(values); } PyObject * diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 3c4df2facfe2e2..4fba9b0a62486c 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -167,8 +167,14 @@ BaseException_setstate(PyObject *self, PyObject *state) return NULL; } while (PyDict_Next(state, &i, &d_key, &d_value)) { - if (PyObject_SetAttr(self, d_key, d_value) < 0) + Py_INCREF(d_key); + Py_INCREF(d_value); + int res = PyObject_SetAttr(self, d_key, d_value); + Py_DECREF(d_value); + Py_DECREF(d_key); + if (res < 0) { return NULL; + } } } Py_RETURN_NONE; @@ -768,7 +774,19 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } } else { - /* Do nothing - we don't interfere with subclasses */ + /* user-defined subclass */ + if (nested_base_exceptions) { + int nonbase = PyObject_IsSubclass((PyObject*)cls, PyExc_Exception); + if (nonbase == -1) { + goto error; + } + else if (nonbase == 1) { + PyErr_Format(PyExc_TypeError, + "Cannot nest BaseExceptions in '%.200s'", + cls->tp_name); + goto error; + } + } } if (!cls) { @@ -964,11 +982,11 @@ typedef enum { EXCEPTION_GROUP_MATCH_BY_TYPE = 0, /* A PyFunction returning True for matching exceptions */ EXCEPTION_GROUP_MATCH_BY_PREDICATE = 1, - /* A set of leaf exceptions to include in the result. + /* A set of the IDs of leaf exceptions to include in the result. * This matcher type is used internally by the interpreter * to construct reraised exceptions. */ - EXCEPTION_GROUP_MATCH_INSTANCES = 2 + EXCEPTION_GROUP_MATCH_INSTANCE_IDS = 2 } _exceptiongroup_split_matcher_type; static int @@ -1026,10 +1044,16 @@ exceptiongroup_split_check_match(PyObject *exc, Py_DECREF(exc_matches); return is_true; } - case EXCEPTION_GROUP_MATCH_INSTANCES: { + case EXCEPTION_GROUP_MATCH_INSTANCE_IDS: { assert(PySet_Check(matcher_value)); if (!_PyBaseExceptionGroup_Check(exc)) { - return PySet_Contains(matcher_value, exc); + PyObject *exc_id = PyLong_FromVoidPtr(exc); + if (exc_id == NULL) { + return -1; + } + int res = PySet_Contains(matcher_value, exc_id); + Py_DECREF(exc_id); + return res; } return 0; } @@ -1214,32 +1238,35 @@ BaseExceptionGroup_subgroup(PyObject *self, PyObject *args) } static int -collect_exception_group_leaves(PyObject *exc, PyObject *leaves) +collect_exception_group_leaf_ids(PyObject *exc, PyObject *leaf_ids) { if (Py_IsNone(exc)) { return 0; } assert(PyExceptionInstance_Check(exc)); - assert(PySet_Check(leaves)); + assert(PySet_Check(leaf_ids)); - /* Add all leaf exceptions in exc to the leaves set */ + /* Add IDs of all leaf exceptions in exc to the leaf_ids set */ if (!_PyBaseExceptionGroup_Check(exc)) { - if (PySet_Add(leaves, exc) < 0) { + PyObject *exc_id = PyLong_FromVoidPtr(exc); + if (exc_id == NULL) { return -1; } - return 0; + int res = PySet_Add(leaf_ids, exc_id); + Py_DECREF(exc_id); + return res; } PyBaseExceptionGroupObject *eg = _PyBaseExceptionGroupObject_cast(exc); Py_ssize_t num_excs = PyTuple_GET_SIZE(eg->excs); /* recursive calls */ for (Py_ssize_t i = 0; i < num_excs; i++) { PyObject *e = PyTuple_GET_ITEM(eg->excs, i); - if (_Py_EnterRecursiveCall(" in collect_exception_group_leaves")) { + if (_Py_EnterRecursiveCall(" in collect_exception_group_leaf_ids")) { return -1; } - int res = collect_exception_group_leaves(e, leaves); + int res = collect_exception_group_leaf_ids(e, leaf_ids); _Py_LeaveRecursiveCall(); if (res < 0) { return -1; @@ -1260,8 +1287,8 @@ exception_group_projection(PyObject *eg, PyObject *keep) assert(_PyBaseExceptionGroup_Check(eg)); assert(PyList_CheckExact(keep)); - PyObject *leaves = PySet_New(NULL); - if (!leaves) { + PyObject *leaf_ids = PySet_New(NULL); + if (!leaf_ids) { return NULL; } @@ -1270,8 +1297,8 @@ exception_group_projection(PyObject *eg, PyObject *keep) PyObject *e = PyList_GET_ITEM(keep, i); assert(e != NULL); assert(_PyBaseExceptionGroup_Check(e)); - if (collect_exception_group_leaves(e, leaves) < 0) { - Py_DECREF(leaves); + if (collect_exception_group_leaf_ids(e, leaf_ids) < 0) { + Py_DECREF(leaf_ids); return NULL; } } @@ -1279,9 +1306,9 @@ exception_group_projection(PyObject *eg, PyObject *keep) _exceptiongroup_split_result split_result; bool construct_rest = false; int err = exceptiongroup_split_recursive( - eg, EXCEPTION_GROUP_MATCH_INSTANCES, leaves, + eg, EXCEPTION_GROUP_MATCH_INSTANCE_IDS, leaf_ids, construct_rest, &split_result); - Py_DECREF(leaves); + Py_DECREF(leaf_ids); if (err < 0) { return NULL; } @@ -3644,6 +3671,11 @@ _PyExc_InitState(PyInterpreterState *interp) ADD_ERRNO(InterruptedError, EINTR); ADD_ERRNO(PermissionError, EACCES); ADD_ERRNO(PermissionError, EPERM); +#ifdef ENOTCAPABLE + // Extension for WASI capability-based security. Process lacks + // capability to access a resource. + ADD_ERRNO(PermissionError, ENOTCAPABLE); +#endif ADD_ERRNO(ProcessLookupError, ESRCH); ADD_ERRNO(TimeoutError, ETIMEDOUT); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 1ddfec2fb08ab7..f77f5ed39392e2 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -590,6 +590,7 @@ first_line_not_before(int *lines, int len, int line) static PyFrameState _PyFrame_GetState(PyFrameObject *frame) { + assert(!_PyFrame_IsIncomplete(frame->f_frame)); if (frame->f_frame->stacktop == 0) { return FRAME_CLEARED; } @@ -1063,6 +1064,9 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, init_frame((_PyInterpreterFrame *)f->_f_frame_data, func, locals); f->f_frame = (_PyInterpreterFrame *)f->_f_frame_data; f->f_frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; + // This frame needs to be "complete", so pretend that the first RESUME ran: + f->f_frame->prev_instr = _PyCode_CODE(code) + code->_co_firsttraceable; + assert(!_PyFrame_IsIncomplete(f->f_frame)); Py_DECREF(func); _PyObject_GC_TRACK(f); return f; @@ -1189,6 +1193,7 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) { int PyFrame_FastToLocalsWithError(PyFrameObject *f) { + assert(!_PyFrame_IsIncomplete(f->f_frame)); if (f == NULL) { PyErr_BadInternalCall(); return -1; @@ -1204,7 +1209,7 @@ void PyFrame_FastToLocals(PyFrameObject *f) { int res; - + assert(!_PyFrame_IsIncomplete(f->f_frame)); assert(!PyErr_Occurred()); res = PyFrame_FastToLocalsWithError(f); @@ -1282,6 +1287,7 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) void PyFrame_LocalsToFast(PyFrameObject *f, int clear) { + assert(!_PyFrame_IsIncomplete(f->f_frame)); if (f && f->f_fast_as_locals && _PyFrame_GetState(f) != FRAME_CLEARED) { _PyFrame_LocalsToFast(f->f_frame, clear); f->f_fast_as_locals = 0; @@ -1292,6 +1298,7 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear) int _PyFrame_IsEntryFrame(PyFrameObject *frame) { assert(frame != NULL); + assert(!_PyFrame_IsIncomplete(frame->f_frame)); return frame->f_frame->is_entry; } @@ -1300,6 +1307,7 @@ PyCodeObject * PyFrame_GetCode(PyFrameObject *frame) { assert(frame != NULL); + assert(!_PyFrame_IsIncomplete(frame->f_frame)); PyCodeObject *code = frame->f_frame->f_code; assert(code != NULL); Py_INCREF(code); @@ -1311,6 +1319,7 @@ PyFrameObject* PyFrame_GetBack(PyFrameObject *frame) { assert(frame != NULL); + assert(!_PyFrame_IsIncomplete(frame->f_frame)); PyFrameObject *back = frame->f_back; if (back == NULL) { _PyInterpreterFrame *prev = frame->f_frame->previous; @@ -1328,24 +1337,28 @@ PyFrame_GetBack(PyFrameObject *frame) PyObject* PyFrame_GetLocals(PyFrameObject *frame) { + assert(!_PyFrame_IsIncomplete(frame->f_frame)); return frame_getlocals(frame, NULL); } PyObject* PyFrame_GetGlobals(PyFrameObject *frame) { + assert(!_PyFrame_IsIncomplete(frame->f_frame)); return frame_getglobals(frame, NULL); } PyObject* PyFrame_GetBuiltins(PyFrameObject *frame) { + assert(!_PyFrame_IsIncomplete(frame->f_frame)); return frame_getbuiltins(frame, NULL); } int PyFrame_GetLasti(PyFrameObject *frame) { + assert(!_PyFrame_IsIncomplete(frame->f_frame)); int lasti = _PyInterpreterFrame_LASTI(frame->f_frame); if (lasti < 0) { return -1; @@ -1356,6 +1369,7 @@ PyFrame_GetLasti(PyFrameObject *frame) PyObject * PyFrame_GetGenerator(PyFrameObject *frame) { + assert(!_PyFrame_IsIncomplete(frame->f_frame)); if (frame->f_frame->owner != FRAME_OWNED_BY_GENERATOR) { return NULL; } diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 32b4155c03e6ae..63074634747fa2 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -300,7 +300,6 @@ func_get_annotation_dict(PyFunctionObject *op) } Py_SETREF(op->func_annotations, ann_dict); } - Py_INCREF(op->func_annotations); assert(PyDict_Check(op->func_annotations)); return op->func_annotations; } @@ -532,7 +531,11 @@ func_get_annotations(PyFunctionObject *op, void *Py_UNUSED(ignored)) if (op->func_annotations == NULL) return NULL; } - return func_get_annotation_dict(op); + PyObject *d = func_get_annotation_dict(op); + if (d) { + Py_INCREF(d); + } + return d; } static int diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 19f011fd3a7437..77acd1bc57a568 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -458,6 +458,13 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje } for (Py_ssize_t iarg = 0, jarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(args, iarg); + if (PyType_Check(arg)) { + Py_INCREF(arg); + PyTuple_SET_ITEM(newargs, jarg, arg); + jarg++; + continue; + } + int unpack = _is_unpacked_typevartuple(arg); if (unpack < 0) { Py_DECREF(newargs); diff --git a/Objects/object.c b/Objects/object.c index 6a22f27de0b1c8..c4f2786c50a02b 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1076,8 +1076,9 @@ _PyObject_DictPointer(PyObject *obj) tsize = -tsize; } size_t size = _PyObject_VAR_SIZE(tp, tsize); + assert(size <= (size_t)PY_SSIZE_T_MAX); + dictoffset += (Py_ssize_t)size; - dictoffset += (long)size; _PyObject_ASSERT(obj, dictoffset > 0); _PyObject_ASSERT(obj, dictoffset % SIZEOF_VOID_P == 0); } diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index ce5da5f1b7b92b..fae9c8110f1928 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2999,7 +2999,6 @@ _PyObject_DebugMallocStats(FILE *out) * will be living in full pools -- would be a shame to miss them. */ for (i = 0; i < maxarenas; ++i) { - uint j; uintptr_t base = arenas[i].address; /* Skip arenas which are not allocated. */ @@ -3018,8 +3017,7 @@ _PyObject_DebugMallocStats(FILE *out) /* visit every pool in the arena */ assert(base <= (uintptr_t) arenas[i].pool_address); - for (j = 0; base < (uintptr_t) arenas[i].pool_address; - ++j, base += POOL_SIZE) { + for (; base < (uintptr_t) arenas[i].pool_address; base += POOL_SIZE) { poolp p = (poolp)base; const uint sz = p->szidx; uint freeblocks; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a7bd961c64a838..a19eaadcfc2ee6 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -11134,8 +11134,8 @@ unicode_compare_eq(PyObject *str1, PyObject *str2) int _PyUnicode_Equal(PyObject *str1, PyObject *str2) { - assert(PyUnicode_CheckExact(str1)); - assert(PyUnicode_CheckExact(str2)); + assert(PyUnicode_Check(str1)); + assert(PyUnicode_Check(str2)); if (str1 == str2) { return 1; } diff --git a/PC/clinic/_msi.c.h b/PC/clinic/_msi.c.h index fd21142158312f..b717192b48333a 100644 --- a/PC/clinic/_msi.c.h +++ b/PC/clinic/_msi.c.h @@ -195,7 +195,7 @@ _msi_Record_SetString(msiobj *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int field; - const Py_UNICODE *value; + const Py_UNICODE *value = NULL; if (!_PyArg_CheckPositional("SetString", nargs, 2, 2)) { goto exit; @@ -244,7 +244,7 @@ _msi_Record_SetStream(msiobj *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int field; - const Py_UNICODE *value; + const Py_UNICODE *value = NULL; if (!_PyArg_CheckPositional("SetStream", nargs, 2, 2)) { goto exit; @@ -555,7 +555,7 @@ static PyObject * _msi_Database_OpenView(msiobj *self, PyObject *arg) { PyObject *return_value = NULL; - const Py_UNICODE *sql; + const Py_UNICODE *sql = NULL; if (!PyUnicode_Check(arg)) { _PyArg_BadArgument("OpenView", "argument", "str", arg); @@ -650,7 +650,7 @@ static PyObject * _msi_OpenDatabase(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - const Py_UNICODE *path; + const Py_UNICODE *path = NULL; int persist; if (!_PyArg_CheckPositional("OpenDatabase", nargs, 2, 2)) { @@ -713,4 +713,4 @@ _msi_CreateRecord(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=d7eb07e6bfcdc13f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=276175d60fbfc956 input=a9049054013a1b77]*/ diff --git a/PC/clinic/winreg.c.h b/PC/clinic/winreg.c.h index 2507e46e263334..a413dec5dda546 100644 --- a/PC/clinic/winreg.c.h +++ b/PC/clinic/winreg.c.h @@ -148,7 +148,7 @@ static PyObject * winreg_ConnectRegistry(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - const Py_UNICODE *computer_name; + const Py_UNICODE *computer_name = NULL; HKEY key; HKEY _return_value; @@ -220,7 +220,7 @@ winreg_CreateKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - const Py_UNICODE *sub_key; + const Py_UNICODE *sub_key = NULL; HKEY _return_value; if (!_PyArg_CheckPositional("CreateKey", nargs, 2, 2)) { @@ -301,7 +301,7 @@ winreg_CreateKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py static const char * const _keywords[] = {"key", "sub_key", "reserved", "access", NULL}; static _PyArg_Parser _parser = {"O&O&|ii:CreateKeyEx", _keywords, 0}; HKEY key; - const Py_UNICODE *sub_key; + const Py_UNICODE *sub_key = NULL; int reserved = 0; REGSAM access = KEY_WRITE; HKEY _return_value; @@ -354,7 +354,7 @@ winreg_DeleteKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - const Py_UNICODE *sub_key; + const Py_UNICODE *sub_key = NULL; if (!_PyArg_CheckPositional("DeleteKey", nargs, 2, 2)) { goto exit; @@ -428,7 +428,7 @@ winreg_DeleteKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py static const char * const _keywords[] = {"key", "sub_key", "access", "reserved", NULL}; static _PyArg_Parser _parser = {"O&O&|ii:DeleteKeyEx", _keywords, 0}; HKEY key; - const Py_UNICODE *sub_key; + const Py_UNICODE *sub_key = NULL; REGSAM access = KEY_WOW64_64KEY; int reserved = 0; @@ -469,7 +469,7 @@ winreg_DeleteValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - const Py_UNICODE *value; + const Py_UNICODE *value = NULL; if (!_PyArg_CheckPositional("DeleteValue", nargs, 2, 2)) { goto exit; @@ -619,7 +619,7 @@ static PyObject * winreg_ExpandEnvironmentStrings(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - const Py_UNICODE *string; + const Py_UNICODE *string = NULL; if (!PyUnicode_Check(arg)) { _PyArg_BadArgument("ExpandEnvironmentStrings", "argument", "str", arg); @@ -724,8 +724,8 @@ winreg_LoadKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - const Py_UNICODE *sub_key; - const Py_UNICODE *file_name; + const Py_UNICODE *sub_key = NULL; + const Py_UNICODE *file_name = NULL; if (!_PyArg_CheckPositional("LoadKey", nargs, 3, 3)) { goto exit; @@ -805,7 +805,7 @@ winreg_OpenKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje static const char * const _keywords[] = {"key", "sub_key", "reserved", "access", NULL}; static _PyArg_Parser _parser = {"O&O&|ii:OpenKey", _keywords, 0}; HKEY key; - const Py_UNICODE *sub_key; + const Py_UNICODE *sub_key = NULL; int reserved = 0; REGSAM access = KEY_READ; HKEY _return_value; @@ -862,7 +862,7 @@ winreg_OpenKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb static const char * const _keywords[] = {"key", "sub_key", "reserved", "access", NULL}; static _PyArg_Parser _parser = {"O&O&|ii:OpenKeyEx", _keywords, 0}; HKEY key; - const Py_UNICODE *sub_key; + const Py_UNICODE *sub_key = NULL; int reserved = 0; REGSAM access = KEY_READ; HKEY _return_value; @@ -953,7 +953,7 @@ winreg_QueryValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - const Py_UNICODE *sub_key; + const Py_UNICODE *sub_key = NULL; if (!_PyArg_CheckPositional("QueryValue", nargs, 2, 2)) { goto exit; @@ -1016,7 +1016,7 @@ winreg_QueryValueEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - const Py_UNICODE *name; + const Py_UNICODE *name = NULL; if (!_PyArg_CheckPositional("QueryValueEx", nargs, 2, 2)) { goto exit; @@ -1084,7 +1084,7 @@ winreg_SaveKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - const Py_UNICODE *file_name; + const Py_UNICODE *file_name = NULL; if (!_PyArg_CheckPositional("SaveKey", nargs, 2, 2)) { goto exit; @@ -1153,7 +1153,7 @@ winreg_SetValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - const Py_UNICODE *sub_key; + const Py_UNICODE *sub_key = NULL; DWORD type; PyObject *value_obj; @@ -1228,7 +1228,7 @@ winreg_SetValueEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - const Py_UNICODE *value_name; + const Py_UNICODE *value_name = NULL; PyObject *reserved; DWORD type; PyObject *value; @@ -1349,4 +1349,4 @@ winreg_QueryReflectionKey(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=7ad1db69bc42cab4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e83bdaabb4fa2167 input=a9049054013a1b77]*/ diff --git a/PC/launcher2.c b/PC/launcher2.c index 1f6f97b82092b9..9b3db04aa48b72 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -491,62 +491,39 @@ dumpSearchInfo(SearchInfo *search) int -findArgumentLength(const wchar_t *buffer, int bufferLength) +findArgv0Length(const wchar_t *buffer, int bufferLength) { - if (bufferLength < 0) { - bufferLength = (int)wcsnlen_s(buffer, MAXLEN); - } - if (bufferLength == 0) { - return 0; - } - const wchar_t *end; - int i; - - if (buffer[0] != L'"') { - end = wcschr(buffer, L' '); - if (!end) { - return bufferLength; - } - i = (int)(end - buffer); - return i < bufferLength ? i : bufferLength; - } - - i = 0; - while (i < bufferLength) { - end = wcschr(&buffer[i + 1], L'"'); - if (!end) { - return bufferLength; - } - - i = (int)(end - buffer); - if (i >= bufferLength) { - return bufferLength; - } - - int j = i; - while (j > 1 && buffer[--j] == L'\\') { - if (j > 0 && buffer[--j] == L'\\') { - // Even number, so back up and keep counting - } else { - // Odd number, so it's escaped and we want to keep searching - continue; + // Note: this implements semantics that are only valid for argv0. + // Specifically, there is no escaping of quotes, and quotes within + // the argument have no effect. A quoted argv0 must start and end + // with a double quote character; otherwise, it ends at the first + // ' ' or '\t'. + int quoted = buffer[0] == L'"'; + for (int i = 1; bufferLength < 0 || i < bufferLength; ++i) { + switch (buffer[i]) { + case L'\0': + return i; + case L' ': + case L'\t': + if (!quoted) { + return i; } - } - - // Non-escaped quote with space after it - end of the argument! - if (i + 1 >= bufferLength || isspace(buffer[i + 1])) { - return i + 1; + break; + case L'"': + if (quoted) { + return i + 1; + } + break; } } - return bufferLength; } const wchar_t * -findArgumentEnd(const wchar_t *buffer, int bufferLength) +findArgv0End(const wchar_t *buffer, int bufferLength) { - return &buffer[findArgumentLength(buffer, bufferLength)]; + return &buffer[findArgv0Length(buffer, bufferLength)]; } @@ -562,11 +539,16 @@ parseCommandLine(SearchInfo *search) return RC_NO_COMMANDLINE; } - const wchar_t *tail = findArgumentEnd(search->originalCmdLine, -1); - const wchar_t *end = tail; - search->restOfCmdLine = tail; + const wchar_t *argv0End = findArgv0End(search->originalCmdLine, -1); + const wchar_t *tail = argv0End; // will be start of the executable name + const wchar_t *end = argv0End; // will be end of the executable name + search->restOfCmdLine = argv0End; // will be first space after argv0 while (--tail != search->originalCmdLine) { - if (*tail == L'.' && end == search->restOfCmdLine) { + if (*tail == L'"' && end == argv0End) { + // Move the "end" up to the quote, so we also allow moving for + // a period later on. + end = argv0End = tail; + } else if (*tail == L'.' && end == argv0End) { end = tail; } else if (*tail == L'\\' || *tail == L'/') { ++tail; @@ -871,6 +853,62 @@ _findCommand(SearchInfo *search, const wchar_t *command, int commandLength) } +int +_useShebangAsExecutable(SearchInfo *search, const wchar_t *shebang, int shebangLength) +{ + wchar_t buffer[MAXLEN]; + wchar_t script[MAXLEN]; + wchar_t command[MAXLEN]; + + int commandLength = 0; + int inQuote = 0; + + if (!shebang || !shebangLength) { + return 0; + } + + wchar_t *pC = command; + for (int i = 0; i < shebangLength; ++i) { + wchar_t c = shebang[i]; + if (isspace(c) && !inQuote) { + commandLength = i; + break; + } else if (c == L'"') { + inQuote = !inQuote; + } else if (c == L'/' || c == L'\\') { + *pC++ = L'\\'; + } else { + *pC++ = c; + } + } + *pC = L'\0'; + + if (!GetCurrentDirectoryW(MAXLEN, buffer) || + wcsncpy_s(script, MAXLEN, search->scriptFile, search->scriptFileLength) || + FAILED(PathCchCombineEx(buffer, MAXLEN, buffer, script, + PATHCCH_ALLOW_LONG_PATHS)) || + FAILED(PathCchRemoveFileSpec(buffer, MAXLEN)) || + FAILED(PathCchCombineEx(buffer, MAXLEN, buffer, command, + PATHCCH_ALLOW_LONG_PATHS)) + ) { + return RC_NO_MEMORY; + } + + int n = (int)wcsnlen(buffer, MAXLEN); + wchar_t *path = allocSearchInfoBuffer(search, n + 1); + if (!path) { + return RC_NO_MEMORY; + } + wcscpy_s(path, n + 1, buffer); + search->executablePath = path; + if (commandLength) { + search->executableArgs = &shebang[commandLength]; + search->executableArgsLength = shebangLength - commandLength; + } + return 0; +} + + int checkShebang(SearchInfo *search) { @@ -963,13 +1001,19 @@ checkShebang(SearchInfo *search) L"/usr/bin/env ", L"/usr/bin/", L"/usr/local/bin/", - L"", + L"python", NULL }; for (const wchar_t **tmpl = shebangTemplates; *tmpl; ++tmpl) { if (_shebangStartsWith(shebang, shebangLength, *tmpl, &command)) { commandLength = 0; + // Normally "python" is the start of the command, but we also need it + // as a shebang prefix for back-compat. We move the command marker back + // if we match on that one. + if (0 == wcscmp(*tmpl, L"python")) { + command -= 6; + } while (command[commandLength] && !isspace(command[commandLength])) { commandLength += 1; } @@ -1012,11 +1056,14 @@ checkShebang(SearchInfo *search) debug(L"# Found shebang command but could not execute it: %.*s\n", commandLength, command); } - break; + // search is done by this point + return 0; } } - return 0; + // Unrecognised commands are joined to the script's directory and treated + // as the executable path + return _useShebangAsExecutable(search, shebang, shebangLength); } @@ -1673,6 +1720,7 @@ struct AppxSearchInfo { struct AppxSearchInfo APPX_SEARCH[] = { // Releases made through the Store + { L"PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0", L"3.12", 10 }, { L"PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0", L"3.11", 10 }, { L"PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0", L"3.10", 10 }, { L"PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0", L"3.9", 10 }, @@ -1681,6 +1729,7 @@ struct AppxSearchInfo APPX_SEARCH[] = { // Side-loadable releases. Note that the publisher ID changes whenever we // renew our code-signing certificate, so the newer ID has a higher // priority (lower sortKey) + { L"PythonSoftwareFoundation.Python.3.12_3847v3x7pw1km", L"3.12", 11 }, { L"PythonSoftwareFoundation.Python.3.11_3847v3x7pw1km", L"3.11", 11 }, { L"PythonSoftwareFoundation.Python.3.11_hd69rhyc2wevp", L"3.11", 12 }, { L"PythonSoftwareFoundation.Python.3.10_3847v3x7pw1km", L"3.10", 11 }, @@ -1755,7 +1804,8 @@ struct StoreSearchInfo { struct StoreSearchInfo STORE_SEARCH[] = { - { L"3", /* 3.10 */ L"9PJPW5LDXLZ5" }, + { L"3", /* 3.11 */ L"9NRWMJP3717K" }, + { L"3.12", L"9NCVDN91XZQP" }, { L"3.11", L"9NRWMJP3717K" }, { L"3.10", L"9PJPW5LDXLZ5" }, { L"3.9", L"9P7QFQMJRFP7" }, diff --git a/PC/python_uwp.cpp b/PC/python_uwp.cpp index 88369e8fbfeb38..2beea60e5af1ef 100644 --- a/PC/python_uwp.cpp +++ b/PC/python_uwp.cpp @@ -10,6 +10,7 @@ #include +#include #include #include @@ -28,37 +29,49 @@ const wchar_t *PROGNAME = L"python.exe"; #endif static std::wstring -get_user_base() +get_package_family() { try { - const auto appData = winrt::Windows::Storage::ApplicationData::Current(); - if (appData) { - const auto localCache = appData.LocalCacheFolder(); - if (localCache) { - auto path = localCache.Path(); - if (!path.empty()) { - return std::wstring(path) + L"\\local-packages"; - } - } + UINT32 nameLength = MAX_PATH; + std::wstring name; + name.resize(nameLength); + DWORD rc = GetCurrentPackageFamilyName(&nameLength, name.data()); + if (rc == ERROR_SUCCESS) { + name.resize(nameLength - 1); + return name; } - } catch (...) { + else if (rc != ERROR_INSUFFICIENT_BUFFER) { + throw rc; + } + name.resize(nameLength); + rc = GetCurrentPackageFamilyName(&nameLength, name.data()); + if (rc != ERROR_SUCCESS) { + throw rc; + } + name.resize(nameLength - 1); + return name; } + catch (...) { + } + return std::wstring(); } static std::wstring -get_package_family() +get_user_base() { try { - const auto package = winrt::Windows::ApplicationModel::Package::Current(); - if (package) { - const auto id = package.Id(); - if (id) { - return std::wstring(id.FamilyName()); + const auto appData = winrt::Windows::Storage::ApplicationData::Current(); + if (appData) { + const auto localCache = appData.LocalCacheFolder(); + if (localCache) { + std::wstring path { localCache.Path().c_str() }; + if (!path.empty()) { + return path + L"\\local-packages"; + } } } - } - catch (...) { + } catch (...) { } return std::wstring(); @@ -68,13 +81,24 @@ static std::wstring get_package_home() { try { - const auto package = winrt::Windows::ApplicationModel::Package::Current(); - if (package) { - const auto path = package.InstalledLocation(); - if (path) { - return std::wstring(path.Path()); - } + UINT32 pathLength = MAX_PATH; + std::wstring path; + path.resize(pathLength); + DWORD rc = GetCurrentPackagePath(&pathLength, path.data()); + if (rc == ERROR_SUCCESS) { + path.resize(pathLength - 1); + return path; + } + else if (rc != ERROR_INSUFFICIENT_BUFFER) { + throw rc; + } + path.resize(pathLength); + rc = GetCurrentPackagePath(&pathLength, path.data()); + if (rc != ERROR_SUCCESS) { + throw rc; } + path.resize(pathLength - 1); + return path; } catch (...) { } diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 0a74f5850a1e8e..442e3437dac905 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -424,6 +424,10 @@ + + + + diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index fc1049c5508187..11d6cba7a172c9 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -39,10 +39,10 @@ @if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86" @rem If HOST_PYTHON is recent enough, use that -@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found +@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 9)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found @rem If py.exe finds a recent enough version, use that one -@for %%p in (3.10 3.9 3.8) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found +@for %%p in (3.10 3.9) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" @set _Py_NUGET=%NUGET% diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index cd211f13e355e5..98cca979fdfcd0 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -52,14 +52,14 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 -if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.2 +if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.3 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.1q -set libraries=%libraries% sqlite-3.38.4.0 +set libraries=%libraries% sqlite-3.39.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.12.1 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.12.1 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tix-8.4.3.6 set libraries=%libraries% xz-5.2.5 -set libraries=%libraries% zlib-1.2.12 +set libraries=%libraries% zlib-1.2.13 for %%e in (%libraries%) do ( if exist "%EXTERNALS_DIR%\%%e" ( @@ -76,7 +76,7 @@ for %%e in (%libraries%) do ( echo.Fetching external binaries... set binaries= -if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.2 +if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.3 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-1.1.1q if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.12.1 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index 2ba0627b833695..222821adb1d76b 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -125,6 +125,12 @@ StopOnFirstFailure="false" Condition="%(CleanTarget) != ''" Targets="%(CleanTarget)" /> + @@ -140,6 +146,12 @@ StopOnFirstFailure="false" Condition="%(CleanAllTarget) != ''" Targets="%(CleanAllTarget)" /> + diff --git a/PCbuild/python.props b/PCbuild/python.props index efcb480976b6a2..320d41f4cc20d8 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -61,17 +61,17 @@ $(EXTERNALS_DIR) $([System.IO.Path]::GetFullPath(`$(PySourcePath)externals`)) $(ExternalsDir)\ - $(ExternalsDir)sqlite-3.38.4.0\ + $(ExternalsDir)sqlite-3.39.4.0\ $(ExternalsDir)bzip2-1.0.8\ $(ExternalsDir)xz-5.2.5\ - $(ExternalsDir)libffi-3.4.2\ - $(ExternalsDir)libffi-3.4.2\$(ArchName)\ + $(ExternalsDir)libffi-3.4.3\ + $(ExternalsDir)libffi-3.4.3\$(ArchName)\ $(libffiOutDir)include $(ExternalsDir)openssl-1.1.1q\ $(ExternalsDir)openssl-bin-1.1.1q\$(ArchName)\ $(opensslOutDir)include $(ExternalsDir)\nasm-2.11.06\ - $(ExternalsDir)\zlib-1.2.12\ + $(ExternalsDir)\zlib-1.2.13\ _d diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index a38040159e1fc8..078cc597a129f8 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -585,7 +585,7 @@ - + GITVERSION="$(GitVersion)";GITTAG="$(GitTag)";GITBRANCH="$(GitBranch)";%(PreprocessorDefinitions) diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index c536fac94cc680..5ba3e3940627b2 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -187,7 +187,7 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.38.4, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.39.4, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ _tkinter diff --git a/Parser/parser.c b/Parser/parser.c index 3fc12e50833c56..bee94fe1466696 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -17,18 +17,18 @@ static KeywordToken *reserved_keywords[] = { (KeywordToken[]) {{NULL, -1}}, (KeywordToken[]) {{NULL, -1}}, (KeywordToken[]) { - {"if", 634}, - {"as", 632}, - {"in", 643}, + {"if", 639}, + {"as", 637}, + {"in", 648}, {"or", 574}, {"is", 582}, {NULL, -1}, }, (KeywordToken[]) { {"del", 603}, - {"def", 644}, - {"for", 642}, - {"try", 618}, + {"def", 649}, + {"for", 647}, + {"try", 621}, {"and", 575}, {"not", 581}, {NULL, -1}, @@ -37,8 +37,8 @@ static KeywordToken *reserved_keywords[] = { {"from", 572}, {"pass", 504}, {"with", 612}, - {"elif", 636}, - {"else", 637}, + {"elif", 641}, + {"else", 642}, {"None", 601}, {"True", 600}, {NULL, -1}, @@ -47,8 +47,8 @@ static KeywordToken *reserved_keywords[] = { {"raise", 522}, {"yield", 573}, {"break", 508}, - {"class", 646}, - {"while", 639}, + {"class", 651}, + {"while", 644}, {"False", 602}, {NULL, -1}, }, @@ -57,12 +57,12 @@ static KeywordToken *reserved_keywords[] = { {"import", 531}, {"assert", 526}, {"global", 523}, - {"except", 629}, + {"except", 634}, {"lambda", 586}, {NULL, -1}, }, (KeywordToken[]) { - {"finally", 625}, + {"finally", 630}, {NULL, -1}, }, (KeywordToken[]) { @@ -509,10 +509,10 @@ static char *soft_keywords[] = { #define _gather_200_type 1429 #define _tmp_202_type 1430 #define _loop0_203_type 1431 -#define _tmp_204_type 1432 -#define _loop0_205_type 1433 -#define _tmp_206_type 1434 -#define _tmp_207_type 1435 +#define _loop1_204_type 1432 +#define _tmp_205_type 1433 +#define _loop0_206_type 1434 +#define _loop1_207_type 1435 #define _tmp_208_type 1436 #define _tmp_209_type 1437 #define _tmp_210_type 1438 @@ -520,11 +520,11 @@ static char *soft_keywords[] = { #define _tmp_212_type 1440 #define _tmp_213_type 1441 #define _tmp_214_type 1442 -#define _loop0_216_type 1443 -#define _gather_215_type 1444 +#define _tmp_215_type 1443 +#define _tmp_216_type 1444 #define _tmp_217_type 1445 -#define _tmp_218_type 1446 -#define _tmp_219_type 1447 +#define _loop0_219_type 1446 +#define _gather_218_type 1447 #define _tmp_220_type 1448 #define _tmp_221_type 1449 #define _tmp_222_type 1450 @@ -553,8 +553,8 @@ static char *soft_keywords[] = { #define _tmp_245_type 1473 #define _tmp_246_type 1474 #define _tmp_247_type 1475 -#define _loop1_248_type 1476 -#define _loop1_249_type 1477 +#define _tmp_248_type 1476 +#define _tmp_249_type 1477 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -988,10 +988,10 @@ static asdl_seq *_loop0_201_rule(Parser *p); static asdl_seq *_gather_200_rule(Parser *p); static void *_tmp_202_rule(Parser *p); static asdl_seq *_loop0_203_rule(Parser *p); -static void *_tmp_204_rule(Parser *p); -static asdl_seq *_loop0_205_rule(Parser *p); -static void *_tmp_206_rule(Parser *p); -static void *_tmp_207_rule(Parser *p); +static asdl_seq *_loop1_204_rule(Parser *p); +static void *_tmp_205_rule(Parser *p); +static asdl_seq *_loop0_206_rule(Parser *p); +static asdl_seq *_loop1_207_rule(Parser *p); static void *_tmp_208_rule(Parser *p); static void *_tmp_209_rule(Parser *p); static void *_tmp_210_rule(Parser *p); @@ -999,11 +999,11 @@ static void *_tmp_211_rule(Parser *p); static void *_tmp_212_rule(Parser *p); static void *_tmp_213_rule(Parser *p); static void *_tmp_214_rule(Parser *p); -static asdl_seq *_loop0_216_rule(Parser *p); -static asdl_seq *_gather_215_rule(Parser *p); +static void *_tmp_215_rule(Parser *p); +static void *_tmp_216_rule(Parser *p); static void *_tmp_217_rule(Parser *p); -static void *_tmp_218_rule(Parser *p); -static void *_tmp_219_rule(Parser *p); +static asdl_seq *_loop0_219_rule(Parser *p); +static asdl_seq *_gather_218_rule(Parser *p); static void *_tmp_220_rule(Parser *p); static void *_tmp_221_rule(Parser *p); static void *_tmp_222_rule(Parser *p); @@ -1032,8 +1032,8 @@ static void *_tmp_244_rule(Parser *p); static void *_tmp_245_rule(Parser *p); static void *_tmp_246_rule(Parser *p); static void *_tmp_247_rule(Parser *p); -static asdl_seq *_loop1_248_rule(Parser *p); -static asdl_seq *_loop1_249_rule(Parser *p); +static void *_tmp_248_rule(Parser *p); +static void *_tmp_249_rule(Parser *p); // file: statements? $ @@ -2019,7 +2019,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'if' if_stmt")); stmt_ty if_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 634) // token='if' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 639) // token='if' && (if_stmt_var = if_stmt_rule(p)) // if_stmt ) @@ -2103,7 +2103,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'try' try_stmt")); stmt_ty try_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 618) // token='try' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 621) // token='try' && (try_stmt_var = try_stmt_rule(p)) // try_stmt ) @@ -2124,7 +2124,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'while' while_stmt")); stmt_ty while_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 639) // token='while' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 644) // token='while' && (while_stmt_var = while_stmt_rule(p)) // while_stmt ) @@ -4266,7 +4266,7 @@ class_def_raw_rule(Parser *p) void *b; asdl_stmt_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 646)) // token='class' + (_keyword = _PyPegen_expect_token(p, 651)) // token='class' && (a = _PyPegen_name_token(p)) // NAME && @@ -4432,7 +4432,7 @@ function_def_raw_rule(Parser *p) void *params; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 644)) // token='def' + (_keyword = _PyPegen_expect_token(p, 649)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && @@ -4492,7 +4492,7 @@ function_def_raw_rule(Parser *p) if ( (async_var = _PyPegen_expect_token(p, ASYNC)) // token='ASYNC' && - (_keyword = _PyPegen_expect_token(p, 644)) // token='def' + (_keyword = _PyPegen_expect_token(p, 649)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && @@ -5844,7 +5844,7 @@ if_stmt_rule(Parser *p) asdl_stmt_seq* b; stmt_ty c; if ( - (_keyword = _PyPegen_expect_token(p, 634)) // token='if' + (_keyword = _PyPegen_expect_token(p, 639)) // token='if' && (a = named_expression_rule(p)) // named_expression && @@ -5889,7 +5889,7 @@ if_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 634)) // token='if' + (_keyword = _PyPegen_expect_token(p, 639)) // token='if' && (a = named_expression_rule(p)) // named_expression && @@ -5985,7 +5985,7 @@ elif_stmt_rule(Parser *p) asdl_stmt_seq* b; stmt_ty c; if ( - (_keyword = _PyPegen_expect_token(p, 636)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 641)) // token='elif' && (a = named_expression_rule(p)) // named_expression && @@ -6030,7 +6030,7 @@ elif_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 636)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 641)) // token='elif' && (a = named_expression_rule(p)) // named_expression && @@ -6112,7 +6112,7 @@ else_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 637)) // token='else' + (_keyword = _PyPegen_expect_token(p, 642)) // token='else' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -6192,7 +6192,7 @@ while_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 639)) // token='while' + (_keyword = _PyPegen_expect_token(p, 644)) // token='while' && (a = named_expression_rule(p)) // named_expression && @@ -6293,11 +6293,11 @@ for_stmt_rule(Parser *p) expr_ty t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 642)) // token='for' + (_keyword = _PyPegen_expect_token(p, 647)) // token='for' && (t = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 643)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 648)) // token='in' && (_cut_var = 1) && @@ -6357,11 +6357,11 @@ for_stmt_rule(Parser *p) if ( (async_var = _PyPegen_expect_token(p, ASYNC)) // token='ASYNC' && - (_keyword = _PyPegen_expect_token(p, 642)) // token='for' + (_keyword = _PyPegen_expect_token(p, 647)) // token='for' && (t = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 643)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 648)) // token='in' && (_cut_var = 1) && @@ -6729,7 +6729,7 @@ with_item_rule(Parser *p) if ( (e = expression_rule(p)) // expression && - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && (t = star_target_rule(p)) // star_target && @@ -6855,7 +6855,7 @@ try_stmt_rule(Parser *p) asdl_stmt_seq* b; asdl_stmt_seq* f; if ( - (_keyword = _PyPegen_expect_token(p, 618)) // token='try' + (_keyword = _PyPegen_expect_token(p, 621)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -6899,7 +6899,7 @@ try_stmt_rule(Parser *p) asdl_excepthandler_seq* ex; void *f; if ( - (_keyword = _PyPegen_expect_token(p, 618)) // token='try' + (_keyword = _PyPegen_expect_token(p, 621)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -6947,7 +6947,7 @@ try_stmt_rule(Parser *p) asdl_excepthandler_seq* ex; void *f; if ( - (_keyword = _PyPegen_expect_token(p, 618)) // token='try' + (_keyword = _PyPegen_expect_token(p, 621)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7046,7 +7046,7 @@ except_block_rule(Parser *p) expr_ty e; void *t; if ( - (_keyword = _PyPegen_expect_token(p, 629)) // token='except' + (_keyword = _PyPegen_expect_token(p, 634)) // token='except' && (e = expression_rule(p)) // expression && @@ -7089,7 +7089,7 @@ except_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 629)) // token='except' + (_keyword = _PyPegen_expect_token(p, 634)) // token='except' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -7201,7 +7201,7 @@ except_star_block_rule(Parser *p) expr_ty e; void *t; if ( - (_keyword = _PyPegen_expect_token(p, 629)) // token='except' + (_keyword = _PyPegen_expect_token(p, 634)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -7304,7 +7304,7 @@ finally_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 625)) // token='finally' + (_keyword = _PyPegen_expect_token(p, 630)) // token='finally' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7616,7 +7616,7 @@ guard_rule(Parser *p) Token * _keyword; expr_ty guard; if ( - (_keyword = _PyPegen_expect_token(p, 634)) // token='if' + (_keyword = _PyPegen_expect_token(p, 639)) // token='if' && (guard = named_expression_rule(p)) // named_expression ) @@ -7814,7 +7814,7 @@ as_pattern_rule(Parser *p) if ( (pattern = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && (target = pattern_capture_target_rule(p)) // pattern_capture_target ) @@ -10642,11 +10642,11 @@ expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 634)) // token='if' + (_keyword = _PyPegen_expect_token(p, 639)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 637)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 642)) // token='else' && (c = expression_rule(p)) // expression ) @@ -12203,7 +12203,7 @@ notin_bitwise_or_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 581)) // token='not' && - (_keyword_1 = _PyPegen_expect_token(p, 643)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 648)) // token='in' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -12250,7 +12250,7 @@ in_bitwise_or_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 643)) // token='in' + (_keyword = _PyPegen_expect_token(p, 648)) // token='in' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -16035,11 +16035,11 @@ for_if_clause_rule(Parser *p) if ( (async_var = _PyPegen_expect_token(p, ASYNC)) // token='ASYNC' && - (_keyword = _PyPegen_expect_token(p, 642)) // token='for' + (_keyword = _PyPegen_expect_token(p, 647)) // token='for' && (a = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 643)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 648)) // token='in' && (_cut_var = 1) && @@ -16078,11 +16078,11 @@ for_if_clause_rule(Parser *p) expr_ty b; asdl_expr_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 642)) // token='for' + (_keyword = _PyPegen_expect_token(p, 647)) // token='for' && (a = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 643)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 648)) // token='in' && (_cut_var = 1) && @@ -19284,11 +19284,11 @@ expression_without_invalid_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 634)) // token='if' + (_keyword = _PyPegen_expect_token(p, 639)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 637)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 642)) // token='else' && (c = expression_rule(p)) // expression ) @@ -19470,7 +19470,7 @@ invalid_expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 634)) // token='if' + (_keyword = _PyPegen_expect_token(p, 639)) // token='if' && (b = disjunction_rule(p)) // disjunction && @@ -21385,7 +21385,7 @@ invalid_with_item_rule(Parser *p) if ( (expression_var = expression_rule(p)) // expression && - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && (a = expression_rule(p)) // expression && @@ -21438,7 +21438,7 @@ invalid_for_target_rule(Parser *p) if ( (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && - (_keyword = _PyPegen_expect_token(p, 642)) // token='for' + (_keyword = _PyPegen_expect_token(p, 647)) // token='for' && (a = star_expressions_rule(p)) // star_expressions ) @@ -21808,7 +21808,8 @@ invalid_with_stmt_indent_rule(Parser *p) // invalid_try_stmt: // | 'try' ':' NEWLINE !INDENT // | 'try' ':' block !('except' | 'finally') -// | 'try' ':' block* ((except_block+ except_star_block) | (except_star_block+ except_block)) block* +// | 'try' ':' block* except_block+ 'except' '*' expression ['as' NAME] ':' +// | 'try' ':' block* except_star_block+ 'except' [expression ['as' NAME]] ':' static void * invalid_try_stmt_rule(Parser *p) { @@ -21832,7 +21833,7 @@ invalid_try_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 618)) // token='try' + (a = _PyPegen_expect_token(p, 621)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -21864,7 +21865,7 @@ invalid_try_stmt_rule(Parser *p) Token * _literal; asdl_stmt_seq* block_var; if ( - (_keyword = _PyPegen_expect_token(p, 618)) // token='try' + (_keyword = _PyPegen_expect_token(p, 621)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -21886,31 +21887,87 @@ invalid_try_stmt_rule(Parser *p) D(fprintf(stderr, "%*c%s invalid_try_stmt[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'try' ':' block !('except' | 'finally')")); } - { // 'try' ':' block* ((except_block+ except_star_block) | (except_star_block+ except_block)) block* + { // 'try' ':' block* except_block+ 'except' '*' expression ['as' NAME] ':' if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_try_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'try' ':' block* ((except_block+ except_star_block) | (except_star_block+ except_block)) block*")); + D(fprintf(stderr, "%*c> invalid_try_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'try' ':' block* except_block+ 'except' '*' expression ['as' NAME] ':'")); Token * _keyword; Token * _literal; + Token * _literal_1; asdl_seq * _loop0_203_var; - asdl_seq * _loop0_205_var; - void *_tmp_204_var; + asdl_seq * _loop1_204_var; + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + Token * a; + Token * b; + expr_ty expression_var; if ( - (_keyword = _PyPegen_expect_token(p, 618)) // token='try' + (_keyword = _PyPegen_expect_token(p, 621)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && (_loop0_203_var = _loop0_203_rule(p)) // block* && - (_tmp_204_var = _tmp_204_rule(p)) // (except_block+ except_star_block) | (except_star_block+ except_block) + (_loop1_204_var = _loop1_204_rule(p)) // except_block+ + && + (a = _PyPegen_expect_token(p, 634)) // token='except' + && + (b = _PyPegen_expect_token(p, 16)) // token='*' + && + (expression_var = expression_rule(p)) // expression + && + (_opt_var = _tmp_205_rule(p), !p->error_indicator) // ['as' NAME] + && + (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' + ) + { + D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' block* except_block+ 'except' '*' expression ['as' NAME] ':'")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "cannot have both 'except' and 'except*' on the same 'try'" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_try_stmt[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'try' ':' block* except_block+ 'except' '*' expression ['as' NAME] ':'")); + } + { // 'try' ':' block* except_star_block+ 'except' [expression ['as' NAME]] ':' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_try_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'try' ':' block* except_star_block+ 'except' [expression ['as' NAME]] ':'")); + Token * _keyword; + Token * _literal; + Token * _literal_1; + asdl_seq * _loop0_206_var; + asdl_seq * _loop1_207_var; + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + Token * a; + if ( + (_keyword = _PyPegen_expect_token(p, 621)) // token='try' + && + (_literal = _PyPegen_expect_token(p, 11)) // token=':' + && + (_loop0_206_var = _loop0_206_rule(p)) // block* + && + (_loop1_207_var = _loop1_207_rule(p)) // except_star_block+ + && + (a = _PyPegen_expect_token(p, 634)) // token='except' + && + (_opt_var = _tmp_208_rule(p), !p->error_indicator) // [expression ['as' NAME]] && - (_loop0_205_var = _loop0_205_rule(p)) // block* + (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' block* ((except_block+ except_star_block) | (except_star_block+ except_block)) block*")); - _res = RAISE_SYNTAX_ERROR ( "cannot have both 'except' and 'except*' on the same 'try'" ); + D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' block* except_star_block+ 'except' [expression ['as' NAME]] ':'")); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot have both 'except' and 'except*' on the same 'try'" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -21920,7 +21977,7 @@ invalid_try_stmt_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_try_stmt[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'try' ':' block* ((except_block+ except_star_block) | (except_star_block+ except_block)) block*")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'try' ':' block* except_star_block+ 'except' [expression ['as' NAME]] ':'")); } _res = NULL; done: @@ -21962,7 +22019,7 @@ invalid_except_stmt_rule(Parser *p) expr_ty a; expr_ty expressions_var; if ( - (_keyword = _PyPegen_expect_token(p, 629)) // token='except' + (_keyword = _PyPegen_expect_token(p, 634)) // token='except' && (_opt_var = _PyPegen_expect_token(p, 16), !p->error_indicator) // '*'? && @@ -21972,7 +22029,7 @@ invalid_except_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_opt_var_1 = _tmp_206_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var_1 = _tmp_209_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -22004,13 +22061,13 @@ invalid_except_stmt_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 629)) // token='except' + (a = _PyPegen_expect_token(p, 634)) // token='except' && (_opt_var = _PyPegen_expect_token(p, 16), !p->error_indicator) // '*'? && (expression_var = expression_rule(p)) // expression && - (_opt_var_1 = _tmp_207_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var_1 = _tmp_210_rule(p), !p->error_indicator) // ['as' NAME] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -22037,7 +22094,7 @@ invalid_except_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 629)) // token='except' + (a = _PyPegen_expect_token(p, 634)) // token='except' && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -22062,14 +22119,14 @@ invalid_except_stmt_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_except_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); Token * _literal; - void *_tmp_208_var; + void *_tmp_211_var; Token * a; if ( - (a = _PyPegen_expect_token(p, 629)) // token='except' + (a = _PyPegen_expect_token(p, 634)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_208_var = _tmp_208_rule(p)) // NEWLINE | ':' + (_tmp_211_var = _tmp_211_rule(p)) // NEWLINE | ':' ) { D(fprintf(stderr, "%*c+ invalid_except_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); @@ -22115,7 +22172,7 @@ invalid_finally_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 625)) // token='finally' + (a = _PyPegen_expect_token(p, 630)) // token='finally' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -22172,11 +22229,11 @@ invalid_except_stmt_indent_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 629)) // token='except' + (a = _PyPegen_expect_token(p, 634)) // token='except' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_209_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_212_rule(p), !p->error_indicator) // ['as' NAME] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -22208,7 +22265,7 @@ invalid_except_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 629)) // token='except' + (a = _PyPegen_expect_token(p, 634)) // token='except' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -22265,13 +22322,13 @@ invalid_except_star_stmt_indent_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 629)) // token='except' + (a = _PyPegen_expect_token(p, 634)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_210_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_213_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -22507,7 +22564,7 @@ invalid_as_pattern_rule(Parser *p) if ( (or_pattern_var = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && (a = _PyPegen_expect_soft_keyword(p, "_")) // soft_keyword='"_"' ) @@ -22537,7 +22594,7 @@ invalid_as_pattern_rule(Parser *p) if ( (or_pattern_var = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && _PyPegen_lookahead_with_name(0, _PyPegen_name_token, p) && @@ -22640,7 +22697,7 @@ invalid_class_argument_pattern_rule(Parser *p) asdl_pattern_seq* a; asdl_seq* keyword_patterns_var; if ( - (_opt_var = _tmp_211_rule(p), !p->error_indicator) // [positional_patterns ','] + (_opt_var = _tmp_214_rule(p), !p->error_indicator) // [positional_patterns ','] && (keyword_patterns_var = keyword_patterns_rule(p)) // keyword_patterns && @@ -22694,7 +22751,7 @@ invalid_if_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 634)) // token='if' + (_keyword = _PyPegen_expect_token(p, 639)) // token='if' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -22725,7 +22782,7 @@ invalid_if_stmt_rule(Parser *p) expr_ty a_1; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 634)) // token='if' + (a = _PyPegen_expect_token(p, 639)) // token='if' && (a_1 = named_expression_rule(p)) // named_expression && @@ -22781,7 +22838,7 @@ invalid_elif_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 636)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 641)) // token='elif' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -22812,7 +22869,7 @@ invalid_elif_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 636)) // token='elif' + (a = _PyPegen_expect_token(p, 641)) // token='elif' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -22866,7 +22923,7 @@ invalid_else_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 637)) // token='else' + (a = _PyPegen_expect_token(p, 642)) // token='else' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -22920,7 +22977,7 @@ invalid_while_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 639)) // token='while' + (_keyword = _PyPegen_expect_token(p, 644)) // token='while' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -22951,7 +23008,7 @@ invalid_while_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 639)) // token='while' + (a = _PyPegen_expect_token(p, 644)) // token='while' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -23013,11 +23070,11 @@ invalid_for_stmt_rule(Parser *p) if ( (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && - (_keyword = _PyPegen_expect_token(p, 642)) // token='for' + (_keyword = _PyPegen_expect_token(p, 647)) // token='for' && (star_targets_var = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 643)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 648)) // token='in' && (star_expressions_var = star_expressions_rule(p)) // star_expressions && @@ -23054,11 +23111,11 @@ invalid_for_stmt_rule(Parser *p) if ( (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && - (a = _PyPegen_expect_token(p, 642)) // token='for' + (a = _PyPegen_expect_token(p, 647)) // token='for' && (star_targets_var = star_targets_rule(p)) // star_targets && - (_keyword = _PyPegen_expect_token(p, 643)) // token='in' + (_keyword = _PyPegen_expect_token(p, 648)) // token='in' && (star_expressions_var = star_expressions_rule(p)) // star_expressions && @@ -23124,7 +23181,7 @@ invalid_def_raw_rule(Parser *p) if ( (_opt_var = _PyPegen_expect_token(p, ASYNC), !p->error_indicator) // ASYNC? && - (a = _PyPegen_expect_token(p, 644)) // token='def' + (a = _PyPegen_expect_token(p, 649)) // token='def' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -23134,7 +23191,7 @@ invalid_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (_opt_var_2 = _tmp_212_rule(p), !p->error_indicator) // ['->' expression] + (_opt_var_2 = _tmp_215_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23190,11 +23247,11 @@ invalid_class_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 646)) // token='class' + (_keyword = _PyPegen_expect_token(p, 651)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && - (_opt_var = _tmp_213_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var = _tmp_216_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -23225,11 +23282,11 @@ invalid_class_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 646)) // token='class' + (a = _PyPegen_expect_token(p, 651)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && - (_opt_var = _tmp_214_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var = _tmp_217_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23280,11 +23337,11 @@ invalid_double_starred_kvpairs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_double_starred_kvpairs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - asdl_seq * _gather_215_var; + asdl_seq * _gather_218_var; Token * _literal; void *invalid_kvpair_var; if ( - (_gather_215_var = _gather_215_rule(p)) // ','.double_starred_kvpair+ + (_gather_218_var = _gather_218_rule(p)) // ','.double_starred_kvpair+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -23292,7 +23349,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - _res = _PyPegen_dummy_name(p, _gather_215_var, _literal, invalid_kvpair_var); + _res = _PyPegen_dummy_name(p, _gather_218_var, _literal, invalid_kvpair_var); goto done; } p->mark = _mark; @@ -23345,7 +23402,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_217_rule, p) + _PyPegen_lookahead(1, _tmp_220_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -23456,7 +23513,7 @@ invalid_kvpair_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_218_rule, p) + _PyPegen_lookahead(1, _tmp_221_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -23890,7 +23947,7 @@ _tmp_7_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 644)) // token='def' + (_keyword = _PyPegen_expect_token(p, 649)) // token='def' ) { D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def'")); @@ -23967,7 +24024,7 @@ _tmp_8_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_8[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'class'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 646)) // token='class' + (_keyword = _PyPegen_expect_token(p, 651)) // token='class' ) { D(fprintf(stderr, "%*c+ _tmp_8[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class'")); @@ -24083,7 +24140,7 @@ _tmp_10_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_10[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'for'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 642)) // token='for' + (_keyword = _PyPegen_expect_token(p, 647)) // token='for' ) { D(fprintf(stderr, "%*c+ _tmp_10[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'for'")); @@ -24312,12 +24369,12 @@ _loop1_14_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_14[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_219_var; + void *_tmp_222_var; while ( - (_tmp_219_var = _tmp_219_rule(p)) // star_targets '=' + (_tmp_222_var = _tmp_222_rule(p)) // star_targets '=' ) { - _res = _tmp_219_var; + _res = _tmp_222_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -24894,12 +24951,12 @@ _loop0_24_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_24[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_220_var; + void *_tmp_223_var; while ( - (_tmp_220_var = _tmp_220_rule(p)) // '.' | '...' + (_tmp_223_var = _tmp_223_rule(p)) // '.' | '...' ) { - _res = _tmp_220_var; + _res = _tmp_223_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -24963,12 +25020,12 @@ _loop1_25_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_221_var; + void *_tmp_224_var; while ( - (_tmp_221_var = _tmp_221_rule(p)) // '.' | '...' + (_tmp_224_var = _tmp_224_rule(p)) // '.' | '...' ) { - _res = _tmp_221_var; + _res = _tmp_224_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -25150,7 +25207,7 @@ _tmp_28_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -25317,7 +25374,7 @@ _tmp_31_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -25371,12 +25428,12 @@ _loop1_32_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_32[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); - void *_tmp_222_var; + void *_tmp_225_var; while ( - (_tmp_222_var = _tmp_222_rule(p)) // '@' named_expression NEWLINE + (_tmp_225_var = _tmp_225_rule(p)) // '@' named_expression NEWLINE ) { - _res = _tmp_222_var; + _res = _tmp_225_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -27357,7 +27414,7 @@ _tmp_62_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -27404,7 +27461,7 @@ _tmp_63_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -28460,12 +28517,12 @@ _loop1_80_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_80[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' expression)")); - void *_tmp_223_var; + void *_tmp_226_var; while ( - (_tmp_223_var = _tmp_223_rule(p)) // ',' expression + (_tmp_226_var = _tmp_226_rule(p)) // ',' expression ) { - _res = _tmp_223_var; + _res = _tmp_226_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28534,12 +28591,12 @@ _loop1_81_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_81[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); - void *_tmp_224_var; + void *_tmp_227_var; while ( - (_tmp_224_var = _tmp_224_rule(p)) // ',' star_expression + (_tmp_227_var = _tmp_227_rule(p)) // ',' star_expression ) { - _res = _tmp_224_var; + _res = _tmp_227_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28728,12 +28785,12 @@ _loop1_84_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_84[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); - void *_tmp_225_var; + void *_tmp_228_var; while ( - (_tmp_225_var = _tmp_225_rule(p)) // 'or' conjunction + (_tmp_228_var = _tmp_228_rule(p)) // 'or' conjunction ) { - _res = _tmp_225_var; + _res = _tmp_228_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28802,12 +28859,12 @@ _loop1_85_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_85[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); - void *_tmp_226_var; + void *_tmp_229_var; while ( - (_tmp_226_var = _tmp_226_rule(p)) // 'and' inversion + (_tmp_229_var = _tmp_229_rule(p)) // 'and' inversion ) { - _res = _tmp_226_var; + _res = _tmp_229_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28999,7 +29056,7 @@ _loop0_89_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_227_rule(p)) // slice | starred_expression + (elem = _tmp_230_rule(p)) // slice | starred_expression ) { _res = elem; @@ -29065,7 +29122,7 @@ _gather_88_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_227_rule(p)) // slice | starred_expression + (elem = _tmp_230_rule(p)) // slice | starred_expression && (seq = _loop0_89_rule(p)) // _loop0_89 ) @@ -30769,12 +30826,12 @@ _loop0_114_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_228_var; + void *_tmp_231_var; while ( - (_tmp_228_var = _tmp_228_rule(p)) // 'if' disjunction + (_tmp_231_var = _tmp_231_rule(p)) // 'if' disjunction ) { - _res = _tmp_228_var; + _res = _tmp_231_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30838,12 +30895,12 @@ _loop0_115_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_229_var; + void *_tmp_232_var; while ( - (_tmp_229_var = _tmp_229_rule(p)) // 'if' disjunction + (_tmp_232_var = _tmp_232_rule(p)) // 'if' disjunction ) { - _res = _tmp_229_var; + _res = _tmp_232_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30972,7 +31029,7 @@ _loop0_118_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_230_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_233_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; @@ -31039,7 +31096,7 @@ _gather_117_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_230_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_233_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && (seq = _loop0_118_rule(p)) // _loop0_118 ) @@ -31615,12 +31672,12 @@ _loop0_128_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_231_var; + void *_tmp_234_var; while ( - (_tmp_231_var = _tmp_231_rule(p)) // ',' star_target + (_tmp_234_var = _tmp_234_rule(p)) // ',' star_target ) { - _res = _tmp_231_var; + _res = _tmp_234_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -31804,12 +31861,12 @@ _loop1_131_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_131[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_232_var; + void *_tmp_235_var; while ( - (_tmp_232_var = _tmp_232_rule(p)) // ',' star_target + (_tmp_235_var = _tmp_235_rule(p)) // ',' star_target ) { - _res = _tmp_232_var; + _res = _tmp_235_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32794,7 +32851,7 @@ _tmp_148_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 637)) // token='else' + (_keyword = _PyPegen_expect_token(p, 642)) // token='else' ) { D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); @@ -33179,12 +33236,12 @@ _loop0_153_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_233_var; + void *_tmp_236_var; while ( - (_tmp_233_var = _tmp_233_rule(p)) // star_targets '=' + (_tmp_236_var = _tmp_236_rule(p)) // star_targets '=' ) { - _res = _tmp_233_var; + _res = _tmp_236_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33248,12 +33305,12 @@ _loop0_154_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_234_var; + void *_tmp_237_var; while ( - (_tmp_234_var = _tmp_234_rule(p)) // star_targets '=' + (_tmp_237_var = _tmp_237_rule(p)) // star_targets '=' ) { - _res = _tmp_234_var; + _res = _tmp_237_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -34303,15 +34360,15 @@ _tmp_170_rule(Parser *p) } D(fprintf(stderr, "%*c> _tmp_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); Token * _literal; - void *_tmp_235_var; + void *_tmp_238_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_235_var = _tmp_235_rule(p)) // ')' | '**' + (_tmp_238_var = _tmp_238_rule(p)) // ')' | '**' ) { D(fprintf(stderr, "%*c+ _tmp_170[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_235_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_238_var); goto done; } p->mark = _mark; @@ -35487,15 +35544,15 @@ _tmp_188_rule(Parser *p) } D(fprintf(stderr, "%*c> _tmp_188[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); Token * _literal; - void *_tmp_236_var; + void *_tmp_239_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_236_var = _tmp_236_rule(p)) // ':' | '**' + (_tmp_239_var = _tmp_239_rule(p)) // ':' | '**' ) { D(fprintf(stderr, "%*c+ _tmp_188[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_236_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_239_var); goto done; } p->mark = _mark; @@ -35882,7 +35939,7 @@ _loop0_195_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_237_rule(p)) // expression ['as' star_target] + (elem = _tmp_240_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -35948,7 +36005,7 @@ _gather_194_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_237_rule(p)) // expression ['as' star_target] + (elem = _tmp_240_rule(p)) // expression ['as' star_target] && (seq = _loop0_195_rule(p)) // _loop0_195 ) @@ -36002,7 +36059,7 @@ _loop0_197_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_238_rule(p)) // expressions ['as' star_target] + (elem = _tmp_241_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -36068,7 +36125,7 @@ _gather_196_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_238_rule(p)) // expressions ['as' star_target] + (elem = _tmp_241_rule(p)) // expressions ['as' star_target] && (seq = _loop0_197_rule(p)) // _loop0_197 ) @@ -36122,7 +36179,7 @@ _loop0_199_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_239_rule(p)) // expression ['as' star_target] + (elem = _tmp_242_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -36188,7 +36245,7 @@ _gather_198_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_239_rule(p)) // expression ['as' star_target] + (elem = _tmp_242_rule(p)) // expression ['as' star_target] && (seq = _loop0_199_rule(p)) // _loop0_199 ) @@ -36242,7 +36299,7 @@ _loop0_201_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_240_rule(p)) // expressions ['as' star_target] + (elem = _tmp_243_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -36308,7 +36365,7 @@ _gather_200_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_240_rule(p)) // expressions ['as' star_target] + (elem = _tmp_243_rule(p)) // expressions ['as' star_target] && (seq = _loop0_201_rule(p)) // _loop0_201 ) @@ -36349,7 +36406,7 @@ _tmp_202_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 629)) // token='except' + (_keyword = _PyPegen_expect_token(p, 634)) // token='except' ) { D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); @@ -36368,7 +36425,7 @@ _tmp_202_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 625)) // token='finally' + (_keyword = _PyPegen_expect_token(p, 630)) // token='finally' ) { D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); @@ -36454,9 +36511,9 @@ _loop0_203_rule(Parser *p) return _seq; } -// _tmp_204: (except_block+ except_star_block) | (except_star_block+ except_block) -static void * -_tmp_204_rule(Parser *p) +// _loop1_204: except_block +static asdl_seq * +_loop1_204_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -36466,45 +36523,103 @@ _tmp_204_rule(Parser *p) p->level--; return NULL; } - void * _res = NULL; + void *_res = NULL; int _mark = p->mark; - { // (except_block+ except_star_block) + int _start_mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // except_block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(except_block+ except_star_block)")); - void *_tmp_241_var; - if ( - (_tmp_241_var = _tmp_241_rule(p)) // except_block+ except_star_block + D(fprintf(stderr, "%*c> _loop1_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); + excepthandler_ty except_block_var; + while ( + (except_block_var = except_block_rule(p)) // except_block ) { - D(fprintf(stderr, "%*c+ _tmp_204[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(except_block+ except_star_block)")); - _res = _tmp_241_var; - goto done; + _res = except_block_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_204[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(except_block+ except_star_block)")); + D(fprintf(stderr, "%*c%s _loop1_204[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_block")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + _PyPegen_insert_memo(p, _start_mark, _loop1_204_type, _seq); + p->level--; + return _seq; +} + +// _tmp_205: 'as' NAME +static void * +_tmp_205_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + p->error_indicator = 1; + PyErr_NoMemory(); + } + if (p->error_indicator) { + p->level--; + return NULL; } - { // (except_star_block+ except_block) + void * _res = NULL; + int _mark = p->mark; + { // 'as' NAME if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_204[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(except_star_block+ except_block)")); - void *_tmp_242_var; + D(fprintf(stderr, "%*c> _tmp_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + Token * _keyword; + expr_ty name_var; if ( - (_tmp_242_var = _tmp_242_rule(p)) // except_star_block+ except_block + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' + && + (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_204[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(except_star_block+ except_block)")); - _res = _tmp_242_var; + D(fprintf(stderr, "%*c+ _tmp_205[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_204[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(except_star_block+ except_block)")); + D(fprintf(stderr, "%*c%s _tmp_205[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; done: @@ -36512,9 +36627,9 @@ _tmp_204_rule(Parser *p) return _res; } -// _loop0_205: block +// _loop0_206: block static asdl_seq * -_loop0_205_rule(Parser *p) +_loop0_206_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -36541,7 +36656,7 @@ _loop0_205_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_205[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); + D(fprintf(stderr, "%*c> _loop0_206[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); asdl_stmt_seq* block_var; while ( (block_var = block_rule(p)) // block @@ -36563,7 +36678,7 @@ _loop0_205_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_205[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_206[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36576,14 +36691,131 @@ _loop0_205_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_205_type, _seq); + _PyPegen_insert_memo(p, _start_mark, _loop0_206_type, _seq); p->level--; return _seq; } -// _tmp_206: 'as' NAME +// _loop1_207: except_star_block +static asdl_seq * +_loop1_207_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + p->error_indicator = 1; + PyErr_NoMemory(); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + int _start_mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // except_star_block + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop1_207[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); + excepthandler_ty except_star_block_var; + while ( + (except_star_block_var = except_star_block_rule(p)) // except_star_block + ) + { + _res = except_star_block_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop1_207[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_star_block")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + _PyPegen_insert_memo(p, _start_mark, _loop1_207_type, _seq); + p->level--; + return _seq; +} + +// _tmp_208: expression ['as' NAME] +static void * +_tmp_208_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + p->error_indicator = 1; + PyErr_NoMemory(); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // expression ['as' NAME] + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_208[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + expr_ty expression_var; + if ( + (expression_var = expression_rule(p)) // expression + && + (_opt_var = _tmp_244_rule(p), !p->error_indicator) // ['as' NAME] + ) + { + D(fprintf(stderr, "%*c+ _tmp_208[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + _res = _PyPegen_dummy_name(p, expression_var, _opt_var); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_208[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' NAME]")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_209: 'as' NAME static void * -_tmp_206_rule(Parser *p) +_tmp_209_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -36600,21 +36832,21 @@ _tmp_206_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_206[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_209[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_206[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_209[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_206[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_209[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -36623,9 +36855,9 @@ _tmp_206_rule(Parser *p) return _res; } -// _tmp_207: 'as' NAME +// _tmp_210: 'as' NAME static void * -_tmp_207_rule(Parser *p) +_tmp_210_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -36642,21 +36874,21 @@ _tmp_207_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_207[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_210[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_207[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_210[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_207[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_210[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -36665,9 +36897,9 @@ _tmp_207_rule(Parser *p) return _res; } -// _tmp_208: NEWLINE | ':' +// _tmp_211: NEWLINE | ':' static void * -_tmp_208_rule(Parser *p) +_tmp_211_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -36684,18 +36916,18 @@ _tmp_208_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_208[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); Token * newline_var; if ( (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_208[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_211[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); _res = newline_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_208[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_211[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); } { // ':' @@ -36703,18 +36935,18 @@ _tmp_208_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_208[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_208[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_211[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_208[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_211[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -36723,9 +36955,9 @@ _tmp_208_rule(Parser *p) return _res; } -// _tmp_209: 'as' NAME +// _tmp_212: 'as' NAME static void * -_tmp_209_rule(Parser *p) +_tmp_212_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -36742,21 +36974,21 @@ _tmp_209_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_209[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_209[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_212[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_209[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_212[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -36765,9 +36997,9 @@ _tmp_209_rule(Parser *p) return _res; } -// _tmp_210: 'as' NAME +// _tmp_213: 'as' NAME static void * -_tmp_210_rule(Parser *p) +_tmp_213_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -36784,21 +37016,21 @@ _tmp_210_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_210[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_210[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_213[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_210[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_213[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -36807,9 +37039,9 @@ _tmp_210_rule(Parser *p) return _res; } -// _tmp_211: positional_patterns ',' +// _tmp_214: positional_patterns ',' static void * -_tmp_211_rule(Parser *p) +_tmp_214_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -36826,7 +37058,7 @@ _tmp_211_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_211[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c> _tmp_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); Token * _literal; asdl_pattern_seq* positional_patterns_var; if ( @@ -36835,12 +37067,12 @@ _tmp_211_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_211[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c+ _tmp_214[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); _res = _PyPegen_dummy_name(p, positional_patterns_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_211[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_214[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "positional_patterns ','")); } _res = NULL; @@ -36849,9 +37081,9 @@ _tmp_211_rule(Parser *p) return _res; } -// _tmp_212: '->' expression +// _tmp_215: '->' expression static void * -_tmp_212_rule(Parser *p) +_tmp_215_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -36868,7 +37100,7 @@ _tmp_212_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_212[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c> _tmp_215[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); Token * _literal; expr_ty expression_var; if ( @@ -36877,12 +37109,12 @@ _tmp_212_rule(Parser *p) (expression_var = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_212[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c+ _tmp_215[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); _res = _PyPegen_dummy_name(p, _literal, expression_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_212[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_215[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'->' expression")); } _res = NULL; @@ -36891,9 +37123,9 @@ _tmp_212_rule(Parser *p) return _res; } -// _tmp_213: '(' arguments? ')' +// _tmp_216: '(' arguments? ')' static void * -_tmp_213_rule(Parser *p) +_tmp_216_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -36910,7 +37142,7 @@ _tmp_213_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_213[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_216[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -36923,12 +37155,12 @@ _tmp_213_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_213[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_216[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_213[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_216[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -36937,9 +37169,9 @@ _tmp_213_rule(Parser *p) return _res; } -// _tmp_214: '(' arguments? ')' +// _tmp_217: '(' arguments? ')' static void * -_tmp_214_rule(Parser *p) +_tmp_217_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -36956,7 +37188,7 @@ _tmp_214_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_217[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -36969,12 +37201,12 @@ _tmp_214_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_214[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_217[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_214[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_217[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -36983,9 +37215,9 @@ _tmp_214_rule(Parser *p) return _res; } -// _loop0_216: ',' double_starred_kvpair +// _loop0_219: ',' double_starred_kvpair static asdl_seq * -_loop0_216_rule(Parser *p) +_loop0_219_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37012,7 +37244,7 @@ _loop0_216_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_216[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); + D(fprintf(stderr, "%*c> _loop0_219[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); Token * _literal; KeyValuePair* elem; while ( @@ -37043,7 +37275,7 @@ _loop0_216_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_216[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_219[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' double_starred_kvpair")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37056,14 +37288,14 @@ _loop0_216_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_216_type, _seq); + _PyPegen_insert_memo(p, _start_mark, _loop0_219_type, _seq); p->level--; return _seq; } -// _gather_215: double_starred_kvpair _loop0_216 +// _gather_218: double_starred_kvpair _loop0_219 static asdl_seq * -_gather_215_rule(Parser *p) +_gather_218_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37075,27 +37307,27 @@ _gather_215_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // double_starred_kvpair _loop0_216 + { // double_starred_kvpair _loop0_219 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_215[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_216")); + D(fprintf(stderr, "%*c> _gather_218[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_219")); KeyValuePair* elem; asdl_seq * seq; if ( (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair && - (seq = _loop0_216_rule(p)) // _loop0_216 + (seq = _loop0_219_rule(p)) // _loop0_219 ) { - D(fprintf(stderr, "%*c+ _gather_215[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_216")); + D(fprintf(stderr, "%*c+ _gather_218[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_219")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_215[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_216")); + D(fprintf(stderr, "%*c%s _gather_218[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_219")); } _res = NULL; done: @@ -37103,9 +37335,9 @@ _gather_215_rule(Parser *p) return _res; } -// _tmp_217: '}' | ',' +// _tmp_220: '}' | ',' static void * -_tmp_217_rule(Parser *p) +_tmp_220_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37122,18 +37354,18 @@ _tmp_217_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_217[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_217[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_217[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -37141,18 +37373,18 @@ _tmp_217_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_217[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_217[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_217[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -37161,9 +37393,9 @@ _tmp_217_rule(Parser *p) return _res; } -// _tmp_218: '}' | ',' +// _tmp_221: '}' | ',' static void * -_tmp_218_rule(Parser *p) +_tmp_221_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37180,18 +37412,18 @@ _tmp_218_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_218[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_218[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_218[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -37199,18 +37431,18 @@ _tmp_218_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_218[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_218[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_218[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -37219,9 +37451,9 @@ _tmp_218_rule(Parser *p) return _res; } -// _tmp_219: star_targets '=' +// _tmp_222: star_targets '=' static void * -_tmp_219_rule(Parser *p) +_tmp_222_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37238,7 +37470,7 @@ _tmp_219_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_219[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty z; if ( @@ -37247,7 +37479,7 @@ _tmp_219_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_219[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37257,7 +37489,7 @@ _tmp_219_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_219[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -37266,9 +37498,9 @@ _tmp_219_rule(Parser *p) return _res; } -// _tmp_220: '.' | '...' +// _tmp_223: '.' | '...' static void * -_tmp_220_rule(Parser *p) +_tmp_223_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37285,18 +37517,18 @@ _tmp_220_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -37304,18 +37536,18 @@ _tmp_220_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_220[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_220[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -37324,9 +37556,9 @@ _tmp_220_rule(Parser *p) return _res; } -// _tmp_221: '.' | '...' +// _tmp_224: '.' | '...' static void * -_tmp_221_rule(Parser *p) +_tmp_224_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37343,18 +37575,18 @@ _tmp_221_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_224[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_224[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_224[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -37362,18 +37594,18 @@ _tmp_221_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_221[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_224[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_221[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_224[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_221[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_224[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -37382,9 +37614,9 @@ _tmp_221_rule(Parser *p) return _res; } -// _tmp_222: '@' named_expression NEWLINE +// _tmp_225: '@' named_expression NEWLINE static void * -_tmp_222_rule(Parser *p) +_tmp_225_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37401,7 +37633,7 @@ _tmp_222_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_222[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); Token * _literal; expr_ty f; Token * newline_var; @@ -37413,7 +37645,7 @@ _tmp_222_rule(Parser *p) (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_222[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); _res = f; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37423,7 +37655,7 @@ _tmp_222_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_222[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@' named_expression NEWLINE")); } _res = NULL; @@ -37432,9 +37664,9 @@ _tmp_222_rule(Parser *p) return _res; } -// _tmp_223: ',' expression +// _tmp_226: ',' expression static void * -_tmp_223_rule(Parser *p) +_tmp_226_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37451,7 +37683,7 @@ _tmp_223_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_223[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _tmp_226[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty c; if ( @@ -37460,7 +37692,7 @@ _tmp_223_rule(Parser *p) (c = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_223[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c+ _tmp_226[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37470,7 +37702,7 @@ _tmp_223_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_223[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_226[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } _res = NULL; @@ -37479,9 +37711,9 @@ _tmp_223_rule(Parser *p) return _res; } -// _tmp_224: ',' star_expression +// _tmp_227: ',' star_expression static void * -_tmp_224_rule(Parser *p) +_tmp_227_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37498,7 +37730,7 @@ _tmp_224_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_224[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c> _tmp_227[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); Token * _literal; expr_ty c; if ( @@ -37507,7 +37739,7 @@ _tmp_224_rule(Parser *p) (c = star_expression_rule(p)) // star_expression ) { - D(fprintf(stderr, "%*c+ _tmp_224[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c+ _tmp_227[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37517,7 +37749,7 @@ _tmp_224_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_224[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_227[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_expression")); } _res = NULL; @@ -37526,9 +37758,9 @@ _tmp_224_rule(Parser *p) return _res; } -// _tmp_225: 'or' conjunction +// _tmp_228: 'or' conjunction static void * -_tmp_225_rule(Parser *p) +_tmp_228_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37545,7 +37777,7 @@ _tmp_225_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_225[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c> _tmp_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); Token * _keyword; expr_ty c; if ( @@ -37554,7 +37786,7 @@ _tmp_225_rule(Parser *p) (c = conjunction_rule(p)) // conjunction ) { - D(fprintf(stderr, "%*c+ _tmp_225[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c+ _tmp_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37564,7 +37796,7 @@ _tmp_225_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_225[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_228[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'or' conjunction")); } _res = NULL; @@ -37573,9 +37805,9 @@ _tmp_225_rule(Parser *p) return _res; } -// _tmp_226: 'and' inversion +// _tmp_229: 'and' inversion static void * -_tmp_226_rule(Parser *p) +_tmp_229_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37592,7 +37824,7 @@ _tmp_226_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_226[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c> _tmp_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); Token * _keyword; expr_ty c; if ( @@ -37601,7 +37833,7 @@ _tmp_226_rule(Parser *p) (c = inversion_rule(p)) // inversion ) { - D(fprintf(stderr, "%*c+ _tmp_226[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c+ _tmp_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37611,7 +37843,7 @@ _tmp_226_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_226[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_229[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'and' inversion")); } _res = NULL; @@ -37620,9 +37852,9 @@ _tmp_226_rule(Parser *p) return _res; } -// _tmp_227: slice | starred_expression +// _tmp_230: slice | starred_expression static void * -_tmp_227_rule(Parser *p) +_tmp_230_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37639,18 +37871,18 @@ _tmp_227_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_227[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c> _tmp_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); expr_ty slice_var; if ( (slice_var = slice_rule(p)) // slice ) { - D(fprintf(stderr, "%*c+ _tmp_227[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c+ _tmp_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); _res = slice_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_227[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_230[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slice")); } { // starred_expression @@ -37658,18 +37890,18 @@ _tmp_227_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_227[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_227[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_227[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_230[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } _res = NULL; @@ -37678,9 +37910,9 @@ _tmp_227_rule(Parser *p) return _res; } -// _tmp_228: 'if' disjunction +// _tmp_231: 'if' disjunction static void * -_tmp_228_rule(Parser *p) +_tmp_231_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37697,16 +37929,16 @@ _tmp_228_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 634)) // token='if' + (_keyword = _PyPegen_expect_token(p, 639)) // token='if' && (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37716,7 +37948,7 @@ _tmp_228_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_228[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -37725,9 +37957,9 @@ _tmp_228_rule(Parser *p) return _res; } -// _tmp_229: 'if' disjunction +// _tmp_232: 'if' disjunction static void * -_tmp_229_rule(Parser *p) +_tmp_232_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37744,16 +37976,16 @@ _tmp_229_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 634)) // token='if' + (_keyword = _PyPegen_expect_token(p, 639)) // token='if' && (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37763,7 +37995,7 @@ _tmp_229_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_229[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -37772,9 +38004,9 @@ _tmp_229_rule(Parser *p) return _res; } -// _tmp_230: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_233: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_230_rule(Parser *p) +_tmp_233_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37791,18 +38023,18 @@ _tmp_230_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_230[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -37810,20 +38042,20 @@ _tmp_230_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_243_var; + D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_245_var; if ( - (_tmp_243_var = _tmp_243_rule(p)) // assignment_expression | expression !':=' + (_tmp_245_var = _tmp_245_rule(p)) // assignment_expression | expression !':=' && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_243_var; + D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_245_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_230[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -37832,9 +38064,9 @@ _tmp_230_rule(Parser *p) return _res; } -// _tmp_231: ',' star_target +// _tmp_234: ',' star_target static void * -_tmp_231_rule(Parser *p) +_tmp_234_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37851,7 +38083,7 @@ _tmp_231_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -37860,7 +38092,7 @@ _tmp_231_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37870,7 +38102,7 @@ _tmp_231_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_231[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -37879,9 +38111,9 @@ _tmp_231_rule(Parser *p) return _res; } -// _tmp_232: ',' star_target +// _tmp_235: ',' star_target static void * -_tmp_232_rule(Parser *p) +_tmp_235_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37898,7 +38130,7 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -37907,7 +38139,7 @@ _tmp_232_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37917,7 +38149,7 @@ _tmp_232_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -37926,9 +38158,9 @@ _tmp_232_rule(Parser *p) return _res; } -// _tmp_233: star_targets '=' +// _tmp_236: star_targets '=' static void * -_tmp_233_rule(Parser *p) +_tmp_236_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37945,7 +38177,7 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -37954,12 +38186,12 @@ _tmp_233_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -37968,9 +38200,9 @@ _tmp_233_rule(Parser *p) return _res; } -// _tmp_234: star_targets '=' +// _tmp_237: star_targets '=' static void * -_tmp_234_rule(Parser *p) +_tmp_237_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -37987,7 +38219,7 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -37996,12 +38228,12 @@ _tmp_234_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -38010,9 +38242,9 @@ _tmp_234_rule(Parser *p) return _res; } -// _tmp_235: ')' | '**' +// _tmp_238: ')' | '**' static void * -_tmp_235_rule(Parser *p) +_tmp_238_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38029,18 +38261,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // '**' @@ -38048,18 +38280,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -38068,9 +38300,9 @@ _tmp_235_rule(Parser *p) return _res; } -// _tmp_236: ':' | '**' +// _tmp_239: ':' | '**' static void * -_tmp_236_rule(Parser *p) +_tmp_239_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38087,18 +38319,18 @@ _tmp_236_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '**' @@ -38106,18 +38338,18 @@ _tmp_236_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -38126,9 +38358,9 @@ _tmp_236_rule(Parser *p) return _res; } -// _tmp_237: expression ['as' star_target] +// _tmp_240: expression ['as' star_target] static void * -_tmp_237_rule(Parser *p) +_tmp_240_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38145,22 +38377,22 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_244_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_246_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -38169,9 +38401,9 @@ _tmp_237_rule(Parser *p) return _res; } -// _tmp_238: expressions ['as' star_target] +// _tmp_241: expressions ['as' star_target] static void * -_tmp_238_rule(Parser *p) +_tmp_241_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38188,22 +38420,22 @@ _tmp_238_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_245_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_247_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -38212,9 +38444,9 @@ _tmp_238_rule(Parser *p) return _res; } -// _tmp_239: expression ['as' star_target] +// _tmp_242: expression ['as' star_target] static void * -_tmp_239_rule(Parser *p) +_tmp_242_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38231,22 +38463,22 @@ _tmp_239_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_246_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_248_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -38255,9 +38487,9 @@ _tmp_239_rule(Parser *p) return _res; } -// _tmp_240: expressions ['as' star_target] +// _tmp_243: expressions ['as' star_target] static void * -_tmp_240_rule(Parser *p) +_tmp_243_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38274,22 +38506,22 @@ _tmp_240_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_247_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_249_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -38298,51 +38530,9 @@ _tmp_240_rule(Parser *p) return _res; } -// _tmp_241: except_block+ except_star_block -static void * -_tmp_241_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // except_block+ except_star_block - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block+ except_star_block")); - asdl_seq * _loop1_248_var; - excepthandler_ty except_star_block_var; - if ( - (_loop1_248_var = _loop1_248_rule(p)) // except_block+ - && - (except_star_block_var = except_star_block_rule(p)) // except_star_block - ) - { - D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "except_block+ except_star_block")); - _res = _PyPegen_dummy_name(p, _loop1_248_var, except_star_block_var); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_block+ except_star_block")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_242: except_star_block+ except_block +// _tmp_244: 'as' NAME static void * -_tmp_242_rule(Parser *p) +_tmp_244_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38354,27 +38544,27 @@ _tmp_242_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // except_star_block+ except_block + { // 'as' NAME if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block+ except_block")); - asdl_seq * _loop1_249_var; - excepthandler_ty except_block_var; + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + Token * _keyword; + expr_ty name_var; if ( - (_loop1_249_var = _loop1_249_rule(p)) // except_star_block+ + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && - (except_block_var = except_block_rule(p)) // except_block + (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "except_star_block+ except_block")); - _res = _PyPegen_dummy_name(p, _loop1_249_var, except_block_var); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_star_block+ except_block")); + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; done: @@ -38382,9 +38572,9 @@ _tmp_242_rule(Parser *p) return _res; } -// _tmp_243: assignment_expression | expression !':=' +// _tmp_245: assignment_expression | expression !':=' static void * -_tmp_243_rule(Parser *p) +_tmp_245_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38401,18 +38591,18 @@ _tmp_243_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -38420,7 +38610,7 @@ _tmp_243_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -38428,12 +38618,12 @@ _tmp_243_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; @@ -38442,9 +38632,9 @@ _tmp_243_rule(Parser *p) return _res; } -// _tmp_244: 'as' star_target +// _tmp_246: 'as' star_target static void * -_tmp_244_rule(Parser *p) +_tmp_246_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38461,21 +38651,21 @@ _tmp_244_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -38484,9 +38674,9 @@ _tmp_244_rule(Parser *p) return _res; } -// _tmp_245: 'as' star_target +// _tmp_247: 'as' star_target static void * -_tmp_245_rule(Parser *p) +_tmp_247_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38503,21 +38693,21 @@ _tmp_245_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -38526,9 +38716,9 @@ _tmp_245_rule(Parser *p) return _res; } -// _tmp_246: 'as' star_target +// _tmp_248: 'as' star_target static void * -_tmp_246_rule(Parser *p) +_tmp_248_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38545,21 +38735,21 @@ _tmp_246_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -38568,9 +38758,9 @@ _tmp_246_rule(Parser *p) return _res; } -// _tmp_247: 'as' star_target +// _tmp_249: 'as' star_target static void * -_tmp_247_rule(Parser *p) +_tmp_249_rule(Parser *p) { if (p->level++ == MAXSTACK) { p->error_indicator = 1; @@ -38587,21 +38777,21 @@ _tmp_247_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 632)) // token='as' + (_keyword = _PyPegen_expect_token(p, 637)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -38610,154 +38800,6 @@ _tmp_247_rule(Parser *p) return _res; } -// _loop1_248: except_block -static asdl_seq * -_loop1_248_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - int _start_mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // except_block - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop1_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); - excepthandler_ty except_block_var; - while ( - (except_block_var = except_block_rule(p)) // except_block - ) - { - _res = except_block_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_248[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_block")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_248_type, _seq); - p->level--; - return _seq; -} - -// _loop1_249: except_star_block -static asdl_seq * -_loop1_249_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - int _start_mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // except_star_block - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop1_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); - excepthandler_ty except_star_block_var; - while ( - (except_star_block_var = except_star_block_rule(p)) // except_star_block - ) - { - _res = except_star_block_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_249[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_star_block")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_249_type, _seq); - p->level--; - return _seq; -} - void * _PyPegen_parse(Parser *p) { diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 8d9fbf5cf9596a..ca11c7bebb4eb1 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -88,6 +88,7 @@ tok_new(void) tok->async_def_nl = 0; tok->interactive_underflow = IUNDERFLOW_NORMAL; tok->str = NULL; + tok->report_warnings = 1; return tok; } @@ -396,7 +397,11 @@ tok_readline_recode(struct tok_state *tok) { error_ret(tok); goto error; } - if (!tok_reserve_buf(tok, buflen + 1)) { + // Make room for the null terminator *and* potentially + // an extra newline character that we may need to artificially + // add. + size_t buffer_size = buflen + 2; + if (!tok_reserve_buf(tok, buffer_size)) { goto error; } memcpy(tok->inp, buf, buflen); @@ -983,6 +988,7 @@ tok_underflow_file(struct tok_state *tok) { return 0; } if (tok->inp[-1] != '\n') { + assert(tok->inp + 1 < tok->end); /* Last line does not end in \n, fake one */ *tok->inp++ = '\n'; *tok->inp = '\0'; @@ -1181,6 +1187,10 @@ indenterror(struct tok_state *tok) static int parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) { + if (!tok->report_warnings) { + return 0; + } + PyObject *errmsg; va_list vargs; #ifdef HAVE_STDARG_PROTOTYPES @@ -1542,7 +1552,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) } while (c == ' ' || c == '\t' || c == '\014'); /* Set start of current token */ - tok->start = tok->cur - 1; + tok->start = tok->cur == NULL ? NULL : tok->cur - 1; /* Skip comment, unless it's a type comment */ if (c == '#') { @@ -2189,6 +2199,9 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) return encoding; } } + // We don't want to report warnings here because it could cause infinite recursion + // if fetching the encoding shows a warning. + tok->report_warnings = 0; while (tok->lineno < 2 && tok->done == E_OK) { _PyTokenizer_Get(tok, &p_start, &p_end); } diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 0cb665104b2b86..d9a5f457d9c501 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -84,6 +84,7 @@ struct tok_state { NEWLINE token after it. */ /* How to proceed when asked for a new token in interactive mode */ enum interactive_underflow_t interactive_underflow; + int report_warnings; }; extern struct tok_state *_PyTokenizer_FromString(const char *, int); diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 3d27b79c237c36..7c4d60a500d638 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -194,6 +194,7 @@ write_frozen(const char *outpath, const char *inpath, const char *name, if (ferror(outfile)) { fprintf(stderr, "error when writing to '%s'\n", outpath); + fclose(outfile); return -1; } fclose(outfile); diff --git a/Programs/_testembed.c b/Programs/_testembed.c index d3d658828241df..13eae178f91d62 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -22,7 +22,7 @@ char **main_argv; /********************************************************* * Embedded interpreter tests that need a custom exe * - * Executed via 'EmbeddingTests' in Lib/test/test_capi.py + * Executed via Lib/test/test_embed.py *********************************************************/ // Use to display the usage @@ -73,7 +73,7 @@ static void init_from_config_clear(PyConfig *config) } -static void _testembed_Py_Initialize(void) +static void _testembed_Py_InitializeFromConfig(void) { PyConfig config; _PyConfig_InitCompatConfig(&config); @@ -81,6 +81,12 @@ static void _testembed_Py_Initialize(void) init_from_config_clear(&config); } +static void _testembed_Py_Initialize(void) +{ + Py_SetProgramName(PROGRAM_NAME); + Py_Initialize(); +} + /***************************************************** * Test repeated initialisation and subinterpreters @@ -110,7 +116,7 @@ static int test_repeated_init_and_subinterpreters(void) for (int i=1; i <= INIT_LOOPS; i++) { printf("--- Pass %d ---\n", i); - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); mainstate = PyThreadState_Get(); PyEval_ReleaseThread(mainstate); @@ -168,7 +174,7 @@ static int test_repeated_init_exec(void) fprintf(stderr, "--- Loop #%d ---\n", i); fflush(stderr); - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); int err = PyRun_SimpleString(code); Py_Finalize(); if (err) { @@ -178,6 +184,23 @@ static int test_repeated_init_exec(void) return 0; } +/**************************************************************************** + * Test the Py_Initialize(Ex) convenience/compatibility wrappers + ***************************************************************************/ +// This is here to help ensure there are no wrapper resource leaks (gh-96853) +static int test_repeated_simple_init(void) +{ + for (int i=1; i <= INIT_LOOPS; i++) { + fprintf(stderr, "--- Loop #%d ---\n", i); + fflush(stderr); + + _testembed_Py_Initialize(); + Py_Finalize(); + printf("Finalized\n"); // Give test_embed some output to check + } + return 0; +} + /***************************************************** * Test forcing a particular IO encoding @@ -199,7 +222,7 @@ static void check_stdio_details(const char *encoding, const char * errors) fflush(stdout); /* Force the given IO encoding */ Py_SetStandardStreamEncoding(encoding, errors); - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); PyRun_SimpleString( "import sys;" "print('stdin: {0.encoding}:{0.errors}'.format(sys.stdin));" @@ -308,7 +331,7 @@ static int test_pre_initialization_sys_options(void) dynamic_xoption = NULL; _Py_EMBED_PREINIT_CHECK("Initializing interpreter\n"); - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); _Py_EMBED_PREINIT_CHECK("Check sys module contents\n"); PyRun_SimpleString("import sys; " "print('sys.warnoptions:', sys.warnoptions); " @@ -352,7 +375,7 @@ static int test_bpo20891(void) return 1; } - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); unsigned long thrd = PyThread_start_new_thread(bpo20891_thread, &lock); if (thrd == PYTHREAD_INVALID_THREAD_ID) { @@ -375,7 +398,7 @@ static int test_bpo20891(void) static int test_initialize_twice(void) { - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); /* bpo-33932: Calling Py_Initialize() twice should do nothing * (and not crash!). */ @@ -393,7 +416,7 @@ static int test_initialize_pymain(void) L"print(f'Py_Main() after Py_Initialize: " L"sys.argv={sys.argv}')"), L"arg2"}; - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); /* bpo-34008: Calling Py_Main() after Py_Initialize() must not crash */ Py_Main(Py_ARRAY_LENGTH(argv), argv); @@ -416,7 +439,7 @@ dump_config(void) static int test_init_initialize_config(void) { - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); dump_config(); Py_Finalize(); return 0; @@ -765,7 +788,7 @@ static int test_init_compat_env(void) /* Test initialization from environment variables */ Py_IgnoreEnvironmentFlag = 0; set_all_env_vars(); - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); dump_config(); Py_Finalize(); return 0; @@ -801,7 +824,7 @@ static int test_init_env_dev_mode(void) /* Test initialization from environment variables */ Py_IgnoreEnvironmentFlag = 0; set_all_env_vars_dev_mode(); - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); dump_config(); Py_Finalize(); return 0; @@ -814,7 +837,7 @@ static int test_init_env_dev_mode_alloc(void) Py_IgnoreEnvironmentFlag = 0; set_all_env_vars_dev_mode(); putenv("PYTHONMALLOC=malloc"); - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); dump_config(); Py_Finalize(); return 0; @@ -1154,7 +1177,7 @@ static int test_open_code_hook(void) } Py_IgnoreEnvironmentFlag = 0; - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); result = 0; PyObject *r = PyFile_OpenCode("$$test-filename"); @@ -1218,7 +1241,7 @@ static int _test_audit(Py_ssize_t setValue) Py_IgnoreEnvironmentFlag = 0; PySys_AddAuditHook(_audit_hook, &sawSet); - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); if (PySys_Audit("_testembed.raise", NULL) == 0) { printf("No error raised"); @@ -1274,7 +1297,7 @@ static int test_audit_subinterpreter(void) { Py_IgnoreEnvironmentFlag = 0; PySys_AddAuditHook(_audit_subinterpreter_hook, NULL); - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); Py_NewInterpreter(); Py_NewInterpreter(); @@ -1581,7 +1604,7 @@ static int test_init_is_python_build(void) config._is_python_build = INT_MAX; env = getenv("NEGATIVE_ISPYTHONBUILD"); if (env && strcmp(env, "0") != 0) { - config._is_python_build++; + config._is_python_build = INT_MIN; } init_from_config_clear(&config); Py_Finalize(); @@ -1869,13 +1892,13 @@ static int test_unicode_id_init(void) _Py_IDENTIFIER(test_unicode_id_init); // Initialize Python once without using the identifier - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); Py_Finalize(); // Now initialize Python multiple times and use the identifier. // The first _PyUnicode_FromId() call initializes the identifier index. for (int i=0; i<3; i++) { - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); PyObject *str1, *str2; @@ -2007,7 +2030,7 @@ unwrap_allocator(PyMemAllocatorEx *allocator) static int test_get_incomplete_frame(void) { - _testembed_Py_Initialize(); + _testembed_Py_InitializeFromConfig(); PyMemAllocatorEx allocator; wrap_allocator(&allocator); // Force an allocation with an incomplete (generator) frame: @@ -2039,6 +2062,7 @@ struct TestCase static struct TestCase TestCases[] = { // Python initialization {"test_repeated_init_exec", test_repeated_init_exec}, + {"test_repeated_simple_init", test_repeated_simple_init}, {"test_forced_io_encoding", test_forced_io_encoding}, {"test_repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters}, {"test_repeated_init_and_inittab", test_repeated_init_and_inittab}, diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 072bf75bf8d697..94a7819c252e56 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -260,8 +260,8 @@ importlib.import_module() to programmatically import a module. The globals argument is only used to determine the context; they are not modified. The locals argument is unused. The fromlist -should be a list of names to emulate ``from name import ...'', or an -empty list to emulate ``import name''. +should be a list of names to emulate ``from name import ...``, or an +empty list to emulate ``import name``. When importing a module from a package, note that __import__('A.B', ...) returns package A when fromlist is empty, but its submodule B when fromlist is not empty. The level argument is used to determine whether to @@ -272,7 +272,7 @@ is the number of parent directories to search relative to the current module. static PyObject * builtin___import___impl(PyObject *module, PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) -/*[clinic end generated code: output=4febeda88a0cd245 input=35e9a6460412430f]*/ +/*[clinic end generated code: output=4febeda88a0cd245 input=73f4b960ea5b9dd6]*/ { return PyImport_ImportModuleLevelObject(name, globals, locals, fromlist, level); @@ -1509,13 +1509,13 @@ setattr as builtin_setattr Sets the named attribute on the given object to the specified value. -setattr(x, 'y', v) is equivalent to ``x.y = v'' +setattr(x, 'y', v) is equivalent to ``x.y = v`` [clinic start generated code]*/ static PyObject * builtin_setattr_impl(PyObject *module, PyObject *obj, PyObject *name, PyObject *value) -/*[clinic end generated code: output=dc2ce1d1add9acb4 input=bd2b7ca6875a1899]*/ +/*[clinic end generated code: output=dc2ce1d1add9acb4 input=5e26417f2e8598d4]*/ { if (PyObject_SetAttr(obj, name, value) != 0) return NULL; @@ -1532,12 +1532,12 @@ delattr as builtin_delattr Deletes the named attribute from the given object. -delattr(x, 'y') is equivalent to ``del x.y'' +delattr(x, 'y') is equivalent to ``del x.y`` [clinic start generated code]*/ static PyObject * builtin_delattr_impl(PyObject *module, PyObject *obj, PyObject *name) -/*[clinic end generated code: output=85134bc58dff79fa input=db16685d6b4b9410]*/ +/*[clinic end generated code: output=85134bc58dff79fa input=164865623abe7216]*/ { if (PyObject_SetAttr(obj, name, (PyObject *)NULL) != 0) return NULL; diff --git a/Python/ceval.c b/Python/ceval.c index 478ecd8e9219dd..a34e4ffb72f8f4 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1617,14 +1617,6 @@ trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject return 0; } -static _PyInterpreterFrame * -pop_frame(PyThreadState *tstate, _PyInterpreterFrame *frame) -{ - _PyInterpreterFrame *prev_frame = frame->previous; - _PyEvalFrameClearAndPop(tstate, frame); - return prev_frame; -} - /* It is only between the PRECALL instruction and the following CALL, * that this has any meaning. */ @@ -2158,6 +2150,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyObject *container = SECOND(); next_instr--; if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) { + next_instr++; goto error; } DISPATCH_SAME_OPARG(); @@ -2323,6 +2316,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyObject *container = SECOND(); next_instr--; if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) { + next_instr++; goto error; } DISPATCH_SAME_OPARG(); @@ -2439,7 +2433,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DTRACE_FUNCTION_EXIT(); _Py_LeaveRecursiveCallTstate(tstate); if (!frame->is_entry) { - frame = cframe.current_frame = pop_frame(tstate, frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = cframe.current_frame = dying->previous; + _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; } @@ -3056,6 +3053,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyObject *name = GETITEM(names, oparg>>1); next_instr--; if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) { + next_instr++; goto error; } DISPATCH_SAME_OPARG(); @@ -3481,6 +3479,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyObject *name = GETITEM(names, oparg); next_instr--; if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) { + next_instr++; goto error; } DISPATCH_SAME_OPARG(); @@ -3582,7 +3581,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DISPATCH(); } - DISPATCH(); TARGET(STORE_ATTR_ADAPTIVE) { assert(cframe.use_tracing == 0); _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -3591,6 +3589,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyObject *name = GETITEM(names, oparg); next_instr--; if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) { + next_instr++; goto error; } DISPATCH_SAME_OPARG(); @@ -4528,6 +4527,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyObject *name = GETITEM(names, oparg); next_instr--; if (_Py_Specialize_LoadMethod(owner, next_instr, name) < 0) { + next_instr++; goto error; } DISPATCH_SAME_OPARG(); @@ -4794,7 +4794,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(PRECALL_ADAPTIVE) { _PyPrecallCache *cache = (_PyPrecallCache *)next_instr; - if (cache->counter == 0) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { next_instr--; int is_meth = is_method(stack_pointer, oparg); int nargs = oparg + is_meth; @@ -4802,13 +4802,14 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int int err = _Py_Specialize_Precall(callable, next_instr, nargs, call_shape.kwnames, oparg); if (err < 0) { + next_instr++; goto error; } DISPATCH_SAME_OPARG(); } else { STAT_INC(PRECALL, deferred); - cache->counter--; + DECREMENT_ADAPTIVE_COUNTER(cache); JUMP_TO_INSTRUCTION(PRECALL); } } @@ -4823,6 +4824,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int int err = _Py_Specialize_Call(callable, next_instr, nargs, call_shape.kwnames); if (err < 0) { + next_instr++; goto error; } DISPATCH_SAME_OPARG(); @@ -5018,7 +5020,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto error; } PyObject *arg = TOP(); - PyObject *res = cfunc(PyCFunction_GET_SELF(callable), arg); + PyObject *res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -5185,9 +5187,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyObject *list = SECOND(); DEOPT_IF(!PyList_Check(list), PRECALL); STAT_INC(PRECALL, hit); - // PRECALL + CALL + POP_TOP - JUMPBY(INLINE_CACHE_ENTRIES_PRECALL + 1 + INLINE_CACHE_ENTRIES_CALL + 1); - assert(_Py_OPCODE(next_instr[-1]) == POP_TOP); PyObject *arg = POP(); if (_PyList_AppendTakeRef((PyListObject *)list, arg) < 0) { goto error; @@ -5195,6 +5194,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int STACK_SHRINK(2); Py_DECREF(list); Py_DECREF(callable); + // PRECALL + CALL + POP_TOP + JUMPBY(INLINE_CACHE_ENTRIES_PRECALL + 1 + INLINE_CACHE_ENTRIES_CALL + 1); + assert(_Py_OPCODE(next_instr[-1]) == POP_TOP); DISPATCH(); } @@ -5219,7 +5221,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { goto error; } - PyObject *res = cfunc(self, arg); + PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); @@ -5291,7 +5293,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { goto error; } - PyObject *res = cfunc(self, NULL); + PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); @@ -5747,9 +5749,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif /* Log traceback info. */ - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } } if (tstate->c_tracefunc != NULL) { @@ -5824,7 +5828,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(tstate->cframe->current_frame == frame->previous); return NULL; } - frame = cframe.current_frame = pop_frame(tstate, frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = cframe.current_frame = dying->previous; + _PyEvalFrameClearAndPop(tstate, dying); resume_with_error: SET_LOCALS_FROM_FRAME(); @@ -6931,7 +6938,7 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj, } } /* Always emit an opcode event if we're tracing all opcodes. */ - if (f->f_trace_opcodes) { + if (f->f_trace_opcodes && result == 0) { result = call_trace(func, obj, tstate, frame, PyTrace_OPCODE, Py_None); } return result; diff --git a/Python/ceval_gil.h b/Python/ceval_gil.h index 1b2dc7f8e1dc31..476ed7f1a2dd4b 100644 --- a/Python/ceval_gil.h +++ b/Python/ceval_gil.h @@ -133,12 +133,14 @@ static void destroy_gil(struct _gil_runtime_state *gil) _Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked); } +#ifdef HAVE_FORK static void recreate_gil(struct _gil_runtime_state *gil) { _Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked); /* XXX should we destroy the old OS resources here? */ create_gil(gil); } +#endif static void drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2, @@ -239,6 +241,7 @@ take_gil(PyThreadState *tstate) goto _ready; } + int drop_requested = 0; while (_Py_atomic_load_relaxed(&gil->locked)) { unsigned long saved_switchnum = gil->switch_number; @@ -254,11 +257,21 @@ take_gil(PyThreadState *tstate) { if (tstate_must_exit(tstate)) { MUTEX_UNLOCK(gil->mutex); + // gh-96387: If the loop requested a drop request in a previous + // iteration, reset the request. Otherwise, drop_gil() can + // block forever waiting for the thread which exited. Drop + // requests made by other threads are also reset: these threads + // may have to request again a drop request (iterate one more + // time). + if (drop_requested) { + RESET_GIL_DROP_REQUEST(interp); + } PyThread_exit_thread(); } assert(is_tstate_valid(tstate)); SET_GIL_DROP_REQUEST(interp); + drop_requested = 1; } } diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 48f65091164d04..5d9a16a7b68227 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -15,8 +15,8 @@ PyDoc_STRVAR(builtin___import____doc__, "\n" "The globals argument is only used to determine the context;\n" "they are not modified. The locals argument is unused. The fromlist\n" -"should be a list of names to emulate ``from name import ...\'\', or an\n" -"empty list to emulate ``import name\'\'.\n" +"should be a list of names to emulate ``from name import ...``, or an\n" +"empty list to emulate ``import name``.\n" "When importing a module from a package, note that __import__(\'A.B\', ...)\n" "returns package A when fromlist is empty, but its submodule B when\n" "fromlist is not empty. The level argument is used to determine whether to\n" @@ -539,7 +539,7 @@ PyDoc_STRVAR(builtin_setattr__doc__, "\n" "Sets the named attribute on the given object to the specified value.\n" "\n" -"setattr(x, \'y\', v) is equivalent to ``x.y = v\'\'"); +"setattr(x, \'y\', v) is equivalent to ``x.y = v``"); #define BUILTIN_SETATTR_METHODDEF \ {"setattr", _PyCFunction_CAST(builtin_setattr), METH_FASTCALL, builtin_setattr__doc__}, @@ -574,7 +574,7 @@ PyDoc_STRVAR(builtin_delattr__doc__, "\n" "Deletes the named attribute from the given object.\n" "\n" -"delattr(x, \'y\') is equivalent to ``del x.y\'\'"); +"delattr(x, \'y\') is equivalent to ``del x.y``"); #define BUILTIN_DELATTR_METHODDEF \ {"delattr", _PyCFunction_CAST(builtin_delattr), METH_FASTCALL, builtin_delattr__doc__}, @@ -1045,4 +1045,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=a2c5c53e8aead7c3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cc844ea007c1241f input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index 32fd58e6c4fd9d..f4555b35ab95a1 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2930,6 +2930,7 @@ compiler_jump_if(struct compiler *c, expr_ty e, basicblock *next, int cond) return 1; } case Compare_kind: { + SET_LOC(c, e); Py_ssize_t i, n = asdl_seq_LEN(e->v.Compare.ops) - 1; if (n > 0) { if (!check_compare(c, e)) { diff --git a/Python/frame.c b/Python/frame.c index 9d5cab990c312a..3ea3a2ced40d2b 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -85,6 +85,13 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) frame = (_PyInterpreterFrame *)f->_f_frame_data; f->f_frame = frame; frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; + if (_PyFrame_IsIncomplete(frame)) { + // This may be a newly-created generator or coroutine frame. Since it's + // dead anyways, just pretend that the first RESUME ran: + PyCodeObject *code = frame->f_code; + frame->prev_instr = _PyCode_CODE(code) + code->_co_firsttraceable; + } + assert(!_PyFrame_IsIncomplete(frame)); assert(f->f_back == NULL); _PyInterpreterFrame *prev = frame->previous; while (prev && _PyFrame_IsIncomplete(prev)) { @@ -116,6 +123,9 @@ _PyFrame_Clear(_PyInterpreterFrame *frame) * to have cleared the enclosing generator, if any. */ assert(frame->owner != FRAME_OWNED_BY_GENERATOR || _PyFrame_GetGenerator(frame)->gi_frame_state == FRAME_CLEARED); + // GH-99729: Clearing this frame can expose the stack (via finalizers). It's + // crucial that this frame has been unlinked, and is no longer visible: + assert(_PyThreadState_GET()->cframe->current_frame != frame); if (frame->frame_obj) { PyFrameObject *f = frame->frame_obj; frame->frame_obj = NULL; diff --git a/Python/import.c b/Python/import.c index ca728c424766c7..07a8b9009290f4 100644 --- a/Python/import.c +++ b/Python/import.c @@ -978,7 +978,8 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) if (_PyUnicode_EqualToASCIIString(name, p->name)) { if (p->initfunc == NULL) { /* Cannot re-init internal module ("sys" or "builtins") */ - return PyImport_AddModuleObject(name); + mod = PyImport_AddModuleObject(name); + return Py_XNewRef(mod); } mod = _PyImport_InitFunc_TrampolineCall(*p->initfunc); if (mod == NULL) { diff --git a/Python/marshal.c b/Python/marshal.c index 90a44050918006..2690f55766c83b 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -34,6 +34,8 @@ module marshal */ #if defined(MS_WINDOWS) #define MAX_MARSHAL_STACK_DEPTH 1000 +#elif defined(__wasi__) +#define MAX_MARSHAL_STACK_DEPTH 1500 #else #define MAX_MARSHAL_STACK_DEPTH 2000 #endif diff --git a/Python/mysnprintf.c b/Python/mysnprintf.c index cd69198011e3c9..2a505d14f82c99 100644 --- a/Python/mysnprintf.c +++ b/Python/mysnprintf.c @@ -9,6 +9,7 @@ would have been written had the buffer not been too small, and to set the last byte of the buffer to \0. At least MS _vsnprintf returns a negative value instead, and fills the entire buffer with non-\0 data. + Unlike C99, our wrappers do not support passing a null buffer. The wrappers ensure that str[size-1] is always \0 upon return. diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 69b7e10a3b0288..be0f97c4b204a9 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -261,6 +261,8 @@ Py_SetPythonHome(const wchar_t *home) _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); PyMem_RawFree(_Py_path_config.home); + _Py_path_config.home = NULL; + if (has_value) { _Py_path_config.home = _PyMem_RawWcsdup(home); } @@ -282,6 +284,8 @@ Py_SetProgramName(const wchar_t *program_name) _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); PyMem_RawFree(_Py_path_config.program_name); + _Py_path_config.program_name = NULL; + if (has_value) { _Py_path_config.program_name = _PyMem_RawWcsdup(program_name); } @@ -302,6 +306,8 @@ _Py_SetProgramFullPath(const wchar_t *program_full_path) _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); PyMem_RawFree(_Py_path_config.program_full_path); + _Py_path_config.program_full_path = NULL; + if (has_value) { _Py_path_config.program_full_path = _PyMem_RawWcsdup(program_full_path); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 960a38aebef8dd..4060c23ace1c45 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -82,6 +82,10 @@ int _Py_UnhandledKeyboardInterrupt = 0; * interpreter state for various runtime debugging tools, but is *not* an * officially supported feature */ +/* Suppress deprecation warning for PyBytesObject.ob_shash */ +_Py_COMP_DIAG_PUSH +_Py_COMP_DIAG_IGNORE_DEPR_DECLS + #if defined(MS_WINDOWS) #pragma section("PyRuntime", read, write) @@ -95,9 +99,6 @@ __attribute__(( #endif -/* Suppress deprecation warning for PyBytesObject.ob_shash */ -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS _PyRuntimeState _PyRuntime #if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) __attribute__ ((section (".PyRuntime"))) @@ -1289,6 +1290,7 @@ Py_InitializeEx(int install_sigs) config.install_signal_handlers = install_sigs; status = Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); if (_PyStatus_EXCEPTION(status)) { Py_ExitStatusException(status); } diff --git a/Python/pystate.c b/Python/pystate.c index 12ebbe33cb8aa5..c0d161f894c974 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -341,6 +341,7 @@ PyInterpreterState_New(void) interp = &runtime->_main_interpreter; assert(interp->id == 0); assert(interp->next == NULL); + assert(interp->_static); interpreters->main = interp; } @@ -355,6 +356,9 @@ PyInterpreterState_New(void) // Set to _PyInterpreterState_INIT. memcpy(interp, &initial._main_interpreter, sizeof(*interp)); + // We need to adjust any fields that are different from the initial + // interpreter (as defined in _PyInterpreterState_INIT): + interp->_static = false; if (id < 0) { /* overflow or Py_Initialize() not called yet! */ @@ -817,6 +821,7 @@ new_threadstate(PyInterpreterState *interp) assert(id == 1); used_newtstate = 0; tstate = &interp->_initial_thread; + assert(tstate->_static); } else { // Every valid interpreter must have at least one thread. @@ -828,6 +833,9 @@ new_threadstate(PyInterpreterState *interp) memcpy(tstate, &initial._main_interpreter._initial_thread, sizeof(*tstate)); + // We need to adjust any fields that are different from the initial + // thread (as defined in _PyThreadState_INIT): + tstate->_static = false; } interp->threads.head = tstate; @@ -1391,6 +1399,9 @@ _PyThread_CurrentFrames(void) PyThreadState *t; for (t = i->threads.head; t != NULL; t = t->next) { _PyInterpreterFrame *frame = t->cframe->current_frame; + while (frame && _PyFrame_IsIncomplete(frame)) { + frame = frame->previous; + } if (frame == NULL) { continue; } diff --git a/Python/specialize.c b/Python/specialize.c index b318cb5742faed..08ce2f5caa6bd9 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -688,6 +688,10 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) PyMemberDescrObject *member = (PyMemberDescrObject *)descr; struct PyMemberDef *dmem = member->d_member; Py_ssize_t offset = dmem->offset; + if (!PyObject_TypeCheck(owner, member->d_common.d_type)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); + goto fail; + } if (dmem->flags & PY_AUDIT_READ) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_AUDITED_SLOT); goto fail; @@ -777,6 +781,10 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) PyMemberDescrObject *member = (PyMemberDescrObject *)descr; struct PyMemberDef *dmem = member->d_member; Py_ssize_t offset = dmem->offset; + if (!PyObject_TypeCheck(owner, member->d_common.d_type)) { + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_EXPECTED_ERROR); + goto fail; + } if (dmem->flags & READONLY) { SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_READ_ONLY); goto fail; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index dca97f21a2d31e..0ecfd77ba14c97 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -440,6 +440,8 @@ sys_addaudithook_impl(PyObject *module, PyObject *hook) if (interp->audit_hooks == NULL) { return NULL; } + /* Avoid having our list of hooks show up in the GC module */ + PyObject_GC_UnTrack(interp->audit_hooks); } if (PyList_Append(interp->audit_hooks, hook) < 0) { @@ -2775,14 +2777,18 @@ EM_JS(char *, _Py_emscripten_runtime, (void), { if (typeof navigator == 'object') { info = navigator.userAgent; } else if (typeof process == 'object') { - info = "Node.js ".concat(process.version) + info = "Node.js ".concat(process.version); } else { - info = "UNKNOWN" + info = "UNKNOWN"; } var len = lengthBytesUTF8(info) + 1; var res = _malloc(len); - stringToUTF8(info, res, len); + if (res) stringToUTF8(info, res, len); +#if __wasm64__ + return BigInt(res); +#else return res; +#endif }); static PyObject * diff --git a/Python/traceback.c b/Python/traceback.c index 20348c06fd8020..7f47349a27a1ae 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -705,8 +705,13 @@ extract_anchors_from_line(PyObject *filename, PyObject *line, done: if (res > 0) { - *left_anchor += start_offset; - *right_anchor += start_offset; + // Normalize the AST offsets to byte offsets and adjust them with the + // start of the actual line (instead of the source code segment). + assert(segment != NULL); + assert(*left_anchor >= 0); + assert(*right_anchor >= 0); + *left_anchor = _PyPegen_byte_offset_to_character_offset(segment, *left_anchor) + start_offset; + *right_anchor = _PyPegen_byte_offset_to_character_offset(segment, *right_anchor) + start_offset; } Py_XDECREF(segment); if (arena) { diff --git a/README.rst b/README.rst index 8b5b52a3bdd2cd..8d9ace93b6c910 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.11.0 +This is Python version 3.11.1 ============================= .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg @@ -65,7 +65,7 @@ Building a complete Python installation requires the use of various additional third-party libraries, depending on your build platform and configure options. Not all standard library modules are buildable or useable on all platforms. Refer to the -`Install dependencies `_ +`Install dependencies `_ section of the `Developer Guide`_ for current detailed information on dependencies for various Linux distributions and macOS. @@ -135,7 +135,7 @@ What's New We have a comprehensive overview of the changes in the `What's New in Python 3.11 `_ document. For a more detailed change log, read `Misc/NEWS -`_, but a full +`_, but a full accounting of changes can only be gleaned from the `commit history `_. @@ -189,7 +189,7 @@ your environment, you can `file a bug report `_ and include relevant output from that command to show the issue. -See `Running & Writing Tests `_ +See `Running & Writing Tests `_ for more on running tests. Installing multiple versions diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index cd0446dc4f05d2..97d8d0a9410b63 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -3522,6 +3522,7 @@ def converter_init(self, *, accept={str}, zeroes=False): self.converter = '_PyUnicode_WideCharString_Opt_Converter' else: fail("Py_UNICODE_converter: illegal 'accept' argument " + repr(accept)) + self.c_default = "NULL" def cleanup(self): if not self.length: diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 6f889adffe6c7a..7ada79105db1ca 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -335,9 +335,10 @@ def __waiting(self, ttype, tstring, lineno): if ttype == tokenize.STRING and is_literal_string(tstring): self.__addentry(safe_eval(tstring), lineno, isdocstring=1) self.__freshmodule = 0 - elif ttype not in (tokenize.COMMENT, tokenize.NL): - self.__freshmodule = 0 - return + return + if ttype in (tokenize.COMMENT, tokenize.NL, tokenize.ENCODING): + return + self.__freshmodule = 0 # class or func/method docstring? if ttype == tokenize.NAME and tstring in ('class', 'def'): self.__state = self.__suiteseen diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py index 6a04297879f2cb..5b225f27c4f7cf 100644 --- a/Tools/scripts/generate_opcode_h.py +++ b/Tools/scripts/generate_opcode_h.py @@ -102,7 +102,7 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna opname_including_specialized[255] = 'DO_TRACING' used[255] = True - with (open(outfile, 'w') as fobj, open(internaloutfile, 'w') as iobj): + with open(outfile, 'w') as fobj, open(internaloutfile, 'w') as iobj: fobj.write(header) iobj.write(internal_header) diff --git a/Tools/scripts/get-remote-certificate.py b/Tools/scripts/get-remote-certificate.py index 38901286e19ad1..68272fca83fe27 100755 --- a/Tools/scripts/get-remote-certificate.py +++ b/Tools/scripts/get-remote-certificate.py @@ -15,8 +15,8 @@ def fetch_server_certificate (host, port): def subproc(cmd): - from subprocess import Popen, PIPE, STDOUT - proc = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=True) + from subprocess import Popen, PIPE, STDOUT, DEVNULL + proc = Popen(cmd, stdout=PIPE, stderr=STDOUT, stdin=DEVNULL) status = proc.wait() output = proc.stdout.read() return status, output @@ -33,8 +33,8 @@ def strip_to_x509_cert(certfile_contents, outfile=None): fp.write(m.group(1) + b"\n") try: tn2 = (outfile or tempfile.mktemp()) - status, output = subproc(r'openssl x509 -in "%s" -out "%s"' % - (tn, tn2)) + cmd = ['openssl', 'x509', '-in', tn, '-out', tn2] + status, output = subproc(cmd) if status != 0: raise RuntimeError('OpenSSL x509 failed with status %s and ' 'output: %r' % (status, output)) @@ -45,20 +45,9 @@ def strip_to_x509_cert(certfile_contents, outfile=None): finally: os.unlink(tn) - if sys.platform.startswith("win"): - tfile = tempfile.mktemp() - with open(tfile, "w") as fp: - fp.write("quit\n") - try: - status, output = subproc( - 'openssl s_client -connect "%s:%s" -showcerts < "%s"' % - (host, port, tfile)) - finally: - os.unlink(tfile) - else: - status, output = subproc( - 'openssl s_client -connect "%s:%s" -showcerts < /dev/null' % - (host, port)) + cmd = ['openssl', 's_client', '-connect', '%s:%s' % (host, port), '-showcerts'] + status, output = subproc(cmd) + if status != 0: raise RuntimeError('OpenSSL connect failed with status %s and ' 'output: %r' % (status, output)) diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 6496a29e6ff809..fe9a1dc99b30f4 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -1,11 +1,16 @@ # Python WebAssembly (WASM) build -**WARNING: WASM support is highly experimental! Lots of features are not working yet.** +**WARNING: WASM support is work-in-progress! Lots of features are not working yet.** This directory contains configuration and helpers to facilitate cross -compilation of CPython to WebAssembly (WASM). For now we support -*wasm32-emscripten* builds for modern browser and for *Node.js*. WASI -(*wasm32-wasi*) is work-in-progress +compilation of CPython to WebAssembly (WASM). Python supports Emscripten +(*wasm32-emscripten*) and WASI (*wasm32-wasi*) targets. Emscripten builds +run in modern browsers and JavaScript runtimes like *Node.js*. WASI builds +use WASM runtimes such as *wasmtime*. + +Users and developers are encouraged to use the script +`Tools/wasm/wasm_build.py`. The tool automates the build process and provides +assistance with installation of SDKs. ## wasm32-emscripten build @@ -17,7 +22,7 @@ access the file system directly. Cross compiling to the wasm32-emscripten platform needs the [Emscripten](https://emscripten.org/) SDK and a build Python interpreter. -Emscripten 3.1.8 or newer are recommended. All commands below are relative +Emscripten 3.1.19 or newer are recommended. All commands below are relative to a repository checkout. Christian Heimes maintains a container image with Emscripten SDK, Python @@ -35,7 +40,13 @@ docker run --rm -ti -v $(pwd):/python-wasm/cpython -w /python-wasm/cpython quay. ### Compile a build Python interpreter -From within the container, run the following commands: +From within the container, run the following command: + +```shell +./Tools/wasm/wasm_build.py build +``` + +The command is roughly equivalent to: ```shell mkdir -p builddir/build @@ -45,13 +56,13 @@ make -j$(nproc) popd ``` -### Fetch and build additional emscripten ports +### Cross-compile to wasm32-emscripten for browser ```shell -embuilder build zlib bzip2 +./Tools/wasm/wasm_build.py emscripten-browser ``` -### Cross compile to wasm32-emscripten for browser +The command is roughly equivalent to: ```shell mkdir -p builddir/emscripten-browser @@ -85,14 +96,21 @@ and header files with debug builds. ### Cross compile to wasm32-emscripten for node ```shell -mkdir -p builddir/emscripten-node -pushd builddir/emscripten-node +./Tools/wasm/wasm_build.py emscripten-browser-dl +``` + +The command is roughly equivalent to: + +```shell +mkdir -p builddir/emscripten-node-dl +pushd builddir/emscripten-node-dl CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \ emconfigure ../../configure -C \ --host=wasm32-unknown-emscripten \ --build=$(../../config.guess) \ --with-emscripten-target=node \ + --enable-wasm-dynamic-linking \ --with-build-python=$(pwd)/../build/python emmake make -j$(nproc) @@ -100,7 +118,7 @@ popd ``` ```shell -node --experimental-wasm-threads --experimental-wasm-bulk-memory --experimental-wasm-bigint builddir/emscripten-node/python.js +node --experimental-wasm-threads --experimental-wasm-bulk-memory --experimental-wasm-bigint builddir/emscripten-node-dl/python.js ``` (``--experimental-wasm-bigint`` is not needed with recent NodeJS versions) @@ -199,6 +217,15 @@ Node builds use ``NODERAWFS``. - Node RawFS allows direct access to the host file system without need to perform ``FS.mount()`` call. +## wasm64-emscripten + +- wasm64 requires recent NodeJS and ``--experimental-wasm-memory64``. +- ``EM_JS`` functions must return ``BigInt()``. +- ``Py_BuildValue()`` format strings must match size of types. Confusing 32 + and 64 bits types leads to memory corruption, see + [gh-95876](https://github.com/python/cpython/issues/95876) and + [gh-95878](https://github.com/python/cpython/issues/95878). + # Hosting Python WASM builds The simple REPL terminal uses SharedArrayBuffer. For security reasons @@ -234,6 +261,12 @@ The script ``wasi-env`` sets necessary compiler and linker flags as well as ``pkg-config`` overrides. The script assumes that WASI-SDK is installed in ``/opt/wasi-sdk`` or ``$WASI_SDK_PATH``. +```shell +./Tools/wasm/wasm_build.py wasi +``` + +The command is roughly equivalent to: + ```shell mkdir -p builddir/wasi pushd builddir/wasi @@ -308,26 +341,46 @@ if os.name == "posix": ```python >>> import os, sys >>> os.uname() -posix.uname_result(sysname='Emscripten', nodename='emscripten', release='1.0', version='#1', machine='wasm32') +posix.uname_result( + sysname='Emscripten', + nodename='emscripten', + release='3.1.19', + version='#1', + machine='wasm32' +) >>> os.name 'posix' >>> sys.platform 'emscripten' >>> sys._emscripten_info sys._emscripten_info( - emscripten_version=(3, 1, 8), - runtime='Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/99.0', + emscripten_version=(3, 1, 10), + runtime='Mozilla/5.0 (X11; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0', pthreads=False, shared_memory=False ) +``` + +```python >>> sys._emscripten_info -sys._emscripten_info(emscripten_version=(3, 1, 8), runtime='Node.js v14.18.2', pthreads=True, shared_memory=True) +sys._emscripten_info( + emscripten_version=(3, 1, 19), + runtime='Node.js v14.18.2', + pthreads=True, + shared_memory=True +) ``` ```python >>> import os, sys >>> os.uname() -posix.uname_result(sysname='wasi', nodename='(none)', release='0.0.0', version='0.0.0', machine='wasm32') +posix.uname_result( + sysname='wasi', + nodename='(none)', + release='0.0.0', + version='0.0.0', + machine='wasm32' +) >>> os.name 'posix' >>> sys.platform @@ -418,7 +471,8 @@ embuilder build --pic zlib bzip2 MINIMAL_PIC **NOTE**: WASI-SDK's clang may show a warning on Fedora: ``/lib64/libtinfo.so.6: no version information available``, -[RHBZ#1875587](https://bugzilla.redhat.com/show_bug.cgi?id=1875587). +[RHBZ#1875587](https://bugzilla.redhat.com/show_bug.cgi?id=1875587). The +warning can be ignored. ```shell export WASI_VERSION=16 @@ -443,6 +497,8 @@ ln -srf -t /usr/local/bin/ ~/.wasmtime/bin/wasmtime ### WASI debugging -* ``wasmtime run -g`` generates debugging symbols for gdb and lldb. +* ``wasmtime run -g`` generates debugging symbols for gdb and lldb. The + feature is currently broken, see + https://github.com/bytecodealliance/wasmtime/issues/4669 . * The environment variable ``RUST_LOG=wasi_common`` enables debug and trace logging. diff --git a/Tools/wasm/config.site-wasm32-emscripten b/Tools/wasm/config.site-wasm32-emscripten index a31d60d05dd770..b695a7bf8f04da 100644 --- a/Tools/wasm/config.site-wasm32-emscripten +++ b/Tools/wasm/config.site-wasm32-emscripten @@ -49,6 +49,10 @@ ac_cv_func_geteuid=no ac_cv_func_getegid=no ac_cv_func_seteuid=no ac_cv_func_setegid=no +ac_cv_func_getresuid=no +ac_cv_func_getresgid=no +ac_cv_func_setresuid=no +ac_cv_func_setresgid=no # Syscalls not implemented in emscripten # [Errno 52] Function not implemented diff --git a/Tools/wasm/wasi-env b/Tools/wasm/wasi-env index 6c2d56e0e5e32b..48908b02e60b96 100755 --- a/Tools/wasm/wasi-env +++ b/Tools/wasm/wasi-env @@ -72,4 +72,5 @@ export CFLAGS LDFLAGS export PKG_CONFIG_PATH PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR export PATH -exec "$@" +# no exec, it makes arvg[0] path absolute. +"$@" diff --git a/Tools/wasm/wasm_assets.py b/Tools/wasm/wasm_assets.py index 40acea2efaef27..6557e3f37aa834 100755 --- a/Tools/wasm/wasm_assets.py +++ b/Tools/wasm/wasm_assets.py @@ -58,6 +58,8 @@ # Pure Python implementations of C extensions "_pydecimal.py", "_pyio.py", + # concurrent threading + "concurrent/futures/thread.py", # Misc unused or large files "pydoc_data/", "msilib/", @@ -99,13 +101,12 @@ "_dbm": ["dbm/ndbm.py"], "_gdbm": ["dbm/gnu.py"], "_json": ["json/"], - "_multiprocessing": ["concurrent/", "multiprocessing/"], + "_multiprocessing": ["concurrent/futures/process.py", "multiprocessing/"], "pyexpat": ["xml/", "xmlrpc/"], "readline": ["rlcompleter.py"], "_sqlite3": ["sqlite3/"], "_ssl": ["ssl.py"], "_tkinter": ["idlelib/", "tkinter/", "turtle.py", "turtledemo/"], - "_zoneinfo": ["zoneinfo/"], } @@ -116,19 +117,28 @@ "unittest/test/", ) +SYSCONFIG_NAMES = ( + "_sysconfigdata__emscripten_wasm32-emscripten", + "_sysconfigdata__emscripten_wasm32-emscripten", + "_sysconfigdata__wasi_wasm32-wasi", + "_sysconfigdata__wasi_wasm64-wasi", +) + + def get_builddir(args: argparse.Namespace) -> pathlib.Path: - """Get builddir path from pybuilddir.txt - """ + """Get builddir path from pybuilddir.txt""" with open("pybuilddir.txt", encoding="utf-8") as f: builddir = f.read() return pathlib.Path(builddir) def get_sysconfigdata(args: argparse.Namespace) -> pathlib.Path: - """Get path to sysconfigdata relative to build root - """ + """Get path to sysconfigdata relative to build root""" data_name = sysconfig._get_sysconfigdata_name() - assert "emscripten_wasm32" in data_name + if not data_name.startswith(SYSCONFIG_NAMES): + raise ValueError( + f"Invalid sysconfig data name '{data_name}'.", SYSCONFIG_NAMES + ) filename = data_name + ".py" return args.builddir / filename @@ -138,20 +148,23 @@ def create_stdlib_zip( *, optimize: int = 0, ) -> None: - def filterfunc(name: str) -> bool: - return not name.startswith(args.omit_subdirs_absolute) + def filterfunc(filename: str) -> bool: + pathname = pathlib.Path(filename).resolve() + return pathname not in args.omit_files_absolute with zipfile.PyZipFile( - args.wasm_stdlib_zip, mode="w", compression=args.compression, optimize=optimize + args.wasm_stdlib_zip, + mode="w", + compression=args.compression, + optimize=optimize, ) as pzf: if args.compresslevel is not None: pzf.compresslevel = args.compresslevel pzf.writepy(args.sysconfig_data) for entry in sorted(args.srcdir_lib.iterdir()): + entry = entry.resolve() if entry.name == "__pycache__": continue - if entry in args.omit_files_absolute: - continue if entry.name.endswith(".py") or entry.is_dir(): # writepy() writes .pyc files (bytecode). pzf.writepy(entry, filterfunc=filterfunc) @@ -229,15 +242,15 @@ def main(): extmods = detect_extension_modules(args) omit_files = list(OMIT_FILES) - omit_files.extend(OMIT_NETWORKING_FILES) + if sysconfig.get_platform().startswith("emscripten"): + omit_files.extend(OMIT_NETWORKING_FILES) for modname, modfiles in OMIT_MODULE_FILES.items(): if not extmods.get(modname): omit_files.extend(modfiles) - args.omit_files_absolute = {args.srcdir_lib / name for name in omit_files} - args.omit_subdirs_absolute = tuple( - str(args.srcdir_lib / name) for name in OMIT_SUBDIRS - ) + args.omit_files_absolute = { + (args.srcdir_lib / name).resolve() for name in omit_files + } # Empty, unused directory for dynamic libs, but required for site initialization. args.wasm_dynload.mkdir(parents=True, exist_ok=True) diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py new file mode 100755 index 00000000000000..63812c6f3153f2 --- /dev/null +++ b/Tools/wasm/wasm_build.py @@ -0,0 +1,907 @@ +#!/usr/bin/env python3 +"""Build script for Python on WebAssembly platforms. + + $ ./Tools/wasm/wasm_builder.py emscripten-browser build repl + $ ./Tools/wasm/wasm_builder.py emscripten-node-dl build test + $ ./Tools/wasm/wasm_builder.py wasi build test + +Primary build targets are "emscripten-node-dl" (NodeJS, dynamic linking), +"emscripten-browser", and "wasi". + +Emscripten builds require a recent Emscripten SDK. The tools looks for an +activated EMSDK environment (". /path/to/emsdk_env.sh"). System packages +(Debian, Homebrew) are not supported. + +WASI builds require WASI SDK and wasmtime. The tool looks for 'WASI_SDK_PATH' +and falls back to /opt/wasi-sdk. + +The 'build' Python interpreter must be rebuilt every time Python's byte code +changes. + + ./Tools/wasm/wasm_builder.py --clean build build + +""" +import argparse +import enum +import dataclasses +import logging +import os +import pathlib +import re +import shlex +import shutil +import socket +import subprocess +import sys +import sysconfig +import tempfile +import time +import warnings +import webbrowser + +# for Python 3.8 +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union + +logger = logging.getLogger("wasm_build") + +SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute() +WASMTOOLS = SRCDIR / "Tools" / "wasm" +BUILDDIR = SRCDIR / "builddir" +CONFIGURE = SRCDIR / "configure" +SETUP_LOCAL = SRCDIR / "Modules" / "Setup.local" + +HAS_CCACHE = shutil.which("ccache") is not None + +# path to WASI-SDK root +WASI_SDK_PATH = pathlib.Path(os.environ.get("WASI_SDK_PATH", "/opt/wasi-sdk")) + +# path to Emscripten SDK config file. +# auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh". +EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten")) +EMSDK_MIN_VERSION = (3, 1, 19) +EMSDK_BROKEN_VERSION = { + (3, 1, 14): "https://github.com/emscripten-core/emscripten/issues/17338", + (3, 1, 16): "https://github.com/emscripten-core/emscripten/issues/17393", + (3, 1, 20): "https://github.com/emscripten-core/emscripten/issues/17720", +} +_MISSING = pathlib.PurePath("MISSING") + +WASM_WEBSERVER = WASMTOOLS / "wasm_webserver.py" + +CLEAN_SRCDIR = f""" +Builds require a clean source directory. Please use a clean checkout or +run "make clean -C '{SRCDIR}'". +""" + +INSTALL_NATIVE = f""" +Builds require a C compiler (gcc, clang), make, pkg-config, and development +headers for dependencies like zlib. + +Debian/Ubuntu: sudo apt install build-essential git curl pkg-config zlib1g-dev +Fedora/CentOS: sudo dnf install gcc make git-core curl pkgconfig zlib-devel +""" + +INSTALL_EMSDK = """ +wasm32-emscripten builds need Emscripten SDK. Please follow instructions at +https://emscripten.org/docs/getting_started/downloads.html how to install +Emscripten and how to activate the SDK with "emsdk_env.sh". + + git clone https://github.com/emscripten-core/emsdk.git /path/to/emsdk + cd /path/to/emsdk + ./emsdk install latest + ./emsdk activate latest + source /path/to/emsdk_env.sh +""" + +INSTALL_WASI_SDK = """ +wasm32-wasi builds need WASI SDK. Please fetch the latest SDK from +https://github.com/WebAssembly/wasi-sdk/releases and install it to +"/opt/wasi-sdk". Alternatively you can install the SDK in a different location +and point the environment variable WASI_SDK_PATH to the root directory +of the SDK. The SDK is available for Linux x86_64, macOS x86_64, and MinGW. +""" + +INSTALL_WASMTIME = """ +wasm32-wasi tests require wasmtime on PATH. Please follow instructions at +https://wasmtime.dev/ to install wasmtime. +""" + + +def parse_emconfig( + emconfig: pathlib.Path = EM_CONFIG, +) -> Tuple[pathlib.PurePath, pathlib.PurePath]: + """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT and NODE_JS. + + The ".emscripten" config file is a Python snippet that uses "EM_CONFIG" + environment variable. EMSCRIPTEN_ROOT is the "upstream/emscripten" + subdirectory with tools like "emconfigure". + """ + if not emconfig.exists(): + return _MISSING, _MISSING + with open(emconfig, encoding="utf-8") as f: + code = f.read() + # EM_CONFIG file is a Python snippet + local: Dict[str, Any] = {} + exec(code, globals(), local) + emscripten_root = pathlib.Path(local["EMSCRIPTEN_ROOT"]) + node_js = pathlib.Path(local["NODE_JS"]) + return emscripten_root, node_js + + +EMSCRIPTEN_ROOT, NODE_JS = parse_emconfig() + + +def read_python_version(configure: pathlib.Path = CONFIGURE) -> str: + """Read PACKAGE_VERSION from configure script + + configure and configure.ac are the canonical source for major and + minor version number. + """ + version_re = re.compile("^PACKAGE_VERSION='(\d\.\d+)'") + with configure.open(encoding="utf-8") as f: + for line in f: + mo = version_re.match(line) + if mo: + return mo.group(1) + raise ValueError(f"PACKAGE_VERSION not found in {configure}") + + +PYTHON_VERSION = read_python_version() + + +class ConditionError(ValueError): + def __init__(self, info: str, text: str): + self.info = info + self.text = text + + def __str__(self): + return f"{type(self).__name__}: '{self.info}'\n{self.text}" + + +class MissingDependency(ConditionError): + pass + + +class DirtySourceDirectory(ConditionError): + pass + + +@dataclasses.dataclass +class Platform: + """Platform-specific settings + + - CONFIG_SITE override + - configure wrapper (e.g. emconfigure) + - make wrapper (e.g. emmake) + - additional environment variables + - check function to verify SDK + """ + + name: str + pythonexe: str + config_site: Optional[pathlib.PurePath] + configure_wrapper: Optional[pathlib.PurePath] + make_wrapper: Optional[pathlib.PurePath] + environ: dict + check: Callable[[], None] + # Used for build_emports(). + ports: Optional[pathlib.PurePath] + cc: Optional[pathlib.PurePath] + + def getenv(self, profile: "BuildProfile") -> dict: + return self.environ.copy() + + +def _check_clean_src(): + candidates = [ + SRCDIR / "Programs" / "python.o", + SRCDIR / "Python" / "frozen_modules" / "importlib._bootstrap.h", + ] + for candidate in candidates: + if candidate.exists(): + raise DirtySourceDirectory(os.fspath(candidate), CLEAN_SRCDIR) + + +def _check_native(): + if not any(shutil.which(cc) for cc in ["cc", "gcc", "clang"]): + raise MissingDependency("cc", INSTALL_NATIVE) + if not shutil.which("make"): + raise MissingDependency("make", INSTALL_NATIVE) + if sys.platform == "linux": + # skip pkg-config check on macOS + if not shutil.which("pkg-config"): + raise MissingDependency("pkg-config", INSTALL_NATIVE) + # zlib is needed to create zip files + for devel in ["zlib"]: + try: + subprocess.check_call(["pkg-config", "--exists", devel]) + except subprocess.CalledProcessError: + raise MissingDependency(devel, INSTALL_NATIVE) from None + _check_clean_src() + + +NATIVE = Platform( + "native", + # macOS has python.exe + pythonexe=sysconfig.get_config_var("BUILDPYTHON") or "python", + config_site=None, + configure_wrapper=None, + ports=None, + cc=None, + make_wrapper=None, + environ={}, + check=_check_native, +) + + +def _check_emscripten(): + if EMSCRIPTEN_ROOT is _MISSING: + raise MissingDependency("Emscripten SDK EM_CONFIG", INSTALL_EMSDK) + # sanity check + emconfigure = EMSCRIPTEN.configure_wrapper + if not emconfigure.exists(): + raise MissingDependency(os.fspath(emconfigure), INSTALL_EMSDK) + # version check + version_txt = EMSCRIPTEN_ROOT / "emscripten-version.txt" + if not version_txt.exists(): + raise MissingDependency(os.fspath(version_txt), INSTALL_EMSDK) + with open(version_txt) as f: + version = f.read().strip().strip('"') + if version.endswith("-git"): + # git / upstream / tot-upstream installation + version = version[:-4] + version_tuple = tuple(int(v) for v in version.split(".")) + if version_tuple < EMSDK_MIN_VERSION: + raise ConditionError( + os.fspath(version_txt), + f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' is older than " + "minimum required version " + f"{'.'.join(str(v) for v in EMSDK_MIN_VERSION)}.", + ) + broken = EMSDK_BROKEN_VERSION.get(version_tuple) + if broken is not None: + raise ConditionError( + os.fspath(version_txt), + ( + f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' has known " + f"bugs, see {broken}." + ), + ) + if os.environ.get("PKG_CONFIG_PATH"): + warnings.warn( + "PKG_CONFIG_PATH is set and not empty. emconfigure overrides " + "this environment variable. Use EM_PKG_CONFIG_PATH instead." + ) + _check_clean_src() + + +EMSCRIPTEN = Platform( + "emscripten", + pythonexe="python.js", + config_site=WASMTOOLS / "config.site-wasm32-emscripten", + configure_wrapper=EMSCRIPTEN_ROOT / "emconfigure", + ports=EMSCRIPTEN_ROOT / "embuilder", + cc=EMSCRIPTEN_ROOT / "emcc", + make_wrapper=EMSCRIPTEN_ROOT / "emmake", + environ={ + # workaround for https://github.com/emscripten-core/emscripten/issues/17635 + "TZ": "UTC", + "EM_COMPILER_WRAPPER": "ccache" if HAS_CCACHE else None, + "PATH": [EMSCRIPTEN_ROOT, os.environ["PATH"]], + }, + check=_check_emscripten, +) + + +def _check_wasi(): + wasm_ld = WASI_SDK_PATH / "bin" / "wasm-ld" + if not wasm_ld.exists(): + raise MissingDependency(os.fspath(wasm_ld), INSTALL_WASI_SDK) + wasmtime = shutil.which("wasmtime") + if wasmtime is None: + raise MissingDependency("wasmtime", INSTALL_WASMTIME) + _check_clean_src() + + +WASI = Platform( + "wasi", + pythonexe="python.wasm", + config_site=WASMTOOLS / "config.site-wasm32-wasi", + configure_wrapper=WASMTOOLS / "wasi-env", + ports=None, + cc=WASI_SDK_PATH / "bin" / "clang", + make_wrapper=None, + environ={ + "WASI_SDK_PATH": WASI_SDK_PATH, + # workaround for https://github.com/python/cpython/issues/95952 + "HOSTRUNNER": ( + "wasmtime run " + "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib " + "--mapdir /::{srcdir} --" + ), + "PATH": [WASI_SDK_PATH / "bin", os.environ["PATH"]], + }, + check=_check_wasi, +) + + +class Host(enum.Enum): + """Target host triplet""" + + wasm32_emscripten = "wasm32-unknown-emscripten" + wasm64_emscripten = "wasm64-unknown-emscripten" + wasm32_wasi = "wasm32-unknown-wasi" + wasm64_wasi = "wasm64-unknown-wasi" + # current platform + build = sysconfig.get_config_var("BUILD_GNU_TYPE") + + @property + def platform(self) -> Platform: + if self.is_emscripten: + return EMSCRIPTEN + elif self.is_wasi: + return WASI + else: + return NATIVE + + @property + def is_emscripten(self) -> bool: + cls = type(self) + return self in {cls.wasm32_emscripten, cls.wasm64_emscripten} + + @property + def is_wasi(self) -> bool: + cls = type(self) + return self in {cls.wasm32_wasi, cls.wasm64_wasi} + + def get_extra_paths(self) -> Iterable[pathlib.PurePath]: + """Host-specific os.environ["PATH"] entries. + + Emscripten's Node version 14.x works well for wasm32-emscripten. + wasm64-emscripten requires more recent v8 version, e.g. node 16.x. + Attempt to use system's node command. + """ + cls = type(self) + if self == cls.wasm32_emscripten: + return [NODE_JS.parent] + elif self == cls.wasm64_emscripten: + # TODO: look for recent node + return [] + else: + return [] + + @property + def emport_args(self) -> List[str]: + """Host-specific port args (Emscripten).""" + cls = type(self) + if self is cls.wasm64_emscripten: + return ["-sMEMORY64=1"] + elif self is cls.wasm32_emscripten: + return ["-sMEMORY64=0"] + else: + return [] + + @property + def embuilder_args(self) -> List[str]: + """Host-specific embuilder args (Emscripten).""" + cls = type(self) + if self is cls.wasm64_emscripten: + return ["--wasm64"] + else: + return [] + + +class EmscriptenTarget(enum.Enum): + """Emscripten-specific targets (--with-emscripten-target)""" + + browser = "browser" + browser_debug = "browser-debug" + node = "node" + node_debug = "node-debug" + + @property + def is_browser(self): + cls = type(self) + return self in {cls.browser, cls.browser_debug} + + @property + def emport_args(self) -> List[str]: + """Target-specific port args.""" + cls = type(self) + if self in {cls.browser_debug, cls.node_debug}: + # some libs come in debug and non-debug builds + return ["-O0"] + else: + return ["-O2"] + + +class SupportLevel(enum.Enum): + supported = "tier 3, supported" + working = "working, unsupported" + experimental = "experimental, may be broken" + broken = "broken / unavailable" + + def __bool__(self): + cls = type(self) + return self in {cls.supported, cls.working} + + +@dataclasses.dataclass +class BuildProfile: + name: str + support_level: SupportLevel + host: Host + target: Union[EmscriptenTarget, None] = None + dynamic_linking: Union[bool, None] = None + pthreads: Union[bool, None] = None + default_testopts: str = "-j2" + + @property + def is_browser(self) -> bool: + """Is this a browser build?""" + return self.target is not None and self.target.is_browser + + @property + def builddir(self) -> pathlib.Path: + """Path to build directory""" + return BUILDDIR / self.name + + @property + def python_cmd(self) -> pathlib.Path: + """Path to python executable""" + return self.builddir / self.host.platform.pythonexe + + @property + def makefile(self) -> pathlib.Path: + """Path to Makefile""" + return self.builddir / "Makefile" + + @property + def configure_cmd(self) -> List[str]: + """Generate configure command""" + # use relative path, so WASI tests can find lib prefix. + # pathlib.Path.relative_to() does not work here. + configure = os.path.relpath(CONFIGURE, self.builddir) + cmd = [configure, "-C"] + platform = self.host.platform + if platform.configure_wrapper: + cmd.insert(0, os.fspath(platform.configure_wrapper)) + + cmd.append(f"--host={self.host.value}") + cmd.append(f"--build={Host.build.value}") + + if self.target is not None: + assert self.host.is_emscripten + cmd.append(f"--with-emscripten-target={self.target.value}") + + if self.dynamic_linking is not None: + assert self.host.is_emscripten + opt = "enable" if self.dynamic_linking else "disable" + cmd.append(f"--{opt}-wasm-dynamic-linking") + + if self.pthreads is not None: + assert self.host.is_emscripten + opt = "enable" if self.pthreads else "disable" + cmd.append(f"--{opt}-wasm-pthreads") + + if self.host != Host.build: + cmd.append(f"--with-build-python={BUILD.python_cmd}") + + if platform.config_site is not None: + cmd.append(f"CONFIG_SITE={platform.config_site}") + + return cmd + + @property + def make_cmd(self) -> List[str]: + """Generate make command""" + cmd = ["make"] + platform = self.host.platform + if platform.make_wrapper: + cmd.insert(0, os.fspath(platform.make_wrapper)) + return cmd + + def getenv(self) -> dict: + """Generate environ dict for platform""" + env = os.environ.copy() + env.setdefault("MAKEFLAGS", f"-j{os.cpu_count()}") + platenv = self.host.platform.getenv(self) + for key, value in platenv.items(): + if value is None: + env.pop(key, None) + elif key == "PATH": + # list of path items, prefix with extra paths + new_path: List[pathlib.PurePath] = [] + new_path.extend(self.host.get_extra_paths()) + new_path.extend(value) + env[key] = os.pathsep.join(os.fspath(p) for p in new_path) + elif isinstance(value, str): + env[key] = value.format( + relbuilddir=self.builddir.relative_to(SRCDIR), + srcdir=SRCDIR, + version=PYTHON_VERSION, + ) + else: + env[key] = value + return env + + def _run_cmd( + self, + cmd: Iterable[str], + args: Iterable[str] = (), + cwd: Optional[pathlib.Path] = None, + ): + cmd = list(cmd) + cmd.extend(args) + if cwd is None: + cwd = self.builddir + logger.info('Running "%s" in "%s"', shlex.join(cmd), cwd) + return subprocess.check_call( + cmd, + cwd=os.fspath(cwd), + env=self.getenv(), + ) + + def _check_execute(self): + if self.is_browser: + raise ValueError(f"Cannot execute on {self.target}") + + def run_build(self, *args): + """Run configure (if necessary) and make""" + if not self.makefile.exists(): + logger.info("Makefile not found, running configure") + self.run_configure(*args) + self.run_make("all", *args) + + def run_configure(self, *args): + """Run configure script to generate Makefile""" + os.makedirs(self.builddir, exist_ok=True) + return self._run_cmd(self.configure_cmd, args) + + def run_make(self, *args): + """Run make (defaults to build all)""" + return self._run_cmd(self.make_cmd, args) + + def run_pythoninfo(self, *args): + """Run 'make pythoninfo'""" + self._check_execute() + return self.run_make("pythoninfo", *args) + + def run_test(self, target: str, testopts: Optional[str] = None): + """Run buildbottests""" + self._check_execute() + if testopts is None: + testopts = self.default_testopts + return self.run_make(target, f"TESTOPTS={testopts}") + + def run_py(self, *args): + """Run Python with hostrunner""" + self._check_execute() + self.run_make( + "--eval", f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", "run" + ) + + def run_browser(self, bind="127.0.0.1", port=8000): + """Run WASM webserver and open build in browser""" + relbuilddir = self.builddir.relative_to(SRCDIR) + url = f"http://{bind}:{port}/{relbuilddir}/python.html" + args = [ + sys.executable, + os.fspath(WASM_WEBSERVER), + "--bind", + bind, + "--port", + str(port), + ] + srv = subprocess.Popen(args, cwd=SRCDIR) + # wait for server + end = time.monotonic() + 3.0 + while time.monotonic() < end and srv.returncode is None: + try: + with socket.create_connection((bind, port), timeout=0.1) as s: + pass + except OSError: + time.sleep(0.01) + else: + break + + webbrowser.open(url) + + try: + srv.wait() + except KeyboardInterrupt: + pass + + def clean(self, all: bool = False): + """Clean build directory""" + if all: + if self.builddir.exists(): + shutil.rmtree(self.builddir) + elif self.makefile.exists(): + self.run_make("clean") + + def build_emports(self, force: bool = False): + """Pre-build emscripten ports.""" + platform = self.host.platform + if platform.ports is None or platform.cc is None: + raise ValueError("Need ports and CC command") + + embuilder_cmd = [os.fspath(platform.ports)] + embuilder_cmd.extend(self.host.embuilder_args) + if force: + embuilder_cmd.append("--force") + + ports_cmd = [os.fspath(platform.cc)] + ports_cmd.extend(self.host.emport_args) + if self.target: + ports_cmd.extend(self.target.emport_args) + + if self.dynamic_linking: + # Trigger PIC build. + ports_cmd.append("-sMAIN_MODULE") + embuilder_cmd.append("--pic") + + if self.pthreads: + # Trigger multi-threaded build. + ports_cmd.append("-sUSE_PTHREADS") + + # Pre-build libbz2, libsqlite3, libz, and some system libs. + ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"]) + # Multi-threaded sqlite3 has different suffix + embuilder_cmd.extend( + ["build", "bzip2", "sqlite3-mt" if self.pthreads else "sqlite3", "zlib"] + ) + + self._run_cmd(embuilder_cmd, cwd=SRCDIR) + + with tempfile.TemporaryDirectory(suffix="-py-emport") as tmpdir: + tmppath = pathlib.Path(tmpdir) + main_c = tmppath / "main.c" + main_js = tmppath / "main.js" + with main_c.open("w") as f: + f.write("int main(void) { return 0; }\n") + args = [ + os.fspath(main_c), + "-o", + os.fspath(main_js), + ] + self._run_cmd(ports_cmd, args, cwd=tmppath) + + +# native build (build Python) +BUILD = BuildProfile( + "build", + support_level=SupportLevel.working, + host=Host.build, +) + +_profiles = [ + BUILD, + # wasm32-emscripten + BuildProfile( + "emscripten-browser", + support_level=SupportLevel.supported, + host=Host.wasm32_emscripten, + target=EmscriptenTarget.browser, + dynamic_linking=True, + ), + BuildProfile( + "emscripten-browser-debug", + support_level=SupportLevel.working, + host=Host.wasm32_emscripten, + target=EmscriptenTarget.browser_debug, + dynamic_linking=True, + ), + BuildProfile( + "emscripten-node-dl", + support_level=SupportLevel.supported, + host=Host.wasm32_emscripten, + target=EmscriptenTarget.node, + dynamic_linking=True, + ), + BuildProfile( + "emscripten-node-dl-debug", + support_level=SupportLevel.working, + host=Host.wasm32_emscripten, + target=EmscriptenTarget.node_debug, + dynamic_linking=True, + ), + BuildProfile( + "emscripten-node-pthreads", + support_level=SupportLevel.supported, + host=Host.wasm32_emscripten, + target=EmscriptenTarget.node, + pthreads=True, + ), + BuildProfile( + "emscripten-node-pthreads-debug", + support_level=SupportLevel.working, + host=Host.wasm32_emscripten, + target=EmscriptenTarget.node_debug, + pthreads=True, + ), + # Emscripten build with both pthreads and dynamic linking is crashing. + BuildProfile( + "emscripten-node-dl-pthreads-debug", + support_level=SupportLevel.broken, + host=Host.wasm32_emscripten, + target=EmscriptenTarget.node_debug, + dynamic_linking=True, + pthreads=True, + ), + # wasm64-emscripten (requires Emscripten >= 3.1.21) + BuildProfile( + "wasm64-emscripten-node-debug", + support_level=SupportLevel.experimental, + host=Host.wasm64_emscripten, + target=EmscriptenTarget.node_debug, + # MEMORY64 is not compatible with dynamic linking + dynamic_linking=False, + pthreads=False, + ), + # wasm32-wasi + BuildProfile( + "wasi", + support_level=SupportLevel.supported, + host=Host.wasm32_wasi, + ), + # no SDK available yet + # BuildProfile( + # "wasm64-wasi", + # support_level=SupportLevel.broken, + # host=Host.wasm64_wasi, + # ), +] + +PROFILES = {p.name: p for p in _profiles} + +parser = argparse.ArgumentParser( + "wasm_build.py", + description=__doc__, + formatter_class=argparse.RawTextHelpFormatter, +) + +parser.add_argument( + "--clean", + "-c", + help="Clean build directories first", + action="store_true", +) + +parser.add_argument( + "--verbose", + "-v", + help="Verbose logging", + action="store_true", +) + +parser.add_argument( + "--silent", + help="Run configure and make in silent mode", + action="store_true", +) + +parser.add_argument( + "--testopts", + help=( + "Additional test options for 'test' and 'hostrunnertest', e.g. " + "--testopts='-v test_os'." + ), + default=None, +) + +# Don't list broken and experimental variants in help +platforms_choices = list(p.name for p in _profiles) + ["cleanall"] +platforms_help = list(p.name for p in _profiles if p.support_level) + ["cleanall"] +parser.add_argument( + "platform", + metavar="PLATFORM", + help=f"Build platform: {', '.join(platforms_help)}", + choices=platforms_choices, +) + +ops = dict( + build="auto build (build 'build' Python, emports, configure, compile)", + configure="run ./configure", + compile="run 'make all'", + pythoninfo="run 'make pythoninfo'", + test="run 'make buildbottest TESTOPTS=...' (supports parallel tests)", + hostrunnertest="run 'make hostrunnertest TESTOPTS=...'", + repl="start interactive REPL / webserver + browser session", + clean="run 'make clean'", + cleanall="remove all build directories", + emports="build Emscripten port with embuilder (only Emscripten)", +) +ops_help = "\n".join(f"{op:16s} {help}" for op, help in ops.items()) +parser.add_argument( + "ops", + metavar="OP", + help=f"operation (default: build)\n\n{ops_help}", + choices=tuple(ops), + default="build", + nargs="*", +) + + +def main(): + args = parser.parse_args() + logging.basicConfig( + level=logging.INFO if args.verbose else logging.ERROR, + format="%(message)s", + ) + + if args.platform == "cleanall": + for builder in PROFILES.values(): + builder.clean(all=True) + parser.exit(0) + + # additional configure and make args + cm_args = ("--silent",) if args.silent else () + + # nargs=* with default quirk + if args.ops == "build": + args.ops = ["build"] + + builder = PROFILES[args.platform] + try: + builder.host.platform.check() + except ConditionError as e: + parser.error(str(e)) + + if args.clean: + builder.clean(all=False) + + # hack for WASI + if builder.host.is_wasi and not SETUP_LOCAL.exists(): + SETUP_LOCAL.touch() + + # auto-build + if "build" in args.ops: + # check and create build Python + if builder is not BUILD: + logger.info("Auto-building 'build' Python.") + try: + BUILD.host.platform.check() + except ConditionError as e: + parser.error(str(e)) + if args.clean: + BUILD.clean(all=False) + BUILD.run_build(*cm_args) + # build Emscripten ports with embuilder + if builder.host.is_emscripten and "emports" not in args.ops: + builder.build_emports() + + for op in args.ops: + logger.info("\n*** %s %s", args.platform, op) + if op == "build": + builder.run_build(*cm_args) + elif op == "configure": + builder.run_configure(*cm_args) + elif op == "compile": + builder.run_make("all", *cm_args) + elif op == "pythoninfo": + builder.run_pythoninfo(*cm_args) + elif op == "repl": + if builder.is_browser: + builder.run_browser() + else: + builder.run_py() + elif op == "test": + builder.run_test("buildbottest", testopts=args.testopts) + elif op == "hostrunnertest": + builder.run_test("hostrunnertest", testopts=args.testopts) + elif op == "clean": + builder.clean(all=False) + elif op == "cleanall": + builder.clean(all=True) + elif op == "emports": + builder.build_emports(force=args.clean) + else: + raise ValueError(op) + + print(builder.builddir) + parser.exit(0) + + +if __name__ == "__main__": + main() diff --git a/configure b/configure index 784f8d306092af..9567ac3a62c815 100755 --- a/configure +++ b/configure @@ -888,6 +888,7 @@ AR LINK_PYTHON_OBJS LINK_PYTHON_DEPS LIBRARY_DEPS +NODE HOSTRUNNER STATIC_LIBPYTHON GNULD @@ -4038,6 +4039,16 @@ if test -z "$CFLAGS"; then CFLAGS= fi +case $host in #( + wasm64-*-emscripten) : + + as_fn_append CFLAGS " -sMEMORY64=1" + as_fn_append LDFLAGS " -sMEMORY64=1" + ;; #( + *) : + ;; +esac + if test "$ac_sys_system" = "Darwin" then # Compiler selection on MacOSX is more complicated than @@ -6158,7 +6169,7 @@ cat > conftest.c <conftest.$ac_ext /* end confdefs.h. */ -int main() { return 0; } +int main(void) { return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : @@ -6778,19 +6789,162 @@ if test "$cross_compiling" = yes; then fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 -$as_echo_n "checking HOSTRUNNER... " >&6; } if test -z "$HOSTRUNNER" then case $ac_sys_system/$ac_sys_emscripten_target in #( Emscripten/node*) : + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}node", so it can be a program name with args. +set dummy ${ac_tool_prefix}node; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_NODE+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $NODE in + [\\/]* | ?:[\\/]*) + ac_cv_path_NODE="$NODE" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_NODE="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +NODE=$ac_cv_path_NODE +if test -n "$NODE"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $NODE" >&5 +$as_echo "$NODE" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_path_NODE"; then + ac_pt_NODE=$NODE + # Extract the first word of "node", so it can be a program name with args. +set dummy node; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_ac_pt_NODE+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $ac_pt_NODE in + [\\/]* | ?:[\\/]*) + ac_cv_path_ac_pt_NODE="$ac_pt_NODE" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_ac_pt_NODE="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +ac_pt_NODE=$ac_cv_path_ac_pt_NODE +if test -n "$ac_pt_NODE"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pt_NODE" >&5 +$as_echo "$ac_pt_NODE" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_pt_NODE" = x; then + NODE="node" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + NODE=$ac_pt_NODE + fi +else + NODE="$ac_cv_path_NODE" +fi + + HOSTRUNNER="$NODE" # bigint for ctypes c_longlong, c_longdouble - HOSTRUNNER="node --experimental-wasm-bigint" + # no longer available in Node 16 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for node --experimental-wasm-bigint" >&5 +$as_echo_n "checking for node --experimental-wasm-bigint... " >&6; } +if ${ac_cv_tool_node_wasm_bigint+:} false; then : + $as_echo_n "(cached) " >&6 +else + + if $NODE -v --experimental-wasm-bigint > /dev/null 2>&1; then + ac_cv_tool_node_wasm_bigint=yes + else + ac_cv_tool_node_wasm_bigint=no + fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_tool_node_wasm_bigint" >&5 +$as_echo "$ac_cv_tool_node_wasm_bigint" >&6; } + if test "x$ac_cv_tool_node_wasm_bigint" = xyes; then : + + as_fn_append HOSTRUNNER " --experimental-wasm-bigint" + +fi + if test "x$enable_wasm_pthreads" = xyes; then : - HOSTRUNNER="$HOSTRUNNER --experimental-wasm-threads --experimental-wasm-bulk-memory" + as_fn_append HOSTRUNNER " --experimental-wasm-threads" + # no longer available in Node 16 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for node --experimental-wasm-bulk-memory" >&5 +$as_echo_n "checking for node --experimental-wasm-bulk-memory... " >&6; } +if ${ac_cv_tool_node_wasm_bulk_memory+:} false; then : + $as_echo_n "(cached) " >&6 +else + + if $NODE -v --experimental-wasm-bulk-memory > /dev/null 2>&1; then + ac_cv_tool_node_wasm_bulk_memory=yes + else + ac_cv_tool_node_wasm_bulk_memory=no + fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_tool_node_wasm_bulk_memory" >&5 +$as_echo "$ac_cv_tool_node_wasm_bulk_memory" >&6; } + if test "x$ac_cv_tool_node_wasm_bulk_memory" = xyes; then : + + as_fn_append HOSTRUNNER " --experimental-wasm-bulk-memory" + +fi + +fi + if test "x$host_cpu" = xwasm64; then : + as_fn_append HOSTRUNNER " --experimental-wasm-memory64" fi ;; #( WASI/*) : @@ -6801,6 +6955,8 @@ fi esac fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 +$as_echo_n "checking HOSTRUNNER... " >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HOSTRUNNER" >&5 $as_echo "$HOSTRUNNER" >&6; } @@ -6814,7 +6970,7 @@ $as_echo "$LDLIBRARY" >&6; } # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable case $ac_sys_system/$ac_sys_emscripten_target in #( Emscripten/browser*) : - LIBRARY_DEPS='$(PY3LIBRARY) $(WASM_STDLIB)' ;; #( + LIBRARY_DEPS='$(PY3LIBRARY) $(WASM_STDLIB) python.html python.worker.js' ;; #( *) : LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)' ;; @@ -7410,7 +7566,7 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PROFILE_TASK" >&5 $as_echo "$PROFILE_TASK" >&6; } -# Make llvm-relatec checks work on systems where llvm tools are not installed with their +# Make llvm-related checks work on systems where llvm tools are not installed with their # normal names in the default $PATH (ie: Ubuntu). They exist under the # non-suffixed name in their versioned llvm directory. @@ -7475,7 +7631,42 @@ fi if test "$Py_LTO" = 'true' ; then case $CC in *clang*) - LDFLAGS_NOLTO="-fno-lto" + LDFLAGS_NOLTO="-fno-lto" + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -flto=thin" >&5 +$as_echo_n "checking whether C compiler accepts -flto=thin... " >&6; } +if ${ax_cv_check_cflags___flto_thin+:} false; then : + $as_echo_n "(cached) " >&6 +else + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -flto=thin" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ax_cv_check_cflags___flto_thin=yes +else + ax_cv_check_cflags___flto_thin=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___flto_thin" >&5 +$as_echo "$ax_cv_check_cflags___flto_thin" >&6; } +if test "x$ax_cv_check_cflags___flto_thin" = xyes; then : + LDFLAGS_NOLTO="-flto=thin" +else + LDFLAGS_NOLTO="-flto" +fi + if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}llvm-ar", so it can be a program name with args. @@ -8734,7 +8925,7 @@ else void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -8789,7 +8980,7 @@ else void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -8838,7 +9029,7 @@ else void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -8887,7 +9078,7 @@ else void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -11643,7 +11834,7 @@ else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -int main() +int main(void) { char s[16]; int i, *p1, *p2; @@ -11919,7 +12110,7 @@ else LIBEXPAT_CFLAGS="-I\$(srcdir)/Modules/expat" LIBEXPAT_LDFLAGS="-lm \$(LIBEXPAT_A)" - LIBEXPAT_INTERNAL="\$(LIBEXPAT_A)" + LIBEXPAT_INTERNAL="\$(LIBEXPAT_HEADERS) \$(LIBEXPAT_A)" fi @@ -11993,7 +12184,7 @@ else LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" LIBMPDEC_LDFLAGS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" if test "x$with_pydebug" = xyes; then : @@ -13953,6 +14144,7 @@ $as_echo_n "checking for pthread_create in -lpthread... " >&6; } /* end confdefs.h. */ #include +#include #include void * start_routine (void *arg) { exit (0); } @@ -14263,7 +14455,7 @@ else void *foo(void *parm) { return NULL; } - main() { + int main(void) { pthread_attr_t attr; pthread_t id; if (pthread_attr_init(&attr)) return (-1); @@ -15867,7 +16059,7 @@ else #include #include -int main(int argc, char*argv[]) +int main(int argc, char *argv[]) { if(chflags(argv[0], 0) != 0) return 1; @@ -15916,7 +16108,7 @@ else #include #include -int main(int argc, char*argv[]) +int main(int argc, char *argv[]) { if(lchflags(argv[0], 0) != 0) return 1; @@ -18610,7 +18802,7 @@ else #include #include -int main() +int main(void) { int passive, gaierr, inet4 = 0, inet6 = 0; struct addrinfo hints, *ai, *aitop; @@ -19853,7 +20045,7 @@ else #include #include -int main() { +int main(void) { volatile double x, y, z; /* 1./(1-2**-53) -> 1+2**-52 (correct), 1.0 (double rounding) */ x = 0.99999999999999989; /* 1-2**-53 */ @@ -20632,7 +20824,7 @@ else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -int main() +int main(void) { return (((-1)>>3 == -1) ? 0 : 1); } @@ -21078,7 +21270,7 @@ else #include #include -int main() +int main(void) { int val1 = nice(1); if (val1 != -1 && val1 == nice(2)) @@ -21120,7 +21312,7 @@ else #include #include -int main() +int main(void) { struct pollfd poll_struct = { 42, POLLIN|POLLPRI|POLLOUT, 0 }; int poll_test; @@ -21177,7 +21369,7 @@ else extern char *tzname[]; #endif -int main() +int main(void) { /* Note that we need to ensure that not only does tzset(3) do 'something' with localtime, but it works as documented @@ -22057,9 +22249,10 @@ else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ +#include #include -#include -int main() { +#include +int main(void) { size_t len = -1; const char *str = "text"; len = mbstowcs(NULL, str, 0); @@ -22258,7 +22451,7 @@ else #include #include void foo(void *p, void *q) { memmove(p, q, 19); } -int main() { +int main(void) { char a[32] = "123456789000000000"; foo(&a[9], a); if (strcmp(a, "123456789123456789000000000") != 0) @@ -22313,7 +22506,7 @@ else ); return r; } - int main() { + int main(void) { int p = 8; if ((foo(&p) ? : p) != 6) return 1; @@ -22356,7 +22549,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #include atomic_int int_var; atomic_uintptr_t uintptr_var; - int main() { + int main(void) { atomic_store_explicit(&int_var, 5, memory_order_relaxed); atomic_store_explicit(&uintptr_var, 0, memory_order_relaxed); int loaded_value = atomic_load_explicit(&int_var, memory_order_seq_cst); @@ -22397,7 +22590,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext int val; - int main() { + int main(void) { __atomic_store_n(&val, 1, __ATOMIC_SEQ_CST); (void)__atomic_load_n(&val, __ATOMIC_SEQ_CST); return 0; @@ -22473,7 +22666,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #include - int main() { + int main(void) { struct dirent entry; return entry.d_type == DT_UNKNOWN; } @@ -22511,11 +22704,12 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ + #include #include #include #include - int main() { + int main(void) { char buffer[1]; const size_t buflen = sizeof(buffer); const int flags = GRND_NONBLOCK; @@ -22558,9 +22752,10 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ + #include #include - int main() { + int main(void) { char buffer[1]; const size_t buflen = sizeof(buffer); const int flags = 0; @@ -23376,6 +23571,7 @@ case $ac_sys_system in #( py_cv_module_nis=n/a py_cv_module_ossaudiodev=n/a py_cv_module_pwd=n/a + py_cv_module_resource=n/a py_cv_module_spwd=n/a py_cv_module_syslog=n/a py_cv_module_=n/a @@ -23386,7 +23582,6 @@ case $ac_sys_system in #( py_cv_module_fcntl=n/a - py_cv_module_resource=n/a py_cv_module_readline=n/a py_cv_module_termios=n/a py_cv_module_=n/a @@ -23401,7 +23596,6 @@ case $ac_sys_system in #( py_cv_module__ctypes_test=n/a py_cv_module_fcntl=n/a py_cv_module_mmap=n/a - py_cv_module_resource=n/a py_cv_module_termios=n/a py_cv_module_=n/a diff --git a/configure.ac b/configure.ac index ab5e1de6fabd38..90008bcae17f16 100644 --- a/configure.ac +++ b/configure.ac @@ -753,6 +753,16 @@ if test -z "$CFLAGS"; then CFLAGS= fi +dnl Emscripten SDK and WASI SDK default to wasm32. +dnl On Emscripten use MEMORY64 setting to build target wasm64-emscripten. +dnl for wasm64. +AS_CASE([$host], + [wasm64-*-emscripten], [ + AS_VAR_APPEND([CFLAGS], [" -sMEMORY64=1"]) + AS_VAR_APPEND([LDFLAGS], [" -sMEMORY64=1"]) + ], +) + if test "$ac_sys_system" = "Darwin" then # Compiler selection on MacOSX is more complicated than @@ -1056,7 +1066,7 @@ cat > conftest.c < /dev/null 2>&1; then + ac_cv_tool_node_wasm_bigint=yes + else + ac_cv_tool_node_wasm_bigint=no + fi + ]) + AS_VAR_IF([ac_cv_tool_node_wasm_bigint], [yes], [ + AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-bigint"]) + ]) + AS_VAR_IF([enable_wasm_pthreads], [yes], [ - HOSTRUNNER="$HOSTRUNNER --experimental-wasm-threads --experimental-wasm-bulk-memory" + AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-threads"]) + # no longer available in Node 16 + AC_CACHE_CHECK([for node --experimental-wasm-bulk-memory], [ac_cv_tool_node_wasm_bulk_memory], [ + if $NODE -v --experimental-wasm-bulk-memory > /dev/null 2>&1; then + ac_cv_tool_node_wasm_bulk_memory=yes + else + ac_cv_tool_node_wasm_bulk_memory=no + fi + ]) + AS_VAR_IF([ac_cv_tool_node_wasm_bulk_memory], [yes], [ + AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-bulk-memory"]) + ]) ]) + + AS_VAR_IF([host_cpu], [wasm64], [AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-memory64"])]) ], dnl TODO: support other WASI runtimes dnl wasmtime starts the proces with "/" as CWD. For OOT builds add the @@ -1542,6 +1577,7 @@ then ) fi AC_SUBST([HOSTRUNNER]) +AC_MSG_CHECKING([HOSTRUNNER]) AC_MSG_RESULT([$HOSTRUNNER]) if test -n "$HOSTRUNNER"; then @@ -1553,7 +1589,7 @@ AC_MSG_RESULT($LDLIBRARY) # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/browser*], [LIBRARY_DEPS='$(PY3LIBRARY) $(WASM_STDLIB)'], + [Emscripten/browser*], [LIBRARY_DEPS='$(PY3LIBRARY) $(WASM_STDLIB) python.html python.worker.js'], [LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)'] ) LINK_PYTHON_DEPS='$(LIBRARY_DEPS)' @@ -1746,7 +1782,7 @@ then fi AC_MSG_RESULT($PROFILE_TASK) -# Make llvm-relatec checks work on systems where llvm tools are not installed with their +# Make llvm-related checks work on systems where llvm tools are not installed with their # normal names in the default $PATH (ie: Ubuntu). They exist under the # non-suffixed name in their versioned llvm directory. @@ -1800,8 +1836,10 @@ esac if test "$Py_LTO" = 'true' ; then case $CC in *clang*) - dnl flag to disable lto during linking LDFLAGS_NOLTO="-fno-lto" + dnl Clang linker requires -flto in order to link objects with LTO information. + dnl Thin LTO is faster and works for object files with full LTO information, too. + AX_CHECK_COMPILE_FLAG([-flto=thin],[LDFLAGS_NOLTO="-flto=thin"],[LDFLAGS_NOLTO="-flto"]) AC_SUBST(LLVM_AR) AC_PATH_TOOL(LLVM_AR, llvm-ar, '', ${llvm_path}) AC_SUBST(LLVM_AR_FOUND) @@ -2473,7 +2511,7 @@ AC_CACHE_CHECK([whether pthreads are available without options], void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -2506,7 +2544,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -2533,7 +2571,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -2560,7 +2598,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -3451,7 +3489,7 @@ esac # check for systems that require aligned memory access AC_CACHE_CHECK([aligned memory access is required], [ac_cv_aligned_required], [AC_RUN_IFELSE([AC_LANG_SOURCE([[ -int main() +int main(void) { char s[16]; int i, *p1, *p2; @@ -3572,7 +3610,7 @@ AS_VAR_IF([with_system_expat], [yes], [ ], [ LIBEXPAT_CFLAGS="-I\$(srcdir)/Modules/expat" LIBEXPAT_LDFLAGS="-lm \$(LIBEXPAT_A)" - LIBEXPAT_INTERNAL="\$(LIBEXPAT_A)" + LIBEXPAT_INTERNAL="\$(LIBEXPAT_HEADERS) \$(LIBEXPAT_A)" ]) AC_SUBST([LIBEXPAT_CFLAGS]) @@ -3628,7 +3666,7 @@ AS_VAR_IF([with_system_libmpdec], [yes], [ ], [ LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" LIBMPDEC_LDFLAGS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" dnl Disable forced inlining in debug builds, see GH-94847 AS_VAR_IF([with_pydebug], [yes], [ @@ -4065,6 +4103,7 @@ yes AC_MSG_CHECKING([for pthread_create in -lpthread]) AC_LINK_IFELSE([AC_LANG_PROGRAM([[ #include +#include #include void * start_routine (void *arg) { exit (0); }]], [[ @@ -4134,7 +4173,7 @@ if test "$posix_threads" = "yes"; then void *foo(void *parm) { return NULL; } - main() { + int main(void) { pthread_attr_t attr; pthread_t id; if (pthread_attr_init(&attr)) return (-1); @@ -4691,7 +4730,7 @@ AC_CACHE_CHECK([for chflags], [ac_cv_have_chflags], [dnl AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main(int argc, char*argv[]) +int main(int argc, char *argv[]) { if(chflags(argv[0], 0) != 0) return 1; @@ -4713,7 +4752,7 @@ AC_CACHE_CHECK([for lchflags], [ac_cv_have_lchflags], [dnl AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main(int argc, char*argv[]) +int main(int argc, char *argv[]) { if(lchflags(argv[0], 0) != 0) return 1; @@ -4986,7 +5025,7 @@ AS_VAR_IF([ac_cv_func_getaddrinfo], [yes], [ #include #include -int main() +int main(void) { int passive, gaierr, inet4 = 0, inet6 = 0; struct addrinfo hints, *ai, *aitop; @@ -5423,7 +5462,7 @@ CC="$CC $BASECFLAGS" AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main() { +int main(void) { volatile double x, y, z; /* 1./(1-2**-53) -> 1+2**-52 (correct), 1.0 (double rounding) */ x = 0.99999999999999989; /* 1-2**-53 */ @@ -5730,7 +5769,7 @@ fi], # or fills with zeros (like the Cray J90, according to Tim Peters). AC_CACHE_CHECK([whether right shift extends the sign bit], [ac_cv_rshift_extends_sign], [ AC_RUN_IFELSE([AC_LANG_SOURCE([[ -int main() +int main(void) { return (((-1)>>3 == -1) ? 0 : 1); } @@ -5885,7 +5924,7 @@ AC_CACHE_CHECK([for broken nice()], [ac_cv_broken_nice], [ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main() +int main(void) { int val1 = nice(1); if (val1 != -1 && val1 == nice(2)) @@ -5907,7 +5946,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main() +int main(void) { struct pollfd poll_struct = { 42, POLLIN|POLLPRI|POLLOUT, 0 }; int poll_test; @@ -5943,7 +5982,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ extern char *tzname[]; #endif -int main() +int main(void) { /* Note that we need to ensure that not only does tzset(3) do 'something' with localtime, but it works as documented @@ -6216,9 +6255,10 @@ AC_CHECK_TYPE(socklen_t,, AC_CACHE_CHECK([for broken mbstowcs], [ac_cv_broken_mbstowcs], AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include #include -#include -int main() { +#include +int main(void) { size_t len = -1; const char *str = "text"; len = mbstowcs(NULL, str, 0); @@ -6344,7 +6384,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include void foo(void *p, void *q) { memmove(p, q, 19); } -int main() { +int main(void) { char a[32] = "123456789000000000"; foo(&a[9], a); if (strcmp(a, "123456789123456789000000000") != 0) @@ -6385,7 +6425,7 @@ if test "$ac_cv_gcc_asm_for_x87" = yes; then ); return r; } - int main() { + int main(void) { int p = 8; if ((foo(&p) ? : p) != 6) return 1; @@ -6413,7 +6453,7 @@ AC_LINK_IFELSE( #include atomic_int int_var; atomic_uintptr_t uintptr_var; - int main() { + int main(void) { atomic_store_explicit(&int_var, 5, memory_order_relaxed); atomic_store_explicit(&uintptr_var, 0, memory_order_relaxed); int loaded_value = atomic_load_explicit(&int_var, memory_order_seq_cst); @@ -6434,7 +6474,7 @@ AC_LINK_IFELSE( [ AC_LANG_SOURCE([[ int val; - int main() { + int main(void) { __atomic_store_n(&val, 1, __ATOMIC_SEQ_CST); (void)__atomic_load_n(&val, __ATOMIC_SEQ_CST); return 0; @@ -6475,7 +6515,7 @@ AC_LINK_IFELSE( AC_LANG_SOURCE([[ #include - int main() { + int main(void) { struct dirent entry; return entry.d_type == DT_UNKNOWN; } @@ -6493,11 +6533,12 @@ AC_CACHE_CHECK([for the Linux getrandom() syscall], [ac_cv_getrandom_syscall], [ AC_LINK_IFELSE( [ AC_LANG_SOURCE([[ + #include #include #include #include - int main() { + int main(void) { char buffer[1]; const size_t buflen = sizeof(buffer); const int flags = GRND_NONBLOCK; @@ -6520,9 +6561,10 @@ AC_CACHE_CHECK([for the getrandom() function], [ac_cv_func_getrandom], [ AC_LINK_IFELSE( [ AC_LANG_SOURCE([[ + #include #include - int main() { + int main(void) { char buffer[1]; const size_t buflen = sizeof(buffer); const int flags = 0; @@ -6796,6 +6838,7 @@ AS_CASE([$ac_sys_system], dnl curses and tkinter user interface are not available. dnl dbm and gdbm aren't available, too. dnl Emscripten and WASI provide only stubs for pwd, grp APIs. + dnl resource functions (get/setrusage) are stubs, too. PY_STDLIB_MOD_SET_NA( [_curses], [_curses_panel], @@ -6811,6 +6854,7 @@ AS_CASE([$ac_sys_system], [nis], [ossaudiodev], [pwd], + [resource], [spwd], [syslog], ) @@ -6819,7 +6863,6 @@ AS_CASE([$ac_sys_system], dnl These modules are not particularly useful in browsers. PY_STDLIB_MOD_SET_NA( [fcntl], - [resource], [readline], [termios], ) @@ -6831,7 +6874,6 @@ AS_CASE([$ac_sys_system], [_ctypes_test], [fcntl], [mmap], - [resource], [termios], ) ]