From e6bb17fe29713368e1fd93d9ac9611017c4f570c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 13 Jan 2022 10:50:09 -0800 Subject: [PATCH 001/220] bpo-46070: _PyGC_Fini() untracks objects (GH-30577) Py_EndInterpreter() now explicitly untracks all objects currently tracked by the GC. Previously, if an object was used later by another interpreter, calling PyObject_GC_UnTrack() on the object crashed if the previous or the next object of the PyGC_Head structure became a dangling pointer. (cherry picked from commit 1a4d1c1c9b08e75e88aeac90901920938f649832) Co-authored-by: Victor Stinner --- .../2022-01-13-17-58-56.bpo-46070.q8IGth.rst | 5 ++++ Modules/gcmodule.c | 24 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-01-13-17-58-56.bpo-46070.q8IGth.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-01-13-17-58-56.bpo-46070.q8IGth.rst b/Misc/NEWS.d/next/Core and Builtins/2022-01-13-17-58-56.bpo-46070.q8IGth.rst new file mode 100644 index 00000000000000..4ed088f9898ebe --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-01-13-17-58-56.bpo-46070.q8IGth.rst @@ -0,0 +1,5 @@ +:c:func:`Py_EndInterpreter` now explicitly untracks all objects currently +tracked by the GC. Previously, if an object was used later by another +interpreter, calling :c:func:`PyObject_GC_UnTrack` on the object crashed if the +previous or the next object of the :c:type:`PyGC_Head` structure became a +dangling pointer. Patch by Victor Stinner. diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index e5e5aa3287b0d6..805a159d53d60f 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -2162,12 +2162,36 @@ _PyGC_DumpShutdownStats(PyInterpreterState *interp) } } + +static void +gc_fini_untrack(PyGC_Head *list) +{ + PyGC_Head *gc; + for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(list)) { + PyObject *op = FROM_GC(gc); + _PyObject_GC_UNTRACK(op); + } +} + + void _PyGC_Fini(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; Py_CLEAR(gcstate->garbage); Py_CLEAR(gcstate->callbacks); + + if (!_Py_IsMainInterpreter(interp)) { + // bpo-46070: Explicitly untrack all objects currently tracked by the + // GC. Otherwise, if an object is used later by another interpreter, + // calling PyObject_GC_UnTrack() on the object crashs if the previous + // or the next object of the PyGC_Head structure became a dangling + // pointer. + for (int i = 0; i < NUM_GENERATIONS; i++) { + PyGC_Head *gen = GEN_HEAD(gcstate, i); + gc_fini_untrack(gen); + } + } } /* for debugging */ From 47422a852de14a8ec11d058136c7c864d2cc7fc9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 13 Jan 2022 13:08:47 -0800 Subject: [PATCH 002/220] [3.10] bpo-40479: Fix typo, flag must be set for OpenSSL < 3.0.0 (GH-30584) (GH-30585) (cherry picked from commit 276c234ce0fa6732237f1b187989837324d9dea3) Co-authored-by: Christian Heimes Automerge-Triggered-By: GH:tiran --- Modules/_hashopenssl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index cb8460ab2fcf2a..2eaa5f7d85d80a 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -883,7 +883,7 @@ py_evp_fromname(PyObject *module, const char *digestname, PyObject *data_obj, goto exit; } -#if defined(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW) && OPENSSL_VERSION_NUMBER >= 0x30000000L +#if defined(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW) && OPENSSL_VERSION_NUMBER < 0x30000000L // In OpenSSL 1.1.1 the non FIPS allowed flag is context specific while // in 3.0.0 it is a different EVP_MD provider. if (!usedforsecurity) { From 86d18019e96167c5ab6f5157fa90598202849904 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 13 Jan 2022 20:32:40 -0800 Subject: [PATCH 003/220] bpo-46280: Fix tracemalloc_copy_domain() (GH-30591) Test if tracemalloc_copy_traces() failed to allocated memory in tracemalloc_copy_domain(). (cherry picked from commit 7c770d3350813a82a639fcb3babae0de2b87aaae) Co-authored-by: Victor Stinner --- Modules/_tracemalloc.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 90498fb7a7897d..ba0eb738abcbc5 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -1241,6 +1241,9 @@ tracemalloc_copy_domain(_Py_hashtable_t *domains, _Py_hashtable_t *traces = (_Py_hashtable_t *)value; _Py_hashtable_t *traces2 = tracemalloc_copy_traces(traces); + if (traces2 == NULL) { + return -1; + } if (_Py_hashtable_set(domains2, TO_PTR(domain), traces2) < 0) { _Py_hashtable_destroy(traces2); return -1; From 26039d1e0a1da897d28688895126eb8bbd16f2c9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 14 Jan 2022 09:47:51 -0800 Subject: [PATCH 004/220] bpo-23183: Document the timeit output (GH-30359) Co-authored-by: Robert Collins (cherry picked from commit 73140de97cbeb01bb6c9af1da89ecb9355921e91) Co-authored-by: Hugo van Kemenade --- Doc/library/timeit.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index d4e8b749db4808..ca21fe622323ff 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -282,6 +282,13 @@ It is possible to provide a setup statement that is executed only once at the be $ python -m timeit -s 'text = "sample string"; char = "g"' 'text.find(char)' 1000000 loops, best of 5: 0.342 usec per loop +In the output, there are three fields. The loop count, which tells you how many +times the statement body was run per timing loop repetition. The repetition +count ('best of 5') which tells you how many times the timing loop was +repeated, and finally the time the statement body took on average within the +best repetition of the timing loop. That is, the time the fastest repetition +took divided by the loop count. + :: >>> import timeit From d5c4ccfe0de375f5d3950abb4eff9d75384213b5 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 14 Jan 2022 21:11:47 +0000 Subject: [PATCH 005/220] Post 3.10.2 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 5769674a3272cc..ee537c7f34f9af 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.10.2" +#define PY_VERSION "3.10.2+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. From 93dc1654dc3c925c062e19f0ef8587aa8961ef8a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 14 Jan 2022 15:59:20 -0800 Subject: [PATCH 006/220] bpo-20281, bpo-29964: update datetime docs to refer %z and %Z to a pre-existing footnote (GH-30354) (cherry picked from commit 305588c67cdede4ef127ada90c1557bc1ef7c200) Co-authored-by: Hugo van Kemenade --- Doc/library/time.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index cfd67e87501cd4..735588a17e6e34 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -456,10 +456,10 @@ Functions | | negative time difference from UTC/GMT of the | | | | form +HHMM or -HHMM, where H represents decimal| | | | hour digits and M represents decimal minute | | - | | digits [-23:59, +23:59]. | | + | | digits [-23:59, +23:59]. [1]_ | | +-----------+------------------------------------------------+-------+ | ``%Z`` | Time zone name (no characters if no time zone | | - | | exists). | | + | | exists). Deprecated. [1]_ | | +-----------+------------------------------------------------+-------+ | ``%%`` | A literal ``'%'`` character. | | +-----------+------------------------------------------------+-------+ @@ -480,7 +480,7 @@ Functions calculations when the day of the week and the year are specified. Here is an example, a format for dates compatible with that specified in the - :rfc:`2822` Internet email standard. [#]_ :: + :rfc:`2822` Internet email standard. [1]_ :: >>> from time import gmtime, strftime >>> strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) @@ -908,10 +908,9 @@ Timezone Constants .. rubric:: Footnotes -.. [#] The use of ``%Z`` is now deprecated, but the ``%z`` escape that expands to the - preferred hour/minute offset is not supported by all ANSI C libraries. Also, a +.. [1] The use of ``%Z`` is now deprecated, but the ``%z`` escape that expands to the + preferred hour/minute offset is not supported by all ANSI C libraries. Also, a strict reading of the original 1982 :rfc:`822` standard calls for a two-digit - year (%y rather than %Y), but practice moved to 4-digit years long before the + year (``%y`` rather than ``%Y``), but practice moved to 4-digit years long before the year 2000. After that, :rfc:`822` became obsolete and the 4-digit year has been first recommended by :rfc:`1123` and then mandated by :rfc:`2822`. - From 1345b460f568afa8a6f9c0e2b23adba5015f208e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 17 Jan 2022 05:35:07 -0800 Subject: [PATCH 007/220] bpo-13886: Skip PTY non-ASCII tests if readline is loaded (GH-30631) Skip test_builtin PTY tests on non-ASCII characters if the readline module is loaded. The readline module changes input() behavior, but test_builtin is not intented to test the readline module. When the readline module is loaded, PyOS_Readline() uses the readline implementation. In some cases, the Python readline callback rlhandler() is called by readline with a string without non-ASCII characters. (cherry picked from commit ad6e640f910787e73fd00f59117fbd22cdf88c78) Co-authored-by: Victor Stinner --- Lib/test/test_builtin.py | 16 ++++++++++++++-- .../2022-01-17-13-10-04.bpo-13886.5mZH4b.rst | 3 +++ 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2022-01-17-13-10-04.bpo-13886.5mZH4b.rst diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 6dc4fa555021cc..74568032219649 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2090,12 +2090,24 @@ def test_input_tty(self): # is different and invokes GNU readline if available). self.check_input_tty("prompt", b"quux") + def skip_if_readline(self): + # bpo-13886: When the readline module is loaded, PyOS_Readline() uses + # the readline implementation. In some cases, the Python readline + # callback rlhandler() is called by readline with a string without + # non-ASCII characters. Skip tests on non-ASCII characters if the + # readline module is loaded, since test_builtin is not intented to test + # the readline module, but the builtins module. + if 'readline' in sys.modules: + self.skipTest("the readline module is loaded") + def test_input_tty_non_ascii(self): - # Check stdin/stdout encoding is used when invoking GNU readline + self.skip_if_readline() + # Check stdin/stdout encoding is used when invoking PyOS_Readline() self.check_input_tty("prompté", b"quux\xe9", "utf-8") def test_input_tty_non_ascii_unicode_errors(self): - # Check stdin/stdout error handler is used when invoking GNU readline + self.skip_if_readline() + # Check stdin/stdout error handler is used when invoking PyOS_Readline() self.check_input_tty("prompté", b"quux\xe9", "ascii") def test_input_no_stdout_fileno(self): diff --git a/Misc/NEWS.d/next/Tests/2022-01-17-13-10-04.bpo-13886.5mZH4b.rst b/Misc/NEWS.d/next/Tests/2022-01-17-13-10-04.bpo-13886.5mZH4b.rst new file mode 100644 index 00000000000000..cd19dce37d5c85 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-01-17-13-10-04.bpo-13886.5mZH4b.rst @@ -0,0 +1,3 @@ +Skip test_builtin PTY tests on non-ASCII characters if the readline module +is loaded. The readline module changes input() behavior, but test_builtin is +not intented to test the readline module. Patch by Victor Stinner. From 7a822c92782ffda8fa32a4b30a95b9de7cc1b8e6 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 17 Jan 2022 05:47:51 -0800 Subject: [PATCH 008/220] bpo-46383: Fix signature of zoneinfo module_free function (GH-30607) (GH-30610) (cherry picked from commit cfbde65df318eea243706ff876e5ef834c085e5f) Co-authored-by: Christian Heimes Co-authored-by: Christian Heimes --- .../Core and Builtins/2022-01-14-20-55-34.bpo-46383.v8MTl4.rst | 2 ++ Modules/_zoneinfo.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-01-14-20-55-34.bpo-46383.v8MTl4.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-01-14-20-55-34.bpo-46383.v8MTl4.rst b/Misc/NEWS.d/next/Core and Builtins/2022-01-14-20-55-34.bpo-46383.v8MTl4.rst new file mode 100644 index 00000000000000..8f8b12732a690e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-01-14-20-55-34.bpo-46383.v8MTl4.rst @@ -0,0 +1,2 @@ +Fix invalid signature of ``_zoneinfo``'s ``module_free`` function to resolve +a crash on wasm32-emscripten platform. diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 04fa09422b2134..0388d27ce10a48 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -2608,7 +2608,7 @@ static PyTypeObject PyZoneInfo_ZoneInfoType = { // Specify the _zoneinfo module static PyMethodDef module_methods[] = {{NULL, NULL}}; static void -module_free(void) +module_free(void *m) { Py_XDECREF(_tzpath_find_tzfile); _tzpath_find_tzfile = NULL; From 42038d00ea7b0b5455e371285102d85006fbf687 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 17 Jan 2022 23:33:00 -0800 Subject: [PATCH 009/220] bpo-46411: Remove unnecessary calls to sys.exc_info() in tests (GH-30638) (cherry picked from commit a287b31bcb065e4122400cb59167340d25480e6d) Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Lib/test/test_argparse.py | 10 ++++------ Lib/test/test_builtin.py | 4 ++-- Lib/test/test_inspect.py | 4 ++-- Lib/test/test_logging.py | 4 ++-- Lib/test/test_raise.py | 4 ++-- Lib/test/test_zipimport.py | 4 ++-- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index c96a540a8b3156..37a73e06863774 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -101,8 +101,8 @@ def stderr_to_parser_error(parse_args, *args, **kwargs): if getattr(result, key) is sys.stderr: setattr(result, key, old_stderr) return result - except SystemExit: - code = sys.exc_info()[1].code + except SystemExit as e: + code = e.code stdout = sys.stdout.getvalue() stderr = sys.stderr.getvalue() raise ArgumentParserError( @@ -1830,8 +1830,7 @@ def __call__(self, parser, namespace, value, option_string=None): raise AssertionError('value: %s' % value) assert expected_ns == namespace, ('expected %s, got %s' % (expected_ns, namespace)) - except AssertionError: - e = sys.exc_info()[1] + except AssertionError as e: raise ArgumentParserError('opt_action failed: %s' % e) setattr(namespace, 'spam', value) @@ -1856,8 +1855,7 @@ def __call__(self, parser, namespace, value, option_string=None): raise AssertionError('value: %s' % value) assert expected_ns == namespace, ('expected %s, got %s' % (expected_ns, namespace)) - except AssertionError: - e = sys.exc_info()[1] + except AssertionError as e: raise ArgumentParserError('arg_action failed: %s' % e) setattr(namespace, 'badger', value) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 74568032219649..4b0b15f0a93615 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -581,8 +581,8 @@ def __dir__(self): # dir(traceback) try: raise IndexError - except: - self.assertEqual(len(dir(sys.exc_info()[2])), 4) + except IndexError as e: + self.assertEqual(len(dir(e.__traceback__)), 4) # test that object has a __dir__() self.assertEqual(sorted([].__dir__()), dir([])) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 93ff2f85df193d..545dab5c6348f7 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -132,8 +132,8 @@ def test_excluding_predicates(self): self.istest(inspect.iscode, 'mod.spam.__code__') try: 1/0 - except: - tb = sys.exc_info()[2] + except Exception as e: + tb = e.__traceback__ self.istest(inspect.isframe, 'tb.tb_frame') self.istest(inspect.istraceback, 'tb') if hasattr(types, 'GetSetDescriptorType'): diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 03d0319306a480..8212cf7a9a964d 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -5508,8 +5508,8 @@ def test_compute_rollover(self, when=when, exp=exp): print('currentSecond: %s' % currentSecond, file=sys.stderr) print('r: %s' % r, file=sys.stderr) print('result: %s' % result, file=sys.stderr) - except Exception: - print('exception in diagnostic code: %s' % sys.exc_info()[1], file=sys.stderr) + except Exception as e: + print('exception in diagnostic code: %s' % e, file=sys.stderr) self.assertEqual(exp, actual) rh.close() setattr(TimedRotatingFileHandlerTest, "test_compute_rollover_%s" % when, test_compute_rollover) diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index 8dc62a933e872e..8225504c4756db 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -12,8 +12,8 @@ def get_tb(): try: raise OSError() - except: - return sys.exc_info()[2] + except OSError as e: + return e.__traceback__ class Context: diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index 19d3a880f4cd7b..b291d5301690dc 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -709,8 +709,8 @@ def testDoctestSuite(self): def doTraceback(self, module): try: module.do_raise() - except: - tb = sys.exc_info()[2].tb_next + except Exception as e: + tb = e.__traceback__.tb_next f,lno,n,line = extract_tb(tb, 1)[0] self.assertEqual(line, raise_src.strip()) From 9238a52cbc39c17ca6c7a8cbda32808dd5522a59 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 18 Jan 2022 12:16:54 -0800 Subject: [PATCH 010/220] bpo-20823: Clarify copyreg.pickle() documentation (GH-30230) (cherry picked from commit 65940fa5c12a4b4a0650c7845044ffd63b94e227) Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> --- Doc/library/copyreg.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 43920210095951..dc35965be3e40d 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -33,8 +33,8 @@ Such constructors may be factory functions or class instances. The optional *constructor* parameter, if provided, is a callable object which can be used to reconstruct the object when called with the tuple of arguments - returned by *function* at pickling time. :exc:`TypeError` will be raised if - *object* is a class or *constructor* is not callable. + returned by *function* at pickling time. A :exc:`TypeError` is raised if the + *constructor* is not callable. See the :mod:`pickle` module for more details on the interface expected of *function* and *constructor*. Note that the From 4449a1694a0fd2c63fcef5eb7d0ad1d7dfbb6077 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 18 Jan 2022 13:51:30 -0800 Subject: [PATCH 011/220] bpo-45554: Document multiprocessing.Process.exitcode values (GH-30142) This addresses [bpo-45554]() by expanding the `exitcode` documentation to also describe what `exitcode` will be in cases of normal termination, `sys.exit()` called, and on uncaught exceptions. Automerge-Triggered-By: GH:pitrou (cherry picked from commit 3852269b91fcc8ee668cd876b3669eba6da5b1ac) Co-authored-by: John Marshall --- Doc/library/multiprocessing.rst | 11 +++++++++-- Misc/ACKS | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index e81dd7e648f7fe..e0954b285b37b8 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -569,8 +569,15 @@ The :mod:`multiprocessing` package mostly replicates the API of the .. attribute:: exitcode The child's exit code. This will be ``None`` if the process has not yet - terminated. A negative value *-N* indicates that the child was terminated - by signal *N*. + terminated. + + If the child's :meth:`run` method returned normally, the exit code + will be 0. If it terminated via :func:`sys.exit` with an integer + argument *N*, the exit code will be *N*. + + If the child terminated due to an exception not caught within + :meth:`run`, the exit code will be 1. If it was terminated by + signal *N*, the exit code will be the negative value *-N*. .. attribute:: authkey diff --git a/Misc/ACKS b/Misc/ACKS index 94b0ed0b241cd6..9292bdc8dc73b7 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1115,6 +1115,7 @@ Vincent Marchetti David Marek Doug Marien Sven Marnach +John Marshall Alex Martelli Dennis Mårtensson Anthony Martin From 01e6cbefd3d0f60c942ed711131f5d638dde1227 Mon Sep 17 00:00:00 2001 From: Erlend Egeberg Aasland Date: Tue, 18 Jan 2022 22:57:33 +0100 Subject: [PATCH 012/220] [3.10] bpo-46402: Promote SQLite URI tricks in sqlite3 docs (GH-30660) (GH-30671) * bpo-46402: Promote SQLite URI tricks in `sqlite3` docs (GH-30660) Provide some examples of URI parameters in sqlite connect(). Co-authored-by: Ned Batchelder (cherry picked from commit bdf2ab1887a2edfb089a3c2a1590cf1e84ea0048) Co-authored-by: Erlend Egeberg Aasland * Update suspicious rules --- Doc/library/sqlite3.rst | 30 ++++++++++++++++++++++-------- Doc/tools/susp-ignored.csv | 5 ++++- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 1c3bde3b914d0c..492dadb2746acc 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -255,14 +255,28 @@ Module functions and constants for the connection, you can set the *cached_statements* parameter. The currently implemented default is to cache 100 statements. - If *uri* is true, *database* is interpreted as a URI. This allows you - to specify options. For example, to open a database in read-only mode - you can use:: - - db = sqlite3.connect('file:path/to/database?mode=ro', uri=True) - - More information about this feature, including a list of recognized options, can - be found in the `SQLite URI documentation `_. + If *uri* is :const:`True`, *database* is interpreted as a + :abbr:`URI (Uniform Resource Identifier)` with a file path and an optional + query string. The scheme part *must* be ``"file:"``. The path can be a + relative or absolute file path. The query string allows us to pass + parameters to SQLite. Some useful URI tricks include:: + + # Open a database in read-only mode. + con = sqlite3.connect("file:template.db?mode=ro", uri=True) + + # Don't implicitly create a new database file if it does not already exist. + # Will raise sqlite3.OperationalError if unable to open a database file. + con = sqlite3.connect("file:nosuchdb.db?mode=rw", uri=True) + + # Create a shared named in-memory database. + con1 = sqlite3.connect("file:mem1?mode=memory&cache=shared", uri=True) + con2 = sqlite3.connect("file:mem1?mode=memory&cache=shared", uri=True) + con1.executescript("create table t(t); insert into t values(28);") + rows = con2.execute("select * from t").fetchall() + + More information about this feature, including a list of recognized + parameters, can be found in the + `SQLite URI documentation `_. .. audit-event:: sqlite3.connect database sqlite3.connect .. audit-event:: sqlite3.connect/handle connection_handle sqlite3.connect diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index 2453faa8ccd746..9f4a44f1de64e1 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -212,7 +212,10 @@ library/socket,,:can,"return (can_id, can_dlc, data[:can_dlc])" library/socket,,:len,fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) library/sqlite3,,:year,"cur.execute(""select * from lang where first_appeared=:year"", {""year"": 1972})" library/sqlite3,,:memory, -library/sqlite3,,:path,"db = sqlite3.connect('file:path/to/database?mode=ro', uri=True)" +library/sqlite3,,:template,"con = sqlite3.connect(""file:template.db?mode=ro"", uri=True)" +library/sqlite3,,:nosuchdb,"con = sqlite3.connect(""file:nosuchdb.db?mode=rw"", uri=True)" +library/sqlite3,,:mem1,"con1 = sqlite3.connect(""file:mem1?mode=memory&cache=shared"", uri=True)" +library/sqlite3,,:mem1,"con2 = sqlite3.connect(""file:mem1?mode=memory&cache=shared"", uri=True)" library/ssl,,:My,"Organizational Unit Name (eg, section) []:My Group" library/ssl,,:My,"Organization Name (eg, company) [Internet Widgits Pty Ltd]:My Organization, Inc." library/ssl,,:myserver,"Common Name (eg, YOUR name) []:myserver.mygroup.myorganization.com" From 0861a50bd434d1f3e12fe7122e37356f1fce93dc Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 19 Jan 2022 04:34:17 -0800 Subject: [PATCH 013/220] bpo-22039: [doc] clarify that there are no plans to disable deleting an attribute via PyObject_SetAttr (GH-30639) (GH-30684) (cherry picked from commit 3bf6315c4cabf72d64e65e6f85bf72c65137255a) Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Doc/c-api/object.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 17e37077994968..41a3affcf9842a 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -81,8 +81,9 @@ Object Protocol return ``0`` on success. This is the equivalent of the Python statement ``o.attr_name = v``. - If *v* is ``NULL``, the attribute is deleted, however this feature is - deprecated in favour of using :c:func:`PyObject_DelAttr`. + If *v* is ``NULL``, the attribute is deleted. This behaviour is deprecated + in favour of using :c:func:`PyObject_DelAttr`, but there are currently no + plans to remove it. .. c:function:: int PyObject_SetAttrString(PyObject *o, const char *attr_name, PyObject *v) From 24d0b331e81b4e4af8dd4c1b66ea7159c1fdabc5 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 19 Jan 2022 07:24:14 -0800 Subject: [PATCH 014/220] [3.10] bpo-45680: Clarify documentation on ``GenericAlias`` objects (GH-29335) (GH-30688) The documentation on ``GenericAlias`` objects implies at multiple points that only container classes can define ``__class_getitem__``. This is misleading. This PR proposes a rewrite of the documentation to clarify that non-container classes can define ``__class_getitem__``, and to clarify what it means when a non-container class is parameterized. See also: initial discussion of issues with this piece of documentation in GH-29308, and previous BPO issue [42280](). Also improved references in glossary and typing docs. Fixed some links. Co-authored-by: Erlend Egeberg Aasland Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> (cherry picked from commit 0eae9a2a2db6cc5a72535f61bb988cc417011640) Co-authored-by: Alex Waygood Automerge-Triggered-By: GH:Fidget-Spinner --- Doc/library/stdtypes.rst | 117 ++++++++++++++++++++++++++++----------- 1 file changed, 84 insertions(+), 33 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 8fa252b04d7068..08e7c0db8cc973 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4796,33 +4796,54 @@ Generic Alias Type object: GenericAlias pair: Generic; Alias -``GenericAlias`` objects are created by subscripting a class (usually a -container), such as ``list[int]``. They are intended primarily for +``GenericAlias`` objects are generally created by +:ref:`subscripting ` a class. They are most often used with +:ref:`container classes `, such as :class:`list` or +:class:`dict`. For example, ``list[int]`` is a ``GenericAlias`` object created +by subscripting the ``list`` class with the argument :class:`int`. +``GenericAlias`` objects are intended primarily for use with :term:`type annotations `. -Usually, the :ref:`subscription ` of container objects calls the -method :meth:`__getitem__` of the object. However, the subscription of some -containers' classes may call the classmethod :meth:`__class_getitem__` of the -class instead. The classmethod :meth:`__class_getitem__` should return a -``GenericAlias`` object. - .. note:: - If the :meth:`__getitem__` of the class' metaclass is present, it will take - precedence over the :meth:`__class_getitem__` defined in the class (see - :pep:`560` for more details). -The ``GenericAlias`` object acts as a proxy for :term:`generic types -`, implementing *parameterized generics* - a specific instance -of a generic which provides the types for container elements. + It is generally only possible to subscript a class if the class implements + the special method :meth:`~object.__class_getitem__`. + +A ``GenericAlias`` object acts as a proxy for a :term:`generic type`, +implementing *parameterized generics*. + +For a container class, the +argument(s) supplied to a :ref:`subscription ` of the class may +indicate the type(s) of the elements an object contains. For example, +``set[bytes]`` can be used in type annotations to signify a :class:`set` in +which all the elements are of type :class:`bytes`. + +For a class which defines :meth:`~object.__class_getitem__` but is not a +container, the argument(s) supplied to a subscription of the class will often +indicate the return type(s) of one or more methods defined on an object. For +example, :mod:`regular expressions ` can be used on both the :class:`str` data +type and the :class:`bytes` data type: + +* If ``x = re.search('foo', 'foo')``, ``x`` will be a + :ref:`re.Match ` object where the return values of + ``x.group(0)`` and ``x[0]`` will both be of type :class:`str`. We can + represent this kind of object in type annotations with the ``GenericAlias`` + ``re.Match[str]``. -The user-exposed type for the ``GenericAlias`` object can be accessed from -:class:`types.GenericAlias` and used for :func:`isinstance` checks. It can -also be used to create ``GenericAlias`` objects directly. +* If ``y = re.search(b'bar', b'bar')``, (note the ``b`` for :class:`bytes`), + ``y`` will also be an instance of ``re.Match``, but the return + values of ``y.group(0)`` and ``y[0]`` will both be of type + :class:`bytes`. In type annotations, we would represent this + variety of :ref:`re.Match ` objects with ``re.Match[bytes]``. + +``GenericAlias`` objects are instances of the class +:class:`types.GenericAlias`, which can also be used to create ``GenericAlias`` +objects directly. .. describe:: T[X, Y, ...] - Creates a ``GenericAlias`` representing a type ``T`` containing elements - of types *X*, *Y*, and more depending on the ``T`` used. + Creates a ``GenericAlias`` representing a type ``T`` parameterized by types + *X*, *Y*, and more depending on the ``T`` used. For example, a function expecting a :class:`list` containing :class:`float` elements:: @@ -4847,7 +4868,7 @@ The builtin functions :func:`isinstance` and :func:`issubclass` do not accept The Python runtime does not enforce :term:`type annotations `. This extends to generic types and their type parameters. When creating -an object from a ``GenericAlias``, container elements are not checked +a container object from a ``GenericAlias``, the elements in the container are not checked against their type. For example, the following code is discouraged, but will run without errors:: @@ -4874,8 +4895,8 @@ Calling :func:`repr` or :func:`str` on a generic shows the parameterized type:: >>> str(list[int]) 'list[int]' -The :meth:`__getitem__` method of generics will raise an exception to disallow -mistakes like ``dict[str][str]``:: +The :meth:`~object.__getitem__` method of generic containers will raise an +exception to disallow mistakes like ``dict[str][str]``:: >>> dict[str][str] Traceback (most recent call last): @@ -4884,7 +4905,7 @@ mistakes like ``dict[str][str]``:: However, such expressions are valid when :ref:`type variables ` are used. The index must have as many elements as there are type variable items -in the ``GenericAlias`` object's :attr:`__args__ `. :: +in the ``GenericAlias`` object's :attr:`~genericalias.__args__`. :: >>> from typing import TypeVar >>> Y = TypeVar('Y') @@ -4892,10 +4913,11 @@ in the ``GenericAlias`` object's :attr:`__args__ `. :: dict[str, int] -Standard Generic Collections -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Standard Generic Classes +^^^^^^^^^^^^^^^^^^^^^^^^ -These standard library collections support parameterized generics. +The following standard library classes support parameterized generics. This +list is non-exhaustive. * :class:`tuple` * :class:`list` @@ -4933,12 +4955,33 @@ These standard library collections support parameterized generics. * :class:`collections.abc.ValuesView` * :class:`contextlib.AbstractContextManager` * :class:`contextlib.AbstractAsyncContextManager` +* :class:`dataclasses.Field` +* :class:`functools.cached_property` +* :class:`functools.partialmethod` +* :class:`os.PathLike` +* :class:`pathlib.Path` +* :class:`pathlib.PurePath` +* :class:`pathlib.PurePosixPath` +* :class:`pathlib.PureWindowsPath` +* :class:`queue.LifoQueue` +* :class:`queue.Queue` +* :class:`queue.PriorityQueue` +* :class:`queue.SimpleQueue` * :ref:`re.Pattern ` * :ref:`re.Match ` +* :class:`shelve.BsdDbShelf` +* :class:`shelve.DbfilenameShelf` +* :class:`shelve.Shelf` +* :class:`types.MappingProxyType` +* :class:`weakref.WeakKeyDictionary` +* :class:`weakref.WeakMethod` +* :class:`weakref.WeakSet` +* :class:`weakref.WeakValueDictionary` + -Special Attributes of Generic Alias -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Special Attributes of ``GenericAlias`` objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ All parameterized generics implement special read-only attributes. @@ -4953,8 +4996,8 @@ All parameterized generics implement special read-only attributes. .. attribute:: genericalias.__args__ This attribute is a :class:`tuple` (possibly of length 1) of generic - types passed to the original :meth:`__class_getitem__` - of the generic container:: + types passed to the original :meth:`~object.__class_getitem__` of the + generic class:: >>> dict[str, list[int]].__args__ (, list[int]) @@ -4979,9 +5022,17 @@ All parameterized generics implement special read-only attributes. .. seealso:: - * :pep:`585` -- "Type Hinting Generics In Standard Collections" - * :meth:`__class_getitem__` -- Used to implement parameterized generics. - * :ref:`generics` -- Generics in the :mod:`typing` module. + :pep:`484` - Type Hints + Introducing Python's framework for type annotations. + + :pep:`585` - Type Hinting Generics In Standard Collections + Introducing the ability to natively parameterize standard-library + classes, provided they implement the special class method + :meth:`~object.__class_getitem__`. + + :ref:`Generics`, :ref:`user-defined generics ` and :class:`typing.Generic` + Documentation on how to implement generic classes that can be + parameterized at runtime and understood by static type-checkers. .. versionadded:: 3.9 From baf26d07a634b0ea3ff052716bdeaee985b3a3a9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 19 Jan 2022 07:54:07 -0800 Subject: [PATCH 015/220] bpo-46424: [typing] cover `Annotation[arg]` invalid usage in tests (GH-30663) (cherry picked from commit 32398294fb3fcf4ee74da54722fd0221c4e6cb74) Co-authored-by: Nikita Sobolev --- Lib/test/test_typing.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f943aed73614cd..b886c38827f1ff 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4512,6 +4512,10 @@ def test_cannot_check_subclass(self): with self.assertRaises(TypeError): issubclass(int, Annotated[int, "positive"]) + def test_too_few_type_args(self): + with self.assertRaisesRegex(TypeError, 'at least two arguments'): + Annotated[int] + def test_pickle(self): samples = [typing.Any, typing.Union[int, str], typing.Optional[str], Tuple[int, ...], From 39374c44d98b470213256ceead0e2b4e44b14b92 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 19 Jan 2022 08:11:12 -0800 Subject: [PATCH 016/220] [3.10] bpo-46413: properly test `__{r}or__` code paths in `_SpecialGenericAlias` (GH-30640) (GH-30694) Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> (cherry picked from commit 0a49148e87cca11e3820cbff2abfd316986a68c6) Co-authored-by: Nikita Sobolev Automerge-Triggered-By: GH:Fidget-Spinner --- Lib/test/test_typing.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index b886c38827f1ff..ee432b65cf5df5 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -515,6 +515,10 @@ def test_ellipsis_in_generic(self): # Shouldn't crash; see https://github.com/python/typing/issues/259 typing.List[Callable[..., str]] + def test_or_and_ror(self): + Callable = self.Callable + self.assertEqual(Callable | Tuple, Union[Callable, Tuple]) + self.assertEqual(Tuple | Callable, Union[Tuple, Callable]) def test_basic(self): Callable = self.Callable @@ -3834,6 +3838,10 @@ class B: ... A.register(B) self.assertIsSubclass(B, typing.Mapping) + def test_or_and_ror(self): + self.assertEqual(typing.Sized | typing.Awaitable, Union[typing.Sized, typing.Awaitable]) + self.assertEqual(typing.Coroutine | typing.Hashable, Union[typing.Coroutine, typing.Hashable]) + class OtherABCTests(BaseTestCase): From c4fe0aa670480d887f1f736d1a4251234914b58c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 19 Jan 2022 10:02:07 -0800 Subject: [PATCH 017/220] Update documentation in datetime module strftime-and-strptime-behavior fix typo in '%W' format code description (GH-30232) A small change to the documentation of datetime module , in the format codes section of stftime and strptime. Changed the description of format code '%W' from 'as a decimal number' to 'a zero padded decimal number' so it's in line with the example having leading zeros. Similar to the format code '%U' above. Automerge-Triggered-By: GH:pganssle (cherry picked from commit d45cd2d20770f72a000ba6dfa9ac88dd49423c27) Co-authored-by: Evan --- Doc/library/datetime.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 217cdf222b89b7..f447b7bc9491e4 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -2375,7 +2375,7 @@ requires, and these work on all platforms with a standard C implementation. +-----------+--------------------------------+------------------------+-------+ | ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7), | | | (Sunday as the first day of | | \(9) | -| | the week) as a zero padded | | | +| | the week) as a zero-padded | | | | | decimal number. All days in a | | | | | new year preceding the first | | | | | Sunday are considered to be in | | | @@ -2383,10 +2383,10 @@ requires, and these work on all platforms with a standard C implementation. +-----------+--------------------------------+------------------------+-------+ | ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7), | | | (Monday as the first day of | | \(9) | -| | the week) as a decimal number. | | | -| | All days in a new year | | | -| | preceding the first Monday | | | -| | are considered to be in | | | +| | the week) as a zero-padded | | | +| | decimal number. All days in a | | | +| | new year preceding the first | | | +| | Monday are considered to be in | | | | | week 0. | | | +-----------+--------------------------------+------------------------+-------+ | ``%c`` | Locale's appropriate date and || Tue Aug 16 21:30:00 | \(1) | From d2b7e08d86874be7d4375a4994617ba8f068a65e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 19 Jan 2022 13:57:09 -0800 Subject: [PATCH 018/220] docs: correct outdated MappingProxyType docstrings (GH-30281) The docstrings for MappingProxyType's keys(), values(), and items() methods were never updated to reflect the changes that Python 3 brought to these APIs, namely returning views rather than lists. (cherry picked from commit 2d10fa9bc4cf83c5e5dd73decc9a138d6d247374) Co-authored-by: Joshua Bronson --- Objects/descrobject.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 97669bef368da8..26726cc0973df7 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1135,11 +1135,11 @@ static PyMethodDef mappingproxy_methods[] = { PyDoc_STR("D.get(k[,d]) -> D[k] if k in D, else d." " d defaults to None.")}, {"keys", (PyCFunction)mappingproxy_keys, METH_NOARGS, - PyDoc_STR("D.keys() -> list of D's keys")}, + PyDoc_STR("D.keys() -> a set-like object providing a view on D's keys")}, {"values", (PyCFunction)mappingproxy_values, METH_NOARGS, - PyDoc_STR("D.values() -> list of D's values")}, + PyDoc_STR("D.values() -> an object providing a view on D's values")}, {"items", (PyCFunction)mappingproxy_items, METH_NOARGS, - PyDoc_STR("D.items() -> list of D's (key, value) pairs, as 2-tuples")}, + PyDoc_STR("D.items() -> a set-like object providing a view on D's items")}, {"copy", (PyCFunction)mappingproxy_copy, METH_NOARGS, PyDoc_STR("D.copy() -> a shallow copy of D")}, {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, From ee077500888ca5c1360bbd224b3af4a0fbbf6e02 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 19 Jan 2022 13:58:13 -0800 Subject: [PATCH 019/220] doc: Clarify os.urandom return type (GH-30282) Other descriptions in the same file also use 'bytestring' to refer to bytes objects (cherry picked from commit 4b99803b861e58eb476a7a30e2e8aacdec5df104) Co-authored-by: Florian Bruhin --- Doc/library/os.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 629a32f1b63e7d..5b2c2e0d0f2d03 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -4869,7 +4869,7 @@ Random numbers .. function:: urandom(size) - Return a string of *size* random bytes suitable for cryptographic use. + Return a bytestring of *size* random bytes suitable for cryptographic use. This function returns random bytes from an OS-specific randomness source. The returned data should be unpredictable enough for cryptographic applications, From 07b12fdf5545a20e0fb7be9d6ad35344337e00ae Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 19 Jan 2022 14:30:07 -0800 Subject: [PATCH 020/220] bpo-46437: remove useless `hasattr` from `test_typing` (GH-30704) (cherry picked from commit 263c0dd16017613c5ea2fbfc270be4de2b41b5ad) Co-authored-by: Nikita Sobolev --- Lib/test/test_typing.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index ee432b65cf5df5..1d16e78d422cd0 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3439,11 +3439,10 @@ def test_container(self): self.assertNotIsInstance(42, typing.Container) def test_collection(self): - if hasattr(typing, 'Collection'): - self.assertIsInstance(tuple(), typing.Collection) - self.assertIsInstance(frozenset(), typing.Collection) - self.assertIsSubclass(dict, typing.Collection) - self.assertNotIsInstance(42, typing.Collection) + self.assertIsInstance(tuple(), typing.Collection) + self.assertIsInstance(frozenset(), typing.Collection) + self.assertIsSubclass(dict, typing.Collection) + self.assertNotIsInstance(42, typing.Collection) def test_abstractset(self): self.assertIsInstance(set(), typing.AbstractSet) @@ -5033,8 +5032,9 @@ def test_all(self): self.assertIn('ValuesView', a) self.assertIn('cast', a) self.assertIn('overload', a) - if hasattr(contextlib, 'AbstractContextManager'): - self.assertIn('ContextManager', a) + # Context managers. + self.assertIn('ContextManager', a) + self.assertIn('AsyncContextManager', a) # Check that io and re are not exported. self.assertNotIn('io', a) self.assertNotIn('re', a) @@ -5048,8 +5048,6 @@ def test_all(self): self.assertIn('SupportsComplex', a) def test_all_exported_names(self): - import typing - actual_all = set(typing.__all__) computed_all = { k for k, v in vars(typing).items() From a6a088548063226233c08b8d35dde130746fdd10 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 20 Jan 2022 04:44:21 +0300 Subject: [PATCH 021/220] [3.10] bpo-46425: Fix direct invocation of multiple test modules (GH-30666) (GH-30699) --- Lib/test/test_compileall.py | 6 ++---- Lib/test/test_distutils.py | 2 +- Lib/test/test_dtrace.py | 2 +- Lib/test/test_zipfile64.py | 3 ++- Lib/unittest/test/test_program.py | 2 +- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 9e15ecf3aae299..33f0c939325f53 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -3,7 +3,6 @@ import filecmp import importlib.util import io -import itertools import os import pathlib import py_compile @@ -29,9 +28,8 @@ from test import support from test.support import os_helper from test.support import script_helper - -from .test_py_compile import without_source_date_epoch -from .test_py_compile import SourceDateEpochTestMeta +from test.test_py_compile import without_source_date_epoch +from test.test_py_compile import SourceDateEpochTestMeta def get_pyc(script, opt): diff --git a/Lib/test/test_distutils.py b/Lib/test/test_distutils.py index 4b40af0213234e..d82d2b6423433e 100644 --- a/Lib/test/test_distutils.py +++ b/Lib/test/test_distutils.py @@ -5,7 +5,7 @@ be run. """ -import warnings +import unittest from test import support from test.support import warnings_helper diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py index 3957077f5d6123..8a436ad123b80f 100644 --- a/Lib/test/test_dtrace.py +++ b/Lib/test/test_dtrace.py @@ -170,4 +170,4 @@ class SystemTapOptimizedTests(TraceTests, unittest.TestCase): if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_zipfile64.py b/Lib/test/test_zipfile64.py index 810fdedef39ddd..0947013afbc6ed 100644 --- a/Lib/test/test_zipfile64.py +++ b/Lib/test/test_zipfile64.py @@ -18,8 +18,9 @@ from tempfile import TemporaryFile from test.support import os_helper -from test.support import TESTFN, requires_zlib +from test.support import requires_zlib +TESTFN = os_helper.TESTFN TESTFN2 = TESTFN + "2" # How much time in seconds can pass before we print a 'Still working' message. diff --git a/Lib/unittest/test/test_program.py b/Lib/unittest/test/test_program.py index 4746d71e0b6039..b7fbbc1e7baddd 100644 --- a/Lib/unittest/test/test_program.py +++ b/Lib/unittest/test/test_program.py @@ -6,7 +6,7 @@ from test import support import unittest import unittest.test -from .test_result import BufferedWriter +from unittest.test.test_result import BufferedWriter class Test_TestProgram(unittest.TestCase): From 1fb1f5d8bd084c20f0a5fde547b563c08d103f09 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 20 Jan 2022 05:05:10 -0800 Subject: [PATCH 022/220] [3.10] bpo-46339: Fix crash in the parser when computing error text for multi-line f-strings (GH-30529) (GH-30542) * bpo-46339: Fix crash in the parser when computing error text for multi-line f-strings (GH-30529) Automerge-Triggered-By: GH:pablogsal (cherry picked from commit cedec19be81e6bd153678bfb28c8e217af8bda58) Co-authored-by: Pablo Galindo Salgado * Fix interactive mode Co-authored-by: Pablo Galindo Salgado --- Lib/test/test_exceptions.py | 6 + .../2022-01-11-11-50-19.bpo-46339.OVumDZ.rst | 3 + Parser/pegen.c | 12 +- Parser/pegen_errors.c | 425 ------------------ 4 files changed, 19 insertions(+), 427 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-01-11-11-50-19.bpo-46339.OVumDZ.rst delete mode 100644 Parser/pegen_errors.c diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 86b5dccaaed985..b3d1c35274c719 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -278,6 +278,12 @@ def baz(): } \"\"\" }'''""", 5, 17) + check('''f""" + + + { + 6 + 0="""''', 5, 13) # Errors thrown by symtable.c check('x = [(yield i) for i in range(3)]', 1, 7) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-01-11-11-50-19.bpo-46339.OVumDZ.rst b/Misc/NEWS.d/next/Core and Builtins/2022-01-11-11-50-19.bpo-46339.OVumDZ.rst new file mode 100644 index 00000000000000..cd04f060826b28 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-01-11-11-50-19.bpo-46339.OVumDZ.rst @@ -0,0 +1,3 @@ +Fix a crash in the parser when retrieving the error text for multi-line +f-strings expressions that do not start in the first line of the string. +Patch by Pablo Galindo diff --git a/Parser/pegen.c b/Parser/pegen.c index e507415f6d14ce..f9812c0ea8f02b 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -436,9 +436,17 @@ get_error_line(Parser *p, Py_ssize_t lineno) char *cur_line = p->tok->fp_interactive ? p->tok->interactive_src_start : p->tok->str; assert(cur_line != NULL); + const char* buf_end = p->tok->fp_interactive ? p->tok->interactive_src_end : p->tok->inp; - for (int i = 0; i < lineno - 1; i++) { - cur_line = strchr(cur_line, '\n') + 1; + Py_ssize_t relative_lineno = p->starting_lineno ? lineno - p->starting_lineno + 1 : lineno; + + for (int i = 0; i < relative_lineno - 1; i++) { + char *new_line = strchr(cur_line, '\n') + 1; + assert(new_line != NULL && new_line <= buf_end); + if (new_line == NULL || new_line > buf_end) { + break; + } + cur_line = new_line; } char *next_newline; diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c deleted file mode 100644 index 93057d151db386..00000000000000 --- a/Parser/pegen_errors.c +++ /dev/null @@ -1,425 +0,0 @@ -#include -#include - -#include "tokenizer.h" -#include "pegen.h" - -// TOKENIZER ERRORS - -void -_PyPegen_raise_tokenizer_init_error(PyObject *filename) -{ - if (!(PyErr_ExceptionMatches(PyExc_LookupError) - || PyErr_ExceptionMatches(PyExc_SyntaxError) - || PyErr_ExceptionMatches(PyExc_ValueError) - || PyErr_ExceptionMatches(PyExc_UnicodeDecodeError))) { - return; - } - PyObject *errstr = NULL; - PyObject *tuple = NULL; - PyObject *type; - PyObject *value; - PyObject *tback; - PyErr_Fetch(&type, &value, &tback); - errstr = PyObject_Str(value); - if (!errstr) { - goto error; - } - - PyObject *tmp = Py_BuildValue("(OiiO)", filename, 0, -1, Py_None); - if (!tmp) { - goto error; - } - - tuple = PyTuple_Pack(2, errstr, tmp); - Py_DECREF(tmp); - if (!value) { - goto error; - } - PyErr_SetObject(PyExc_SyntaxError, tuple); - -error: - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(tback); - Py_XDECREF(errstr); - Py_XDECREF(tuple); -} - -static inline void -raise_unclosed_parentheses_error(Parser *p) { - int error_lineno = p->tok->parenlinenostack[p->tok->level-1]; - int error_col = p->tok->parencolstack[p->tok->level-1]; - RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, - error_lineno, error_col, error_lineno, -1, - "'%c' was never closed", - p->tok->parenstack[p->tok->level-1]); -} - -int -_Pypegen_tokenizer_error(Parser *p) -{ - if (PyErr_Occurred()) { - return -1; - } - - const char *msg = NULL; - PyObject* errtype = PyExc_SyntaxError; - Py_ssize_t col_offset = -1; - switch (p->tok->done) { - case E_TOKEN: - msg = "invalid token"; - break; - case E_EOF: - if (p->tok->level) { - raise_unclosed_parentheses_error(p); - } else { - RAISE_SYNTAX_ERROR("unexpected EOF while parsing"); - } - return -1; - case E_DEDENT: - RAISE_INDENTATION_ERROR("unindent does not match any outer indentation level"); - return -1; - case E_INTR: - if (!PyErr_Occurred()) { - PyErr_SetNone(PyExc_KeyboardInterrupt); - } - return -1; - case E_NOMEM: - PyErr_NoMemory(); - return -1; - case E_TABSPACE: - errtype = PyExc_TabError; - msg = "inconsistent use of tabs and spaces in indentation"; - break; - case E_TOODEEP: - errtype = PyExc_IndentationError; - msg = "too many levels of indentation"; - break; - case E_LINECONT: { - col_offset = p->tok->cur - p->tok->buf - 1; - msg = "unexpected character after line continuation character"; - break; - } - default: - msg = "unknown parsing error"; - } - - RAISE_ERROR_KNOWN_LOCATION(p, errtype, p->tok->lineno, - col_offset >= 0 ? col_offset : 0, - p->tok->lineno, -1, msg); - return -1; -} - -int -_Pypegen_raise_decode_error(Parser *p) -{ - assert(PyErr_Occurred()); - const char *errtype = NULL; - if (PyErr_ExceptionMatches(PyExc_UnicodeError)) { - errtype = "unicode error"; - } - else if (PyErr_ExceptionMatches(PyExc_ValueError)) { - errtype = "value error"; - } - if (errtype) { - PyObject *type; - PyObject *value; - PyObject *tback; - PyObject *errstr; - PyErr_Fetch(&type, &value, &tback); - errstr = PyObject_Str(value); - if (errstr) { - RAISE_SYNTAX_ERROR("(%s) %U", errtype, errstr); - Py_DECREF(errstr); - } - else { - PyErr_Clear(); - RAISE_SYNTAX_ERROR("(%s) unknown error", errtype); - } - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(tback); - } - - return -1; -} - -static int -_PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) { - // Tokenize the whole input to see if there are any tokenization - // errors such as mistmatching parentheses. These will get priority - // over generic syntax errors only if the line number of the error is - // before the one that we had for the generic error. - - // We don't want to tokenize to the end for interactive input - if (p->tok->prompt != NULL) { - return 0; - } - - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); - - Token *current_token = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1]; - Py_ssize_t current_err_line = current_token->lineno; - - int ret = 0; - - for (;;) { - const char *start; - const char *end; - switch (_PyTokenizer_Get(p->tok, &start, &end)) { - case ERRORTOKEN: - if (p->tok->level != 0) { - int error_lineno = p->tok->parenlinenostack[p->tok->level-1]; - if (current_err_line > error_lineno) { - raise_unclosed_parentheses_error(p); - ret = -1; - goto exit; - } - } - break; - case ENDMARKER: - break; - default: - continue; - } - break; - } - - -exit: - if (PyErr_Occurred()) { - Py_XDECREF(value); - Py_XDECREF(type); - Py_XDECREF(traceback); - } else { - PyErr_Restore(type, value, traceback); - } - return ret; -} - -// PARSER ERRORS - -void * -_PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...) -{ - if (p->fill == 0) { - va_list va; - va_start(va, errmsg); - _PyPegen_raise_error_known_location(p, errtype, 0, 0, 0, -1, errmsg, va); - va_end(va); - return NULL; - } - - Token *t = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1]; - Py_ssize_t col_offset; - Py_ssize_t end_col_offset = -1; - if (t->col_offset == -1) { - if (p->tok->cur == p->tok->buf) { - col_offset = 0; - } else { - const char* start = p->tok->buf ? p->tok->line_start : p->tok->buf; - col_offset = Py_SAFE_DOWNCAST(p->tok->cur - start, intptr_t, int); - } - } else { - col_offset = t->col_offset + 1; - } - - if (t->end_col_offset != -1) { - end_col_offset = t->end_col_offset + 1; - } - - va_list va; - va_start(va, errmsg); - _PyPegen_raise_error_known_location(p, errtype, t->lineno, col_offset, t->end_lineno, end_col_offset, errmsg, va); - va_end(va); - - return NULL; -} - -static PyObject * -get_error_line_from_tokenizer_buffers(Parser *p, Py_ssize_t lineno) -{ - /* If the file descriptor is interactive, the source lines of the current - * (multi-line) statement are stored in p->tok->interactive_src_start. - * If not, we're parsing from a string, which means that the whole source - * is stored in p->tok->str. */ - assert((p->tok->fp == NULL && p->tok->str != NULL) || p->tok->fp == stdin); - - char *cur_line = p->tok->fp_interactive ? p->tok->interactive_src_start : p->tok->str; - assert(cur_line != NULL); - - for (int i = 0; i < lineno - 1; i++) { - cur_line = strchr(cur_line, '\n') + 1; - } - - char *next_newline; - if ((next_newline = strchr(cur_line, '\n')) == NULL) { // This is the last line - next_newline = cur_line + strlen(cur_line); - } - return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace"); -} - -void * -_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, - Py_ssize_t lineno, Py_ssize_t col_offset, - Py_ssize_t end_lineno, Py_ssize_t end_col_offset, - const char *errmsg, va_list va) -{ - PyObject *value = NULL; - PyObject *errstr = NULL; - PyObject *error_line = NULL; - PyObject *tmp = NULL; - p->error_indicator = 1; - - if (end_lineno == CURRENT_POS) { - end_lineno = p->tok->lineno; - } - if (end_col_offset == CURRENT_POS) { - end_col_offset = p->tok->cur - p->tok->line_start; - } - - if (p->start_rule == Py_fstring_input) { - const char *fstring_msg = "f-string: "; - Py_ssize_t len = strlen(fstring_msg) + strlen(errmsg); - - char *new_errmsg = PyMem_Malloc(len + 1); // Lengths of both strings plus NULL character - if (!new_errmsg) { - return (void *) PyErr_NoMemory(); - } - - // Copy both strings into new buffer - memcpy(new_errmsg, fstring_msg, strlen(fstring_msg)); - memcpy(new_errmsg + strlen(fstring_msg), errmsg, strlen(errmsg)); - new_errmsg[len] = 0; - errmsg = new_errmsg; - } - errstr = PyUnicode_FromFormatV(errmsg, va); - if (!errstr) { - goto error; - } - - if (p->tok->fp_interactive) { - error_line = get_error_line_from_tokenizer_buffers(p, lineno); - } - else if (p->start_rule == Py_file_input) { - error_line = _PyErr_ProgramDecodedTextObject(p->tok->filename, - (int) lineno, p->tok->encoding); - } - - if (!error_line) { - /* PyErr_ProgramTextObject was not called or returned NULL. If it was not called, - then we need to find the error line from some other source, because - p->start_rule != Py_file_input. If it returned NULL, then it either unexpectedly - failed or we're parsing from a string or the REPL. There's a third edge case where - we're actually parsing from a file, which has an E_EOF SyntaxError and in that case - `PyErr_ProgramTextObject` fails because lineno points to last_file_line + 1, which - does not physically exist */ - assert(p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF); - - if (p->tok->lineno <= lineno && p->tok->inp > p->tok->buf) { - Py_ssize_t size = p->tok->inp - p->tok->buf; - error_line = PyUnicode_DecodeUTF8(p->tok->buf, size, "replace"); - } - else if (p->tok->fp == NULL || p->tok->fp == stdin) { - error_line = get_error_line_from_tokenizer_buffers(p, lineno); - } - else { - error_line = PyUnicode_FromStringAndSize("", 0); - } - if (!error_line) { - goto error; - } - } - - if (p->start_rule == Py_fstring_input) { - col_offset -= p->starting_col_offset; - end_col_offset -= p->starting_col_offset; - } - - Py_ssize_t col_number = col_offset; - Py_ssize_t end_col_number = end_col_offset; - - if (p->tok->encoding != NULL) { - col_number = _PyPegen_byte_offset_to_character_offset(error_line, col_offset); - if (col_number < 0) { - goto error; - } - if (end_col_number > 0) { - Py_ssize_t end_col_offset = _PyPegen_byte_offset_to_character_offset(error_line, end_col_number); - if (end_col_offset < 0) { - goto error; - } else { - end_col_number = end_col_offset; - } - } - } - tmp = Py_BuildValue("(OiiNii)", p->tok->filename, lineno, col_number, error_line, end_lineno, end_col_number); - if (!tmp) { - goto error; - } - value = PyTuple_Pack(2, errstr, tmp); - Py_DECREF(tmp); - if (!value) { - goto error; - } - PyErr_SetObject(errtype, value); - - Py_DECREF(errstr); - Py_DECREF(value); - if (p->start_rule == Py_fstring_input) { - PyMem_Free((void *)errmsg); - } - return NULL; - -error: - Py_XDECREF(errstr); - Py_XDECREF(error_line); - if (p->start_rule == Py_fstring_input) { - PyMem_Free((void *)errmsg); - } - return NULL; -} - -void -_Pypegen_set_syntax_error(Parser* p, Token* last_token) { - // Existing sintax error - if (PyErr_Occurred()) { - // Prioritize tokenizer errors to custom syntax errors raised - // on the second phase only if the errors come from the parser. - if (p->tok->done == E_DONE && PyErr_ExceptionMatches(PyExc_SyntaxError)) { - _PyPegen_tokenize_full_source_to_check_for_errors(p); - } - // Propagate the existing syntax error. - return; - } - // Initialization error - if (p->fill == 0) { - RAISE_SYNTAX_ERROR("error at start before reading any input"); - } - // Parser encountered EOF (End of File) unexpectedtly - if (last_token->type == ERRORTOKEN && p->tok->done == E_EOF) { - if (p->tok->level) { - raise_unclosed_parentheses_error(p); - } else { - RAISE_SYNTAX_ERROR("unexpected EOF while parsing"); - } - return; - } - // Indentation error in the tokenizer - if (last_token->type == INDENT || last_token->type == DEDENT) { - RAISE_INDENTATION_ERROR(last_token->type == INDENT ? "unexpected indent" : "unexpected unindent"); - return; - } - // Unknown error (generic case) - - // Use the last token we found on the first pass to avoid reporting - // incorrect locations for generic syntax errors just because we reached - // further away when trying to find specific syntax errors in the second - // pass. - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(last_token, "invalid syntax"); - // _PyPegen_tokenize_full_source_to_check_for_errors will override the existing - // generic SyntaxError we just raised if errors are found. - _PyPegen_tokenize_full_source_to_check_for_errors(p); -} From 876ade1ae3a805b546a211fd7303253c10395569 Mon Sep 17 00:00:00 2001 From: Erlend Egeberg Aasland Date: Thu, 20 Jan 2022 22:06:47 +0100 Subject: [PATCH 023/220] [3.10] Mark all clinic headers as generated (GH-30679). (GH-30726) (cherry picked from commit 71734d0b9ca584bcbdcb2fb44ae16bb2fbfcaf6e) Co-authored-by: Erlend Egeberg Aasland --- .gitattributes | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.gitattributes b/.gitattributes index c66e765266382f..be369d2a5c63c4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -40,11 +40,8 @@ PCbuild/readme.txt text eol=crlf PC/readme.txt text eol=crlf # Generated files -# https://github.com/github/linguist#generated-code -Modules/clinic/*.h linguist-generated=true -Objects/clinic/*.h linguist-generated=true -PC/clinic/*.h linguist-generated=true -Python/clinic/*.h linguist-generated=true +# https://github.com/github/linguist/blob/master/docs/overrides.md +**/clinic/*.h linguist-generated=true Python/importlib.h linguist-generated=true Python/importlib_external.h linguist-generated=true Include/internal/pycore_ast.h linguist-generated=true From e5edc8d737a45d9d8b9b93b8be52f85d79d0f417 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 20 Jan 2022 15:13:17 -0800 Subject: [PATCH 024/220] bpo-46080: fix argparse help generation exception in edge case (GH-30111) Fix an uncaught exception during help text generation when argparse.BooleanOptionalAction is used with default=argparse.SUPPRESS and help is specified. (cherry picked from commit 9e87c0e03fa501fb90008547983ce4c1dcaaf90c) Co-authored-by: Felix Fontein --- Lib/argparse.py | 2 +- Lib/test/test_argparse.py | 7 +++++-- .../next/Library/2021-12-15-06-29-00.bpo-46080.AuQpLt.rst | 3 +++ 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-12-15-06-29-00.bpo-46080.AuQpLt.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index e177e4fe034d32..b71a6703c3982b 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -878,7 +878,7 @@ def __init__(self, option_string = '--no-' + option_string[2:] _option_strings.append(option_string) - if help is not None and default is not None: + if help is not None and default is not None and default is not SUPPRESS: help += " (default: %(default)s)" super().__init__( diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 37a73e06863774..9d66ace5474a1c 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -3597,6 +3597,8 @@ class TestHelpUsage(HelpTestCase): Sig('--bar', help='Whether to bar', default=True, action=argparse.BooleanOptionalAction), Sig('-f', '--foobar', '--barfoo', action=argparse.BooleanOptionalAction), + Sig('--bazz', action=argparse.BooleanOptionalAction, + default=argparse.SUPPRESS, help='Bazz!'), ] argument_group_signatures = [ (Sig('group'), [ @@ -3609,8 +3611,8 @@ class TestHelpUsage(HelpTestCase): usage = '''\ usage: PROG [-h] [-w W [W ...]] [-x [X ...]] [--foo | --no-foo] [--bar | --no-bar] - [-f | --foobar | --no-foobar | --barfoo | --no-barfoo] [-y [Y]] - [-z Z Z Z] + [-f | --foobar | --no-foobar | --barfoo | --no-barfoo] + [--bazz | --no-bazz] [-y [Y]] [-z Z Z Z] a b b [c] [d ...] e [e ...] ''' help = usage + '''\ @@ -3627,6 +3629,7 @@ class TestHelpUsage(HelpTestCase): --foo, --no-foo Whether to foo --bar, --no-bar Whether to bar (default: True) -f, --foobar, --no-foobar, --barfoo, --no-barfoo + --bazz, --no-bazz Bazz! group: -y [Y] y diff --git a/Misc/NEWS.d/next/Library/2021-12-15-06-29-00.bpo-46080.AuQpLt.rst b/Misc/NEWS.d/next/Library/2021-12-15-06-29-00.bpo-46080.AuQpLt.rst new file mode 100644 index 00000000000000..e42d84e31e759f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-12-15-06-29-00.bpo-46080.AuQpLt.rst @@ -0,0 +1,3 @@ +Fix exception in argparse help text generation if a +:class:`argparse.BooleanOptionalAction` argument's default is +``argparse.SUPPRESS`` and it has ``help`` specified. Patch by Felix Fontein. \ No newline at end of file From 1d11fdd3eeff77ba600278433b7ab0ce4d2a7f3b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 21 Jan 2022 00:05:57 -0800 Subject: [PATCH 025/220] bpo-21987: Fix TarFile.getmember getting a dir with a trailing slash (GH-30283) (cherry picked from commit cfadcc31ea84617b1c73022ce54d4ae831333e8d) Co-authored-by: andrei kulakov --- Lib/tarfile.py | 2 +- Lib/test/test_tarfile.py | 19 +++++++++++++++++++ .../2021-12-28-11-55-10.bpo-21987.avBK-p.rst | 2 ++ 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2021-12-28-11-55-10.bpo-21987.avBK-p.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index c1ee1222e09b5a..e187da2b1994a6 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1789,7 +1789,7 @@ def getmember(self, name): than once in the archive, its last occurrence is assumed to be the most up-to-date version. """ - tarinfo = self._getmember(name) + tarinfo = self._getmember(name.rstrip('/')) if tarinfo is None: raise KeyError("filename %r not found" % name) return tarinfo diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index e4b5c52bf1eaf4..1357df57eb1797 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -220,6 +220,25 @@ def test_fileobj_symlink2(self): def test_issue14160(self): self._test_fileobj_link("symtype2", "ustar/regtype") + def test_add_dir_getmember(self): + # bpo-21987 + self.add_dir_and_getmember('bar') + self.add_dir_and_getmember('a'*101) + + def add_dir_and_getmember(self, name): + with os_helper.temp_cwd(): + with tarfile.open(tmpname, 'w') as tar: + try: + os.mkdir(name) + tar.add(name) + finally: + os.rmdir(name) + with tarfile.open(tmpname) as tar: + self.assertEqual( + tar.getmember(name), + tar.getmember(name + '/') + ) + class GzipUstarReadTest(GzipTest, UstarReadTest): pass diff --git a/Misc/NEWS.d/next/Library/2021-12-28-11-55-10.bpo-21987.avBK-p.rst b/Misc/NEWS.d/next/Library/2021-12-28-11-55-10.bpo-21987.avBK-p.rst new file mode 100644 index 00000000000000..305dd16d53b495 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-12-28-11-55-10.bpo-21987.avBK-p.rst @@ -0,0 +1,2 @@ +Fix an issue with :meth:`tarfile.TarFile.getmember` getting a directory name +with a trailing slash. From f6e5972fa984c10d47694973db1c91c6486d654a Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Fri, 21 Jan 2022 11:02:25 +0200 Subject: [PATCH 026/220] [3.10] bpo-41857: mention timeout argument units in select.poll() and select.depoll() doc-strings (GH-22406) (cherry picked from commit 27df7566bc19699b967e0e30d7808637b90141f6) Co-authored-by: Zane Bitter --- Modules/clinic/selectmodule.c.h | 10 +++++++++- Modules/selectmodule.c | 8 ++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index d7095dfb00ead9..be752e981667ce 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -193,6 +193,10 @@ PyDoc_STRVAR(select_poll_poll__doc__, "\n" "Polls the set of registered file descriptors.\n" "\n" +" timeout\n" +" The maximum time to wait in milliseconds, or else None (or a negative\n" +" value) to wait indefinitely.\n" +"\n" "Returns a list containing any descriptors that have events or errors to\n" "report, as a list of (fd, event) 2-tuples."); @@ -363,6 +367,10 @@ PyDoc_STRVAR(select_devpoll_poll__doc__, "\n" "Polls the set of registered file descriptors.\n" "\n" +" timeout\n" +" The maximum time to wait in milliseconds, or else None (or a negative\n" +" value) to wait indefinitely.\n" +"\n" "Returns a list containing any descriptors that have events or errors to\n" "report, as a list of (fd, event) 2-tuples."); @@ -1179,4 +1187,4 @@ select_kqueue_control(kqueue_queue_Object *self, PyObject *const *args, Py_ssize #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=cd2062a787e13b35 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a8fc031269d28454 input=a9049054013a1b77]*/ diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 3ecd0c32b30389..3afcb0e2a02206 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -564,6 +564,8 @@ select_poll_unregister_impl(pollObject *self, int fd) select.poll.poll timeout as timeout_obj: object = None + The maximum time to wait in milliseconds, or else None (or a negative + value) to wait indefinitely. / Polls the set of registered file descriptors. @@ -574,7 +576,7 @@ report, as a list of (fd, event) 2-tuples. static PyObject * select_poll_poll_impl(pollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=876e837d193ed7e4 input=7a446ed45189e894]*/ +/*[clinic end generated code: output=876e837d193ed7e4 input=c2f6953ec45e5622]*/ { PyObject *result_list = NULL; int poll_result, i, j; @@ -888,6 +890,8 @@ select_devpoll_unregister_impl(devpollObject *self, int fd) /*[clinic input] select.devpoll.poll timeout as timeout_obj: object = None + The maximum time to wait in milliseconds, or else None (or a negative + value) to wait indefinitely. / Polls the set of registered file descriptors. @@ -898,7 +902,7 @@ report, as a list of (fd, event) 2-tuples. static PyObject * select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=2654e5457cca0b3c input=fd0db698d84f0333]*/ +/*[clinic end generated code: output=2654e5457cca0b3c input=3c3f0a355ec2bedb]*/ { struct dvpoll dvp; PyObject *result_list = NULL; From 68a31dba975419b7b4432fa31730e5ca67071d9f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 21 Jan 2022 04:45:42 -0800 Subject: [PATCH 027/220] no-issue: Fix documentation typos. (GH-30576) (cherry picked from commit d05a66339b5e07d72d96e4c30a34cc3821bb61a2) Co-authored-by: Piotr Fusik --- Doc/c-api/init_config.rst | 2 +- Doc/howto/descriptor.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index c037f19ce64f31..b8b41510b89ec8 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -634,7 +634,7 @@ PyConfig .. c:member:: int dump_refs - Dump Python refererences? + Dump Python references? If non-zero, dump all objects which are still alive at exit. diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 6ce062d0fa853e..f8b1e00d96fadc 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -1544,7 +1544,7 @@ variables: 'Simulate how the type metaclass adds member objects for slots' def __new__(mcls, clsname, bases, mapping): - 'Emuluate type_new() in Objects/typeobject.c' + 'Emulate type_new() in Objects/typeobject.c' # type_new() calls PyTypeReady() which calls add_methods() slot_names = mapping.get('slot_names', []) for offset, name in enumerate(slot_names): From a1015c6478e8cbec2ecb984a3cba733783d168b5 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 21 Jan 2022 09:31:25 -0800 Subject: [PATCH 028/220] bpo-46426: Improve tests for the dir_fd argument (GH-30668) (GH-30739) Ensure that directory file descriptors refer to directories different from the current directory, and that src_dir_fd and dst_dir_fd refer to different directories. Add context manager open_dir_fd() in test.support.os_helper. (cherry picked from commit 54610bb448a9cf5be77d53b66169fca4c11be6cb) Co-authored-by: Serhiy Storchaka --- Lib/test/support/os_helper.py | 11 + Lib/test/test_os.py | 10 +- Lib/test/test_posix.py | 417 ++++++++++++++++------------------ 3 files changed, 208 insertions(+), 230 deletions(-) diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index d9807a1e114b65..82a6de789c866e 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -455,6 +455,17 @@ def create_empty_file(filename): os.close(fd) +@contextlib.contextmanager +def open_dir_fd(path): + """Open a file descriptor to a directory.""" + assert os.path.isdir(path) + dir_fd = os.open(path, os.O_RDONLY) + try: + yield dir_fd + finally: + os.close(dir_fd) + + def fs_is_case_insensitive(directory): """Detects if the file system for the specified directory is case-insensitive.""" diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 0ad13d59ded372..7f7d14ef0a0951 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -848,12 +848,9 @@ def set_time(filename, ns): def test_utime_dir_fd(self): def set_time(filename, ns): dirname, name = os.path.split(filename) - dirfd = os.open(dirname, os.O_RDONLY) - try: + with os_helper.open_dir_fd(dirname) as dirfd: # pass dir_fd to test utimensat(timespec) or futimesat(timeval) os.utime(name, dir_fd=dirfd, ns=ns) - finally: - os.close(dirfd) self._test_utime(set_time) def test_utime_directory(self): @@ -4339,8 +4336,7 @@ def test_fd(self): os.symlink('file.txt', os.path.join(self.path, 'link')) expected_names.append('link') - fd = os.open(self.path, os.O_RDONLY) - try: + with os_helper.open_dir_fd(self.path) as fd: with os.scandir(fd) as it: entries = list(it) names = [entry.name for entry in entries] @@ -4355,8 +4351,6 @@ def test_fd(self): self.assertEqual(entry.stat(), st) st = os.stat(entry.name, dir_fd=fd, follow_symlinks=False) self.assertEqual(entry.stat(follow_symlinks=False), st) - finally: - os.close(fd) def test_empty_path(self): self.assertRaises(FileNotFoundError, os.scandir, '') diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 56b72f465c1c08..974edd766cc809 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -21,6 +21,7 @@ import unittest import warnings import textwrap +from contextlib import contextmanager _DUMMY_SYMLINK = os.path.join(tempfile.gettempdir(), os_helper.TESTFN + '-dummy-symlink') @@ -1081,187 +1082,6 @@ def test_getgroups(self): symdiff = idg_groups.symmetric_difference(posix.getgroups()) self.assertTrue(not symdiff or symdiff == {posix.getegid()}) - # tests for the posix *at functions follow - - @unittest.skipUnless(os.access in os.supports_dir_fd, "test needs dir_fd support for os.access()") - def test_access_dir_fd(self): - f = posix.open(posix.getcwd(), posix.O_RDONLY) - try: - self.assertTrue(posix.access(os_helper.TESTFN, os.R_OK, dir_fd=f)) - finally: - posix.close(f) - - @unittest.skipUnless(os.chmod in os.supports_dir_fd, "test needs dir_fd support in os.chmod()") - def test_chmod_dir_fd(self): - os.chmod(os_helper.TESTFN, stat.S_IRUSR) - - f = posix.open(posix.getcwd(), posix.O_RDONLY) - try: - posix.chmod(os_helper.TESTFN, stat.S_IRUSR | stat.S_IWUSR, dir_fd=f) - - s = posix.stat(os_helper.TESTFN) - self.assertEqual(s[0] & stat.S_IRWXU, stat.S_IRUSR | stat.S_IWUSR) - finally: - posix.close(f) - - @unittest.skipUnless(hasattr(os, 'chown') and (os.chown in os.supports_dir_fd), - "test needs dir_fd support in os.chown()") - def test_chown_dir_fd(self): - os_helper.unlink(os_helper.TESTFN) - os_helper.create_empty_file(os_helper.TESTFN) - - f = posix.open(posix.getcwd(), posix.O_RDONLY) - try: - posix.chown(os_helper.TESTFN, os.getuid(), os.getgid(), dir_fd=f) - finally: - posix.close(f) - - @unittest.skipUnless(os.stat in os.supports_dir_fd, "test needs dir_fd support in os.stat()") - def test_stat_dir_fd(self): - os_helper.unlink(os_helper.TESTFN) - with open(os_helper.TESTFN, 'w') as outfile: - outfile.write("testline\n") - - f = posix.open(posix.getcwd(), posix.O_RDONLY) - try: - s1 = posix.stat(os_helper.TESTFN) - s2 = posix.stat(os_helper.TESTFN, dir_fd=f) - self.assertEqual(s1, s2) - s2 = posix.stat(os_helper.TESTFN, dir_fd=None) - self.assertEqual(s1, s2) - self.assertRaisesRegex(TypeError, 'should be integer or None, not', - posix.stat, os_helper.TESTFN, dir_fd=posix.getcwd()) - self.assertRaisesRegex(TypeError, 'should be integer or None, not', - posix.stat, os_helper.TESTFN, dir_fd=float(f)) - self.assertRaises(OverflowError, - posix.stat, os_helper.TESTFN, dir_fd=10**20) - finally: - posix.close(f) - - @unittest.skipUnless(os.utime in os.supports_dir_fd, "test needs dir_fd support in os.utime()") - def test_utime_dir_fd(self): - f = posix.open(posix.getcwd(), posix.O_RDONLY) - try: - now = time.time() - posix.utime(os_helper.TESTFN, None, dir_fd=f) - posix.utime(os_helper.TESTFN, dir_fd=f) - self.assertRaises(TypeError, posix.utime, os_helper.TESTFN, - now, dir_fd=f) - self.assertRaises(TypeError, posix.utime, os_helper.TESTFN, - (None, None), dir_fd=f) - self.assertRaises(TypeError, posix.utime, os_helper.TESTFN, - (now, None), dir_fd=f) - self.assertRaises(TypeError, posix.utime, os_helper.TESTFN, - (None, now), dir_fd=f) - self.assertRaises(TypeError, posix.utime, os_helper.TESTFN, - (now, "x"), dir_fd=f) - posix.utime(os_helper.TESTFN, (int(now), int(now)), dir_fd=f) - posix.utime(os_helper.TESTFN, (now, now), dir_fd=f) - posix.utime(os_helper.TESTFN, - (int(now), int((now - int(now)) * 1e9)), dir_fd=f) - posix.utime(os_helper.TESTFN, dir_fd=f, - times=(int(now), int((now - int(now)) * 1e9))) - - # try dir_fd and follow_symlinks together - if os.utime in os.supports_follow_symlinks: - try: - posix.utime(os_helper.TESTFN, follow_symlinks=False, - dir_fd=f) - except ValueError: - # whoops! using both together not supported on this platform. - pass - - finally: - posix.close(f) - - @unittest.skipUnless(os.link in os.supports_dir_fd, "test needs dir_fd support in os.link()") - def test_link_dir_fd(self): - f = posix.open(posix.getcwd(), posix.O_RDONLY) - try: - posix.link(os_helper.TESTFN, os_helper.TESTFN + 'link', - src_dir_fd=f, dst_dir_fd=f) - except PermissionError as e: - self.skipTest('posix.link(): %s' % e) - else: - # should have same inodes - self.assertEqual(posix.stat(os_helper.TESTFN)[1], - posix.stat(os_helper.TESTFN + 'link')[1]) - finally: - posix.close(f) - os_helper.unlink(os_helper.TESTFN + 'link') - - @unittest.skipUnless(os.mkdir in os.supports_dir_fd, "test needs dir_fd support in os.mkdir()") - def test_mkdir_dir_fd(self): - f = posix.open(posix.getcwd(), posix.O_RDONLY) - try: - posix.mkdir(os_helper.TESTFN + 'dir', dir_fd=f) - posix.stat(os_helper.TESTFN + 'dir') # should not raise exception - finally: - posix.close(f) - os_helper.rmtree(os_helper.TESTFN + 'dir') - - @unittest.skipUnless(hasattr(os, 'mknod') - and (os.mknod in os.supports_dir_fd) - and hasattr(stat, 'S_IFIFO'), - "test requires both stat.S_IFIFO and dir_fd support for os.mknod()") - def test_mknod_dir_fd(self): - # Test using mknodat() to create a FIFO (the only use specified - # by POSIX). - os_helper.unlink(os_helper.TESTFN) - mode = stat.S_IFIFO | stat.S_IRUSR | stat.S_IWUSR - f = posix.open(posix.getcwd(), posix.O_RDONLY) - try: - posix.mknod(os_helper.TESTFN, mode, 0, dir_fd=f) - except OSError as e: - # Some old systems don't allow unprivileged users to use - # mknod(), or only support creating device nodes. - self.assertIn(e.errno, (errno.EPERM, errno.EINVAL, errno.EACCES)) - else: - self.assertTrue(stat.S_ISFIFO(posix.stat(os_helper.TESTFN).st_mode)) - finally: - posix.close(f) - - @unittest.skipUnless(os.open in os.supports_dir_fd, "test needs dir_fd support in os.open()") - def test_open_dir_fd(self): - os_helper.unlink(os_helper.TESTFN) - with open(os_helper.TESTFN, 'w') as outfile: - outfile.write("testline\n") - a = posix.open(posix.getcwd(), posix.O_RDONLY) - b = posix.open(os_helper.TESTFN, posix.O_RDONLY, dir_fd=a) - try: - res = posix.read(b, 9).decode(encoding="utf-8") - self.assertEqual("testline\n", res) - finally: - posix.close(a) - posix.close(b) - - @unittest.skipUnless(hasattr(os, 'readlink') and (os.readlink in os.supports_dir_fd), - "test needs dir_fd support in os.readlink()") - def test_readlink_dir_fd(self): - os.symlink(os_helper.TESTFN, os_helper.TESTFN + 'link') - f = posix.open(posix.getcwd(), posix.O_RDONLY) - try: - self.assertEqual(posix.readlink(os_helper.TESTFN + 'link'), - posix.readlink(os_helper.TESTFN + 'link', dir_fd=f)) - finally: - os_helper.unlink(os_helper.TESTFN + 'link') - posix.close(f) - - @unittest.skipUnless(os.rename in os.supports_dir_fd, "test needs dir_fd support in os.rename()") - def test_rename_dir_fd(self): - os_helper.unlink(os_helper.TESTFN) - os_helper.create_empty_file(os_helper.TESTFN + 'ren') - f = posix.open(posix.getcwd(), posix.O_RDONLY) - try: - posix.rename(os_helper.TESTFN + 'ren', os_helper.TESTFN, src_dir_fd=f, dst_dir_fd=f) - except: - posix.rename(os_helper.TESTFN + 'ren', os_helper.TESTFN) - raise - else: - posix.stat(os_helper.TESTFN) # should not raise exception - finally: - posix.close(f) - @unittest.skipUnless(hasattr(signal, 'SIGCHLD'), 'CLD_XXXX be placed in si_code for a SIGCHLD signal') @unittest.skipUnless(hasattr(os, 'waitid_result'), "test needs os.waitid_result") def test_cld_xxxx_constants(self): @@ -1272,47 +1092,6 @@ def test_cld_xxxx_constants(self): os.CLD_STOPPED os.CLD_CONTINUED - @unittest.skipUnless(os.symlink in os.supports_dir_fd, "test needs dir_fd support in os.symlink()") - def test_symlink_dir_fd(self): - f = posix.open(posix.getcwd(), posix.O_RDONLY) - try: - posix.symlink(os_helper.TESTFN, os_helper.TESTFN + 'link', - dir_fd=f) - self.assertEqual(posix.readlink(os_helper.TESTFN + 'link'), - os_helper.TESTFN) - finally: - posix.close(f) - os_helper.unlink(os_helper.TESTFN + 'link') - - @unittest.skipUnless(os.unlink in os.supports_dir_fd, "test needs dir_fd support in os.unlink()") - def test_unlink_dir_fd(self): - f = posix.open(posix.getcwd(), posix.O_RDONLY) - os_helper.create_empty_file(os_helper.TESTFN + 'del') - posix.stat(os_helper.TESTFN + 'del') # should not raise exception - try: - posix.unlink(os_helper.TESTFN + 'del', dir_fd=f) - except: - os_helper.unlink(os_helper.TESTFN + 'del') - raise - else: - self.assertRaises(OSError, posix.stat, os_helper.TESTFN + 'link') - finally: - posix.close(f) - - @unittest.skipUnless(os.mkfifo in os.supports_dir_fd, "test needs dir_fd support in os.mkfifo()") - def test_mkfifo_dir_fd(self): - os_helper.unlink(os_helper.TESTFN) - f = posix.open(posix.getcwd(), posix.O_RDONLY) - try: - try: - posix.mkfifo(os_helper.TESTFN, - stat.S_IRUSR | stat.S_IWUSR, dir_fd=f) - except PermissionError as e: - self.skipTest('posix.mkfifo(): %s' % e) - self.assertTrue(stat.S_ISFIFO(posix.stat(os_helper.TESTFN).st_mode)) - finally: - posix.close(f) - requires_sched_h = unittest.skipUnless(hasattr(posix, 'sched_yield'), "don't have scheduling support") requires_sched_affinity = unittest.skipUnless(hasattr(posix, 'sched_setaffinity'), @@ -1519,6 +1298,200 @@ def test_pidfd_open(self): self.assertEqual(cm.exception.errno, errno.EINVAL) os.close(os.pidfd_open(os.getpid(), 0)) + +# tests for the posix *at functions follow +class TestPosixDirFd(unittest.TestCase): + count = 0 + + @contextmanager + def prepare(self): + TestPosixDirFd.count += 1 + name = f'{os_helper.TESTFN}_{self.count}' + base_dir = f'{os_helper.TESTFN}_{self.count}base' + posix.mkdir(base_dir) + self.addCleanup(posix.rmdir, base_dir) + fullname = os.path.join(base_dir, name) + assert not os.path.exists(fullname) + with os_helper.open_dir_fd(base_dir) as dir_fd: + yield (dir_fd, name, fullname) + + @contextmanager + def prepare_file(self): + with self.prepare() as (dir_fd, name, fullname): + os_helper.create_empty_file(fullname) + self.addCleanup(posix.unlink, fullname) + yield (dir_fd, name, fullname) + + @unittest.skipUnless(os.access in os.supports_dir_fd, "test needs dir_fd support for os.access()") + def test_access_dir_fd(self): + with self.prepare_file() as (dir_fd, name, fullname): + self.assertTrue(posix.access(name, os.R_OK, dir_fd=dir_fd)) + + @unittest.skipUnless(os.chmod in os.supports_dir_fd, "test needs dir_fd support in os.chmod()") + def test_chmod_dir_fd(self): + with self.prepare_file() as (dir_fd, name, fullname): + posix.chmod(fullname, stat.S_IRUSR) + posix.chmod(name, stat.S_IRUSR | stat.S_IWUSR, dir_fd=dir_fd) + s = posix.stat(fullname) + self.assertEqual(s.st_mode & stat.S_IRWXU, + stat.S_IRUSR | stat.S_IWUSR) + + @unittest.skipUnless(hasattr(os, 'chown') and (os.chown in os.supports_dir_fd), + "test needs dir_fd support in os.chown()") + def test_chown_dir_fd(self): + with self.prepare_file() as (dir_fd, name, fullname): + posix.chown(name, os.getuid(), os.getgid(), dir_fd=dir_fd) + + @unittest.skipUnless(os.stat in os.supports_dir_fd, "test needs dir_fd support in os.stat()") + def test_stat_dir_fd(self): + with self.prepare() as (dir_fd, name, fullname): + with open(fullname, 'w') as outfile: + outfile.write("testline\n") + self.addCleanup(posix.unlink, fullname) + + s1 = posix.stat(fullname) + s2 = posix.stat(name, dir_fd=dir_fd) + self.assertEqual(s1, s2) + s2 = posix.stat(fullname, dir_fd=None) + self.assertEqual(s1, s2) + + self.assertRaisesRegex(TypeError, 'should be integer or None, not', + posix.stat, name, dir_fd=posix.getcwd()) + self.assertRaisesRegex(TypeError, 'should be integer or None, not', + posix.stat, name, dir_fd=float(dir_fd)) + self.assertRaises(OverflowError, + posix.stat, name, dir_fd=10**20) + + @unittest.skipUnless(os.utime in os.supports_dir_fd, "test needs dir_fd support in os.utime()") + def test_utime_dir_fd(self): + with self.prepare_file() as (dir_fd, name, fullname): + now = time.time() + posix.utime(name, None, dir_fd=dir_fd) + posix.utime(name, dir_fd=dir_fd) + self.assertRaises(TypeError, posix.utime, name, + now, dir_fd=dir_fd) + self.assertRaises(TypeError, posix.utime, name, + (None, None), dir_fd=dir_fd) + self.assertRaises(TypeError, posix.utime, name, + (now, None), dir_fd=dir_fd) + self.assertRaises(TypeError, posix.utime, name, + (None, now), dir_fd=dir_fd) + self.assertRaises(TypeError, posix.utime, name, + (now, "x"), dir_fd=dir_fd) + posix.utime(name, (int(now), int(now)), dir_fd=dir_fd) + posix.utime(name, (now, now), dir_fd=dir_fd) + posix.utime(name, + (int(now), int((now - int(now)) * 1e9)), dir_fd=dir_fd) + posix.utime(name, dir_fd=dir_fd, + times=(int(now), int((now - int(now)) * 1e9))) + + # try dir_fd and follow_symlinks together + if os.utime in os.supports_follow_symlinks: + try: + posix.utime(name, follow_symlinks=False, dir_fd=dir_fd) + except ValueError: + # whoops! using both together not supported on this platform. + pass + + @unittest.skipUnless(os.link in os.supports_dir_fd, "test needs dir_fd support in os.link()") + def test_link_dir_fd(self): + with self.prepare_file() as (dir_fd, name, fullname), \ + self.prepare() as (dir_fd2, linkname, fulllinkname): + try: + posix.link(name, linkname, src_dir_fd=dir_fd, dst_dir_fd=dir_fd2) + except PermissionError as e: + self.skipTest('posix.link(): %s' % e) + self.addCleanup(posix.unlink, fulllinkname) + # should have same inodes + self.assertEqual(posix.stat(fullname)[1], + posix.stat(fulllinkname)[1]) + + @unittest.skipUnless(os.mkdir in os.supports_dir_fd, "test needs dir_fd support in os.mkdir()") + def test_mkdir_dir_fd(self): + with self.prepare() as (dir_fd, name, fullname): + posix.mkdir(name, dir_fd=dir_fd) + self.addCleanup(posix.rmdir, fullname) + posix.stat(fullname) # should not raise exception + + @unittest.skipUnless(hasattr(os, 'mknod') + and (os.mknod in os.supports_dir_fd) + and hasattr(stat, 'S_IFIFO'), + "test requires both stat.S_IFIFO and dir_fd support for os.mknod()") + def test_mknod_dir_fd(self): + # Test using mknodat() to create a FIFO (the only use specified + # by POSIX). + with self.prepare() as (dir_fd, name, fullname): + mode = stat.S_IFIFO | stat.S_IRUSR | stat.S_IWUSR + try: + posix.mknod(name, mode, 0, dir_fd=dir_fd) + except OSError as e: + # Some old systems don't allow unprivileged users to use + # mknod(), or only support creating device nodes. + self.assertIn(e.errno, (errno.EPERM, errno.EINVAL, errno.EACCES)) + else: + self.addCleanup(posix.unlink, fullname) + self.assertTrue(stat.S_ISFIFO(posix.stat(fullname).st_mode)) + + @unittest.skipUnless(os.open in os.supports_dir_fd, "test needs dir_fd support in os.open()") + def test_open_dir_fd(self): + with self.prepare() as (dir_fd, name, fullname): + with open(fullname, 'wb') as outfile: + outfile.write(b"testline\n") + self.addCleanup(posix.unlink, fullname) + fd = posix.open(name, posix.O_RDONLY, dir_fd=dir_fd) + try: + res = posix.read(fd, 9) + self.assertEqual(b"testline\n", res) + finally: + posix.close(fd) + + @unittest.skipUnless(hasattr(os, 'readlink') and (os.readlink in os.supports_dir_fd), + "test needs dir_fd support in os.readlink()") + def test_readlink_dir_fd(self): + with self.prepare() as (dir_fd, name, fullname): + os.symlink('symlink', fullname) + self.addCleanup(posix.unlink, fullname) + self.assertEqual(posix.readlink(name, dir_fd=dir_fd), 'symlink') + + @unittest.skipUnless(os.rename in os.supports_dir_fd, "test needs dir_fd support in os.rename()") + def test_rename_dir_fd(self): + with self.prepare_file() as (dir_fd, name, fullname), \ + self.prepare() as (dir_fd2, name2, fullname2): + posix.rename(name, name2, + src_dir_fd=dir_fd, dst_dir_fd=dir_fd2) + posix.stat(fullname2) # should not raise exception + posix.rename(fullname2, fullname) + + @unittest.skipUnless(os.symlink in os.supports_dir_fd, "test needs dir_fd support in os.symlink()") + def test_symlink_dir_fd(self): + with self.prepare() as (dir_fd, name, fullname): + posix.symlink('symlink', name, dir_fd=dir_fd) + self.addCleanup(posix.unlink, fullname) + self.assertEqual(posix.readlink(fullname), 'symlink') + + @unittest.skipUnless(os.unlink in os.supports_dir_fd, "test needs dir_fd support in os.unlink()") + def test_unlink_dir_fd(self): + with self.prepare() as (dir_fd, name, fullname): + os_helper.create_empty_file(fullname) + posix.stat(fullname) # should not raise exception + try: + posix.unlink(name, dir_fd=dir_fd) + self.assertRaises(OSError, posix.stat, fullname) + except: + self.addCleanup(posix.unlink, fullname) + raise + + @unittest.skipUnless(os.mkfifo in os.supports_dir_fd, "test needs dir_fd support in os.mkfifo()") + def test_mkfifo_dir_fd(self): + with self.prepare() as (dir_fd, name, fullname): + try: + posix.mkfifo(name, stat.S_IRUSR | stat.S_IWUSR, dir_fd=dir_fd) + except PermissionError as e: + self.skipTest('posix.mkfifo(): %s' % e) + self.addCleanup(posix.unlink, fullname) + self.assertTrue(stat.S_ISFIFO(posix.stat(fullname).st_mode)) + + class PosixGroupsTester(unittest.TestCase): def setUp(self): From 05063fa15c594012e6dc9c2c7a3ea72e7cb933f2 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 21 Jan 2022 11:32:43 -0800 Subject: [PATCH 029/220] bpo-46434: Handle missing docstrings in pdb help (GH-30705) (cherry picked from commit 60705cff70576482fea31dcafbf8a37cbb751ea5) Co-authored-by: Tom Sparrow <793763+sparrowt@users.noreply.github.com> --- Lib/pdb.py | 3 +++ Lib/test/test_pdb.py | 21 +++++++++++++++++++ Misc/ACKS | 1 + .../2022-01-20-10-35-10.bpo-46434.geS-aP.rst | 2 ++ 4 files changed, 27 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2022-01-20-10-35-10.bpo-46434.geS-aP.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index 943211158ac41e..7ab50b4845d3e6 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1493,6 +1493,9 @@ def do_help(self, arg): self.error('No help for %r; please do not run Python with -OO ' 'if you need command help' % arg) return + if command.__doc__ is None: + self.error('No help for %r; __doc__ string missing' % arg) + return self.message(command.__doc__.rstrip()) do_h = do_help diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index d4c037dabff97e..6ac1a4a3c30255 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -1463,6 +1463,27 @@ def test_issue7964(self): self.assertNotIn(b'SyntaxError', stdout, "Got a syntax error running test script under PDB") + def test_issue46434(self): + # Temporarily patch in an extra help command which doesn't have a + # docstring to emulate what happens in an embeddable distribution + script = """ + def do_testcmdwithnodocs(self, arg): + pass + + import pdb + pdb.Pdb.do_testcmdwithnodocs = do_testcmdwithnodocs + """ + commands = """ + continue + help testcmdwithnodocs + """ + stdout, stderr = self.run_pdb_script(script, commands) + output = (stdout or '') + (stderr or '') + self.assertNotIn('AttributeError', output, + 'Calling help on a command with no docs should be handled gracefully') + self.assertIn("*** No help for 'testcmdwithnodocs'; __doc__ string missing", output, + 'Calling help on a command with no docs should print an error') + def test_issue13183(self): script = """ from bar import bar diff --git a/Misc/ACKS b/Misc/ACKS index 9292bdc8dc73b7..7f9166cd74cfab 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1668,6 +1668,7 @@ Evgeny Sologubov Cody Somerville Anthony Sottile Edoardo Spadolini +Tom Sparrow Geoffrey Spear Clay Spence Stefan Sperling diff --git a/Misc/NEWS.d/next/Library/2022-01-20-10-35-10.bpo-46434.geS-aP.rst b/Misc/NEWS.d/next/Library/2022-01-20-10-35-10.bpo-46434.geS-aP.rst new file mode 100644 index 00000000000000..6000781fa5aea9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-01-20-10-35-10.bpo-46434.geS-aP.rst @@ -0,0 +1,2 @@ +:mod:`pdb` now gracefully handles ``help`` when :attr:`__doc__` is missing, +for example when run with pregenerated optimized ``.pyc`` files. From d548c871716dfda73714d9f38b4e4219878a414e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 21 Jan 2022 14:11:47 -0800 Subject: [PATCH 030/220] bpo-46463: Fixes escape4chm.py script used when building the CHM documentation file (GH-30768) (cherry picked from commit 57d1855682dbeb9233ef3a531f9535c6442e9992) Co-authored-by: Steve Dower --- Doc/tools/extensions/escape4chm.py | 9 +++++---- .../2022-01-21-21-33-48.bpo-46463.fBbdTG.rst | 2 ++ 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2022-01-21-21-33-48.bpo-46463.fBbdTG.rst diff --git a/Doc/tools/extensions/escape4chm.py b/Doc/tools/extensions/escape4chm.py index e9999716251734..89970975b9032b 100644 --- a/Doc/tools/extensions/escape4chm.py +++ b/Doc/tools/extensions/escape4chm.py @@ -5,6 +5,7 @@ https://bugs.python.org/issue32174 """ +import pathlib import re from html.entities import codepoint2name @@ -39,12 +40,12 @@ def fixup_keywords(app, exception): return getLogger(__name__).info('fixing HTML escapes in keywords file...') - outdir = app.builder.outdir + outdir = pathlib.Path(app.builder.outdir) outname = app.builder.config.htmlhelp_basename - with app.builder.open_file(outdir, outname + '.hhk', 'r') as f: + with open(outdir / (outname + '.hhk'), 'rb') as f: index = f.read() - with app.builder.open_file(outdir, outname + '.hhk', 'w') as f: - f.write(index.replace(''', ''')) + with open(outdir / (outname + '.hhk'), 'wb') as f: + f.write(index.replace(b''', b''')) def setup(app): # `html-page-context` event emitted when the HTML builder has diff --git a/Misc/NEWS.d/next/Documentation/2022-01-21-21-33-48.bpo-46463.fBbdTG.rst b/Misc/NEWS.d/next/Documentation/2022-01-21-21-33-48.bpo-46463.fBbdTG.rst new file mode 100644 index 00000000000000..d418190bb8fc8b --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-01-21-21-33-48.bpo-46463.fBbdTG.rst @@ -0,0 +1,2 @@ +Fixes :file:`escape4chm.py` script used when building the CHM documentation +file From 46e6aad12958d3b73c5377ec034d056bb1a36d65 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 21 Jan 2022 14:37:52 -0800 Subject: [PATCH 031/220] bpo-46445: Cover multiple inheritance of `TypedDict` in `test_typing` (GH-30719) (cherry picked from commit 65b88d5e01c845c0cfa3ff61bc8b2faec8f67a57) Co-authored-by: Nikita Sobolev --- Lib/test/test_typing.py | 88 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 1d16e78d422cd0..d6c55ef1de75fe 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -19,7 +19,6 @@ from typing import is_typeddict from typing import no_type_check, no_type_check_decorator from typing import Type -from typing import NewType from typing import NamedTuple, TypedDict from typing import IO, TextIO, BinaryIO from typing import Pattern, Match @@ -4302,6 +4301,93 @@ class Cat(Animal): 'voice': str, } + def test_multiple_inheritance(self): + class One(TypedDict): + one: int + class Two(TypedDict): + two: str + class Untotal(TypedDict, total=False): + untotal: str + Inline = TypedDict('Inline', {'inline': bool}) + class Regular: + pass + + class Child(One, Two): + child: bool + self.assertEqual( + Child.__required_keys__, + frozenset(['one', 'two', 'child']), + ) + self.assertEqual( + Child.__optional_keys__, + frozenset([]), + ) + self.assertEqual( + Child.__annotations__, + {'one': int, 'two': str, 'child': bool}, + ) + + class ChildWithOptional(One, Untotal): + child: bool + self.assertEqual( + ChildWithOptional.__required_keys__, + frozenset(['one', 'child']), + ) + self.assertEqual( + ChildWithOptional.__optional_keys__, + frozenset(['untotal']), + ) + self.assertEqual( + ChildWithOptional.__annotations__, + {'one': int, 'untotal': str, 'child': bool}, + ) + + class ChildWithTotalFalse(One, Untotal, total=False): + child: bool + self.assertEqual( + ChildWithTotalFalse.__required_keys__, + frozenset(['one']), + ) + self.assertEqual( + ChildWithTotalFalse.__optional_keys__, + frozenset(['untotal', 'child']), + ) + self.assertEqual( + ChildWithTotalFalse.__annotations__, + {'one': int, 'untotal': str, 'child': bool}, + ) + + class ChildWithInlineAndOptional(Untotal, Inline): + child: bool + self.assertEqual( + ChildWithInlineAndOptional.__required_keys__, + frozenset(['inline', 'child']), + ) + self.assertEqual( + ChildWithInlineAndOptional.__optional_keys__, + frozenset(['untotal']), + ) + self.assertEqual( + ChildWithInlineAndOptional.__annotations__, + {'inline': bool, 'untotal': str, 'child': bool}, + ) + + wrong_bases = [ + (One, Regular), + (Regular, One), + (One, Two, Regular), + (Inline, Regular), + (Untotal, Regular), + ] + for bases in wrong_bases: + with self.subTest(bases=bases): + with self.assertRaisesRegex( + TypeError, + 'cannot inherit from both a TypedDict type and a non-TypedDict', + ): + class Wrong(*bases): + pass + def test_is_typeddict(self): assert is_typeddict(Point2D) is True assert is_typeddict(Union[str, int]) is False From d4a9e34401d519250d3b3744cd10394069f748c1 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 21 Jan 2022 23:34:29 -0800 Subject: [PATCH 032/220] bpo-46442: improve and rename testExceptionCleanupNames (GH-30758) The test tested that explicitly deleting the local variable bound to the exception did not cause problems, but it did not test what it actually claimed to test, i.e. that the variable is deleted automatically. (cherry picked from commit 82c53229e18f5853c82cb8ab6b9af1925a0e9e58) Co-authored-by: Yellow Dusk --- Lib/test/test_exceptions.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index b3d1c35274c719..802dc9a67eb211 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -648,15 +648,27 @@ def test_str(self): self.assertTrue(str(Exception('a'))) self.assertTrue(str(Exception('a', 'b'))) - def testExceptionCleanupNames(self): + def test_exception_cleanup_names(self): # Make sure the local variable bound to the exception instance by # an "except" statement is only visible inside the except block. try: raise Exception() except Exception as e: - self.assertTrue(e) + self.assertIsInstance(e, Exception) + self.assertNotIn('e', locals()) + with self.assertRaises(UnboundLocalError): + e + + def test_exception_cleanup_names2(self): + # Make sure the cleanup doesn't break if the variable is explicitly deleted. + try: + raise Exception() + except Exception as e: + self.assertIsInstance(e, Exception) del e self.assertNotIn('e', locals()) + with self.assertRaises(UnboundLocalError): + e def testExceptionCleanupState(self): # Make sure exception state is cleaned up as soon as the except From 90e2998db78cd15e45b3c82f6360ac8841e03945 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 22 Jan 2022 04:28:51 -0800 Subject: [PATCH 033/220] [3.10] bpo-46469: Make asyncio generic classes return GenericAlias (GH-30777) (#30784) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * bpo-46469: Make asyncio generic classes return GenericAlias (GH-30777) * bpo-46469: Make asyncio generic classes return GenericAlias * 📜🤖 Added by blurb_it. * Update Misc/NEWS.d/next/Library/2022-01-22-05-05-08.bpo-46469.plUab5.rst Co-authored-by: Jelle Zijlstra Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Jelle Zijlstra (cherry picked from commit ea5b96842e066623a53015d8b2492ed61a5baf96) Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> * Fix tests Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Co-authored-by: Andrew Svetlov --- Lib/asyncio/futures.py | 4 ++-- Lib/asyncio/queues.py | 4 ++-- Lib/asyncio/tasks.py | 4 ++-- Lib/test/test_asyncio/test_futures.py | 7 ++++++- Lib/test/test_asyncio/test_queues.py | 8 ++++++-- Lib/test/test_asyncio/test_tasks.py | 7 +++++++ .../2022-01-22-05-05-08.bpo-46469.plUab5.rst | 1 + Modules/_asynciomodule.c | 18 ++---------------- 8 files changed, 28 insertions(+), 25 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-01-22-05-05-08.bpo-46469.plUab5.rst diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 10f8f0554e4b60..8e8cd87612588c 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -8,6 +8,7 @@ import contextvars import logging import sys +from types import GenericAlias from . import base_futures from . import events @@ -106,8 +107,7 @@ def __del__(self): context['source_traceback'] = self._source_traceback self._loop.call_exception_handler(context) - def __class_getitem__(cls, type): - return cls + __class_getitem__ = classmethod(GenericAlias) @property def _log_traceback(self): diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index a87ec8b2158767..10dd689bbb2efa 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -2,6 +2,7 @@ import collections import heapq +from types import GenericAlias from . import locks from . import mixins @@ -69,8 +70,7 @@ def __repr__(self): def __str__(self): return f'<{type(self).__name__} {self._format()}>' - def __class_getitem__(cls, type): - return cls + __class_getitem__ = classmethod(GenericAlias) def _format(self): result = f'maxsize={self._maxsize!r}' diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 53eef84427be10..445a9f51226ed7 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -17,6 +17,7 @@ import types import warnings import weakref +from types import GenericAlias from . import base_tasks from . import coroutines @@ -123,8 +124,7 @@ def __del__(self): self._loop.call_exception_handler(context) super().__del__() - def __class_getitem__(cls, type): - return cls + __class_getitem__ = classmethod(GenericAlias) def _repr_info(self): return base_tasks._task_repr_info(self) diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 0c379e0fb0f956..838147b1e65f4e 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -7,7 +7,7 @@ import threading import unittest from unittest import mock - +from types import GenericAlias import asyncio from asyncio import futures from test.test_asyncio import utils as test_utils @@ -109,6 +109,11 @@ def setUp(self): self.loop = self.new_test_loop() self.addCleanup(self.loop.close) + def test_generic_alias(self): + future = self.cls[str] + self.assertEqual(future.__args__, (str,)) + self.assertIsInstance(future, GenericAlias) + def test_isfuture(self): class MyFuture: _asyncio_future_blocking = None diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 63a9a5f270cc92..b1a53b859c5ccb 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -1,9 +1,8 @@ """Tests for queues.py""" import unittest -from unittest import mock - import asyncio +from types import GenericAlias from test.test_asyncio import utils as test_utils @@ -74,6 +73,11 @@ def test_repr(self): def test_str(self): self._test_repr_or_str(str, False) + def test_generic_alias(self): + q = asyncio.Queue[int] + self.assertEqual(q.__args__, (int,)) + self.assertIsInstance(q, GenericAlias) + def test_empty(self): q = asyncio.Queue() self.assertTrue(q.empty()) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 1c05944c42d255..4782c92a7c1550 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -15,6 +15,7 @@ import unittest import weakref from unittest import mock +from types import GenericAlias import asyncio from asyncio import coroutines @@ -120,6 +121,12 @@ def setUp(self): self.loop.set_task_factory(self.new_task) self.loop.create_future = lambda: self.new_future(self.loop) + + def test_generic_alias(self): + task = self.__class__.Task[str] + self.assertEqual(task.__args__, (str,)) + self.assertIsInstance(task, GenericAlias) + def test_task_cancel_message_getter(self): async def coro(): pass diff --git a/Misc/NEWS.d/next/Library/2022-01-22-05-05-08.bpo-46469.plUab5.rst b/Misc/NEWS.d/next/Library/2022-01-22-05-05-08.bpo-46469.plUab5.rst new file mode 100644 index 00000000000000..0d0e4b5d3d7358 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-01-22-05-05-08.bpo-46469.plUab5.rst @@ -0,0 +1 @@ +:mod:`asyncio` generic classes now return :class:`types.GenericAlias` in ``__class_getitem__`` instead of the same class. \ No newline at end of file diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index befec9a8342ef4..392e0e7c7357f8 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1476,13 +1476,6 @@ FutureObj_finalize(FutureObj *fut) PyErr_Restore(error_type, error_value, error_traceback); } -static PyObject * -future_cls_getitem(PyObject *cls, PyObject *type) -{ - Py_INCREF(cls); - return cls; -} - static PyAsyncMethods FutureType_as_async = { (unaryfunc)future_new_iter, /* am_await */ 0, /* am_aiter */ @@ -1503,7 +1496,7 @@ static PyMethodDef FutureType_methods[] = { _ASYNCIO_FUTURE_GET_LOOP_METHODDEF _ASYNCIO_FUTURE__MAKE_CANCELLED_ERROR_METHODDEF _ASYNCIO_FUTURE__REPR_INFO_METHODDEF - {"__class_getitem__", future_cls_getitem, METH_O|METH_CLASS, NULL}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* Sentinel */ }; @@ -2445,13 +2438,6 @@ TaskObj_finalize(TaskObj *task) FutureObj_finalize((FutureObj*)task); } -static PyObject * -task_cls_getitem(PyObject *cls, PyObject *type) -{ - Py_INCREF(cls); - return cls; -} - static void TaskObj_dealloc(PyObject *); /* Needs Task_CheckExact */ static PyMethodDef TaskType_methods[] = { @@ -2471,7 +2457,7 @@ static PyMethodDef TaskType_methods[] = { _ASYNCIO_TASK_GET_NAME_METHODDEF _ASYNCIO_TASK_SET_NAME_METHODDEF _ASYNCIO_TASK_GET_CORO_METHODDEF - {"__class_getitem__", task_cls_getitem, METH_O|METH_CLASS, NULL}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* Sentinel */ }; From 6111d5dee2b24916ff95dba56efc569396a31851 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 22 Jan 2022 04:29:51 -0800 Subject: [PATCH 034/220] bpo-46425: fix direct invocation of `asyncio` tests (GH-30725) (#30782) (cherry picked from commit 5a5340044ca98cbe6297668d91bccba04b102923) Co-authored-by: Nikita Sobolev Co-authored-by: Nikita Sobolev --- Lib/test/test_asyncio/test_context.py | 4 ++++ Lib/test/test_asyncio/test_futures2.py | 4 ++++ Lib/test/test_asyncio/test_protocols.py | 4 ++++ Lib/test/test_asyncio/test_runners.py | 6 +++++- Lib/test/test_asyncio/test_sendfile.py | 4 ++++ Lib/test/test_asyncio/test_sock_lowlevel.py | 5 ++++- 6 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_asyncio/test_context.py b/Lib/test/test_asyncio/test_context.py index 63b1eb320ce16b..6b80721873d95c 100644 --- a/Lib/test/test_asyncio/test_context.py +++ b/Lib/test/test_asyncio/test_context.py @@ -32,3 +32,7 @@ async def main(): self.assertEqual(str(r2[0]), '0.333333') self.assertEqual(str(r2[1]), '0.111111') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index 13dbc703277c81..57d24190bc0bd5 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -16,3 +16,7 @@ async def func(): # The check for returned string is not very reliable but # exact comparison for the whole string is even weaker. self.assertIn('...', repr(await asyncio.wait_for(func(), timeout=10))) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_protocols.py b/Lib/test/test_asyncio/test_protocols.py index 438111cccd3478..d8cde6d89aadcd 100644 --- a/Lib/test/test_asyncio/test_protocols.py +++ b/Lib/test/test_asyncio/test_protocols.py @@ -55,3 +55,7 @@ def test_subprocess_protocol(self): self.assertIsNone(sp.pipe_connection_lost(1, f)) self.assertIsNone(sp.process_exited()) self.assertFalse(hasattr(sp, '__dict__')) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index b9ae02dc3c04e0..5c06a1aaa830fa 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -2,7 +2,7 @@ import unittest from unittest import mock -from . import utils as test_utils +from test.test_asyncio import utils as test_utils class TestPolicy(asyncio.AbstractEventLoopPolicy): @@ -180,3 +180,7 @@ async def main(): self.assertIsNone(spinner.ag_frame) self.assertFalse(spinner.ag_running) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index 0a5466a0af152b..57b56bba34100b 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -565,3 +565,7 @@ class SelectEventLoopTests(SendfileTestsBase, def create_event_loop(self): return asyncio.SelectorEventLoop(selectors.SelectSelector()) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_sock_lowlevel.py b/Lib/test/test_asyncio/test_sock_lowlevel.py index ab022a357d205e..448d835b04d570 100644 --- a/Lib/test/test_asyncio/test_sock_lowlevel.py +++ b/Lib/test/test_asyncio/test_sock_lowlevel.py @@ -1,5 +1,4 @@ import socket -import time import asyncio import sys import unittest @@ -512,3 +511,7 @@ class SelectEventLoopTests(BaseSockTestsMixin, def create_event_loop(self): return asyncio.SelectorEventLoop(selectors.SelectSelector()) + + +if __name__ == '__main__': + unittest.main() From acda9f3b90c33e4020237cb9e5c676efb38f7847 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 22 Jan 2022 06:28:36 -0800 Subject: [PATCH 035/220] bpo-46417: Fix race condition on setting type __bases__ (GH-30788) (GH-30789) Fix a race condition on setting a type __bases__ attribute: the internal function add_subclass() now gets the PyTypeObject.tp_subclasses member after calling PyWeakref_NewRef() which can trigger a garbage collection which can indirectly modify PyTypeObject.tp_subclasses. (cherry picked from commit f1c6ae3270913e095d24ae13ecf96f5a32c8c503) Co-authored-by: Victor Stinner Co-authored-by: Victor Stinner --- .../2022-01-22-14-39-23.bpo-46417.3U5SfN.rst | 5 ++++ Objects/typeobject.c | 27 +++++++++++-------- 2 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-01-22-14-39-23.bpo-46417.3U5SfN.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-01-22-14-39-23.bpo-46417.3U5SfN.rst b/Misc/NEWS.d/next/Core and Builtins/2022-01-22-14-39-23.bpo-46417.3U5SfN.rst new file mode 100644 index 00000000000000..54fe09b7ba4544 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-01-22-14-39-23.bpo-46417.3U5SfN.rst @@ -0,0 +1,5 @@ +Fix a race condition on setting a type ``__bases__`` attribute: the internal +function ``add_subclass()`` now gets the ``PyTypeObject.tp_subclasses`` +member after calling :c:func:`PyWeakref_NewRef` which can trigger a garbage +collection which can indirectly modify ``PyTypeObject.tp_subclasses``. Patch +by Victor Stinner. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b23e36a420fa0c..10b69fe1196dbe 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6384,24 +6384,29 @@ PyType_Ready(PyTypeObject *type) static int add_subclass(PyTypeObject *base, PyTypeObject *type) { - int result = -1; - PyObject *dict, *key, *newobj; + PyObject *key = PyLong_FromVoidPtr((void *) type); + if (key == NULL) + return -1; - dict = base->tp_subclasses; + PyObject *ref = PyWeakref_NewRef((PyObject *)type, NULL); + if (ref == NULL) { + Py_DECREF(key); + return -1; + } + + // Only get tp_subclasses after creating the key and value. + // PyWeakref_NewRef() can trigger a garbage collection which can execute + // arbitrary Python code and so modify base->tp_subclasses. + PyObject *dict = base->tp_subclasses; if (dict == NULL) { base->tp_subclasses = dict = PyDict_New(); if (dict == NULL) return -1; } assert(PyDict_CheckExact(dict)); - key = PyLong_FromVoidPtr((void *) type); - if (key == NULL) - return -1; - newobj = PyWeakref_NewRef((PyObject *)type, NULL); - if (newobj != NULL) { - result = PyDict_SetItem(dict, key, newobj); - Py_DECREF(newobj); - } + + int result = PyDict_SetItem(dict, key, ref); + Py_DECREF(ref); Py_DECREF(key); return result; } From 923c994400b3f1c67f95d25c703e131890a16912 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 22 Jan 2022 07:31:52 -0800 Subject: [PATCH 036/220] bpo-46468: document that "-m http.server" defaults to port 8000 (GH-30776) (#30787) Code link: https://github.com/python/cpython/blob/70c16468deee9390e34322d32fda57df6e0f46bb/Lib/http/server.pyGH-L1270 It's been this way since at least 3.4. Also improved some wording in the same section. (cherry picked from commit c8a536624e8f5d6612e3c275c5b19592583a8cf8) Co-authored-by: Jelle Zijlstra Co-authored-by: Jelle Zijlstra --- Doc/library/http.server.rst | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index c3cee079526b2e..0de02834401aa8 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -412,17 +412,22 @@ the current directory:: .. _http-server-cli: :mod:`http.server` can also be invoked directly using the :option:`-m` -switch of the interpreter with a ``port number`` argument. Similar to +switch of the interpreter. Similar to the previous example, this serves files relative to the current directory:: - python -m http.server 8000 + python -m http.server -By default, server binds itself to all interfaces. The option ``-b/--bind`` +The server listens to port 8000 by default. The default can be overridden +by passing the desired port number as an argument:: + + python -m http.server 9000 + +By default, the server binds itself to all interfaces. The option ``-b/--bind`` specifies a specific address to which it should bind. Both IPv4 and IPv6 addresses are supported. For example, the following command causes the server to bind to localhost only:: - python -m http.server 8000 --bind 127.0.0.1 + python -m http.server --bind 127.0.0.1 .. versionadded:: 3.4 ``--bind`` argument was introduced. @@ -430,14 +435,14 @@ to bind to localhost only:: .. versionadded:: 3.8 ``--bind`` argument enhanced to support IPv6 -By default, server uses the current directory. The option ``-d/--directory`` +By default, the server uses the current directory. The option ``-d/--directory`` specifies a directory to which it should serve the files. For example, the following command uses a specific directory:: python -m http.server --directory /tmp/ .. versionadded:: 3.7 - ``--directory`` specify alternate directory + ``--directory`` argument was introduced. .. class:: CGIHTTPRequestHandler(request, client_address, server) @@ -482,4 +487,4 @@ the following command uses a specific directory:: :class:`CGIHTTPRequestHandler` can be enabled in the command line by passing the ``--cgi`` option:: - python -m http.server --cgi 8000 + python -m http.server --cgi From 83aef4d34022f293336f606dba8598cc7ac8f9f2 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 22 Jan 2022 09:28:48 -0800 Subject: [PATCH 037/220] bpo-43118: Fix bug in inspect.signature around 'base.__text_signature__' (GH-30285) (#30765) (cherry picked from commit 881a763cfe07ef4a5806ec78f13a9bc99e8909dc) Co-authored-by: Weipeng Hong Co-authored-by: Weipeng Hong --- Lib/inspect.py | 4 ++-- Lib/test/ann_module7.py | 11 +++++++++++ Lib/test/test_inspect.py | 11 +++++++++++ .../Library/2021-12-29-14-42-09.bpo-43118.BoVi_5.rst | 3 +++ 4 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 Lib/test/ann_module7.py create mode 100644 Misc/NEWS.d/next/Library/2021-12-29-14-42-09.bpo-43118.BoVi_5.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 6d43d8dad46b99..c5881cc808d21a 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2511,9 +2511,9 @@ def _signature_from_callable(obj, *, pass else: if text_sig: - # If 'obj' class has a __text_signature__ attribute: + # If 'base' class has a __text_signature__ attribute: # return a signature based on it - return _signature_fromstr(sigcls, obj, text_sig) + return _signature_fromstr(sigcls, base, text_sig) # No '__text_signature__' was found for the 'obj' class. # Last option is to check if its '__init__' is diff --git a/Lib/test/ann_module7.py b/Lib/test/ann_module7.py new file mode 100644 index 00000000000000..8f890cd28025be --- /dev/null +++ b/Lib/test/ann_module7.py @@ -0,0 +1,11 @@ +# Tests class have ``__text_signature__`` + +from __future__ import annotations + +DEFAULT_BUFFER_SIZE = 8192 + +class BufferedReader(object): + """BufferedReader(raw, buffer_size=DEFAULT_BUFFER_SIZE)\n--\n\n + Create a new buffered reader using the given readable raw IO object. + """ + pass diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 545dab5c6348f7..28e4f5b4a718a4 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -4218,6 +4218,17 @@ def func(*args, **kwargs): sig = inspect.signature(func) self.assertEqual(str(sig), '(self, a, b=1, /, *args, c, d=2, **kwargs)') + def test_base_class_have_text_signature(self): + # see issue 43118 + from test.ann_module7 import BufferedReader + class MyBufferedReader(BufferedReader): + """buffer reader class.""" + + text_signature = BufferedReader.__text_signature__ + self.assertEqual(text_signature, '(raw, buffer_size=DEFAULT_BUFFER_SIZE)') + sig = inspect.signature(MyBufferedReader) + self.assertEqual(str(sig), '(raw, buffer_size=8192)') + class NTimesUnwrappable: def __init__(self, n): diff --git a/Misc/NEWS.d/next/Library/2021-12-29-14-42-09.bpo-43118.BoVi_5.rst b/Misc/NEWS.d/next/Library/2021-12-29-14-42-09.bpo-43118.BoVi_5.rst new file mode 100644 index 00000000000000..a37c22cd78c098 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-12-29-14-42-09.bpo-43118.BoVi_5.rst @@ -0,0 +1,3 @@ +Fix a bug in :func:`inspect.signature` that was causing it to fail on some +subclasses of classes with a ``__text_signature__`` referencing module +globals. Patch by Weipeng Hong. From 486b4d3d8e1a5699a2854e310c58fe12b220b7a9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 22 Jan 2022 16:58:16 -0800 Subject: [PATCH 038/220] bpo-41682: Skip unstable test_asyncio sendfile test on Windows (GH-30801) (GH-30812) (cherry picked from commit 1ded8ed8e817b8f9dae1a0ef92d97983afbc844e) Co-authored-by: Nikita Sobolev Co-authored-by: Nikita Sobolev --- Lib/test/test_asyncio/test_sendfile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index 57b56bba34100b..c8bfa892c73fc6 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -456,6 +456,8 @@ def test_sendfile_ssl_close_peer_after_receiving(self): # themselves). @unittest.skipIf(sys.platform.startswith('sunos'), "Doesn't work on Solaris") + @unittest.skipIf(sys.platform == "win32", + "It is flaky on Windows and needs to be fixed") # TODO: bpo-41682 def test_sendfile_close_peer_in_the_middle_of_receiving(self): srv_proto, cli_proto = self.prepare_sendfile(close_after=1024) with self.assertRaises(ConnectionError): From f66ef3eab62c6d262ddbc8ab16fb43c8921ad33a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 22 Jan 2022 17:00:11 -0800 Subject: [PATCH 039/220] bpo-46266: Add calendar day of week constants to __all__ (GH-30412) (GH-30424) (cherry picked from commit e5894ca8fd05e6a6df1033025b9093b68baa718d) Co-authored-by: Nikita Sobolev Co-authored-by: Nikita Sobolev --- Doc/library/calendar.rst | 11 ++++++++++- Lib/calendar.py | 4 +++- Lib/test/test_calendar.py | 3 +-- .../Library/2022-01-05-12-48-18.bpo-46266.ACQCgX.rst | 4 ++++ 4 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-01-05-12-48-18.bpo-46266.ACQCgX.rst diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index c3c04db853ed2d..f641760d1bd1a9 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -31,7 +31,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is .. class:: Calendar(firstweekday=0) Creates a :class:`Calendar` object. *firstweekday* is an integer specifying the - first day of the week. ``0`` is Monday (the default), ``6`` is Sunday. + first day of the week. :const:`MONDAY` is ``0`` (the default), :const:`SUNDAY` is ``6``. A :class:`Calendar` object provides several methods that can be used for preparing the calendar data for formatting. This class doesn't do any formatting @@ -409,6 +409,15 @@ The :mod:`calendar` module exports the following data attributes: locale. This follows normal convention of January being month number 1, so it has a length of 13 and ``month_abbr[0]`` is the empty string. +.. data:: MONDAY + TUESDAY + WEDNESDAY + THURSDAY + FRIDAY + SATURDAY + SUNDAY + + Aliases for day numbers, where ``MONDAY`` is ``0`` and ``SUNDAY`` is ``6``. .. seealso:: diff --git a/Lib/calendar.py b/Lib/calendar.py index 7311a0173729ec..cbea9ec99f550f 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -15,7 +15,9 @@ "monthcalendar", "prmonth", "month", "prcal", "calendar", "timegm", "month_name", "month_abbr", "day_name", "day_abbr", "Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar", - "LocaleHTMLCalendar", "weekheader"] + "LocaleHTMLCalendar", "weekheader", + "MONDAY", "TUESDAY", "WEDNESDAY", "THURSDAY", "FRIDAY", + "SATURDAY", "SUNDAY"] # Exception raised for bad input (with string parameter for details) error = ValueError diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index c641e8c418318d..39094ad6fd9abd 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -935,8 +935,7 @@ def test_html_output_year_css(self): class MiscTestCase(unittest.TestCase): def test__all__(self): not_exported = { - 'mdays', 'January', 'February', 'EPOCH', 'MONDAY', 'TUESDAY', - 'WEDNESDAY', 'THURSDAY', 'FRIDAY', 'SATURDAY', 'SUNDAY', + 'mdays', 'January', 'February', 'EPOCH', 'different_locale', 'c', 'prweek', 'week', 'format', 'formatstring', 'main', 'monthlen', 'prevmonth', 'nextmonth'} support.check__all__(self, calendar, not_exported=not_exported) diff --git a/Misc/NEWS.d/next/Library/2022-01-05-12-48-18.bpo-46266.ACQCgX.rst b/Misc/NEWS.d/next/Library/2022-01-05-12-48-18.bpo-46266.ACQCgX.rst new file mode 100644 index 00000000000000..354dcb0106595c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-01-05-12-48-18.bpo-46266.ACQCgX.rst @@ -0,0 +1,4 @@ +Improve day constants in :mod:`calendar`. + +Now all constants (`MONDAY` ... `SUNDAY`) are documented, tested, and added +to ``__all__``. From 633db1c4eb863a1340e45c353e36f2f8dcf5945c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 23 Jan 2022 03:10:37 +0000 Subject: [PATCH 040/220] [3.10] bpo-46240: Correct the error for unclosed parentheses when the tokenizer is not finished (GH-30378). (GH-30819) (cherry picked from commit 70f415fb8b632247e28d87998642317ca7a652ae) Co-authored-by: Pablo Galindo Salgado --- Lib/test/test_exceptions.py | 2 +- Lib/test/test_syntax.py | 3 +++ .../Core and Builtins/2022-01-03-23-31-25.bpo-46240.8lGjeK.rst | 3 +++ Parser/pegen.c | 3 ++- 4 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-01-03-23-31-25.bpo-46240.8lGjeK.rst diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 802dc9a67eb211..606e6852627c41 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -227,7 +227,7 @@ def testSyntaxErrorOffset(self): check('x = "a', 1, 5) check('lambda x: x = 2', 1, 1) check('f{a + b + c}', 1, 2) - check('[file for str(file) in []\n])', 1, 11) + check('[file for str(file) in []\n]', 1, 11) check('a = « hello » « world »', 1, 5) check('[\nfile\nfor str(file)\nin\n[]\n]', 3, 5) check('[file for\n str(file) in []]', 2, 2) diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index b5bebb3d0bdfa1..7aa93a012e1133 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1513,6 +1513,9 @@ def test_error_parenthesis(self): for paren in "([{": self._check_error(paren + "1 + 2", f"\\{paren}' was never closed") + for paren in "([{": + self._check_error(f"a = {paren} 1, 2, 3\nb=3", f"\\{paren}' was never closed") + for paren in ")]}": self._check_error(paren + "1 + 2", f"unmatched '\\{paren}'") diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-01-03-23-31-25.bpo-46240.8lGjeK.rst b/Misc/NEWS.d/next/Core and Builtins/2022-01-03-23-31-25.bpo-46240.8lGjeK.rst new file mode 100644 index 00000000000000..a7702ebafbd468 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-01-03-23-31-25.bpo-46240.8lGjeK.rst @@ -0,0 +1,3 @@ +Correct the error message for unclosed parentheses when the tokenizer +doesn't reach the end of the source when the error is reported. Patch by +Pablo Galindo diff --git a/Parser/pegen.c b/Parser/pegen.c index f9812c0ea8f02b..26143f57c09248 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1342,7 +1342,8 @@ _PyPegen_run_parser(Parser *p) if (PyErr_Occurred()) { // Prioritize tokenizer errors to custom syntax errors raised // on the second phase only if the errors come from the parser. - if (p->tok->done == E_DONE && PyErr_ExceptionMatches(PyExc_SyntaxError)) { + int is_tok_ok = (p->tok->done == E_DONE || p->tok->done == E_OK); + if (is_tok_ok && PyErr_ExceptionMatches(PyExc_SyntaxError)) { _PyPegen_check_tokenizer_errors(p); } return NULL; From a7a4ca4f06c8c31d7f403113702ad2e80bfc326b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 23 Jan 2022 10:17:27 -0500 Subject: [PATCH 041/220] [3.10] bpo-46474: Avoid REDoS in EntryPoint.pattern (sync with importlib_metadata 4.10.1) (GH-30803) (GH-30827) (cherry picked from commit 51c3e28c8a163e58dc753765e3cc51d5a717e70d) Co-authored-by: Jason R. Coombs --- Lib/importlib/metadata/__init__.py | 4 ++-- .../next/Library/2022-01-22-14-49-10.bpo-46474.eKQhvx.rst | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-01-22-14-49-10.bpo-46474.eKQhvx.rst diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index ec41ed39157a93..33ce1b6b569640 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -132,8 +132,8 @@ class EntryPoint( pattern = re.compile( r'(?P[\w.]+)\s*' - r'(:\s*(?P[\w.]+))?\s*' - r'(?P\[.*\])?\s*$' + r'(:\s*(?P[\w.]+)\s*)?' + r'((?P\[.*\])\s*)?$' ) """ A regular expression describing the syntax for an entry point, diff --git a/Misc/NEWS.d/next/Library/2022-01-22-14-49-10.bpo-46474.eKQhvx.rst b/Misc/NEWS.d/next/Library/2022-01-22-14-49-10.bpo-46474.eKQhvx.rst new file mode 100644 index 00000000000000..156b7de4f67877 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-01-22-14-49-10.bpo-46474.eKQhvx.rst @@ -0,0 +1,2 @@ +In ``importlib.metadata.EntryPoint.pattern``, avoid potential REDoS by +limiting ambiguity in consecutive whitespace. From e3ade66ec575e0cb4882cfdff155ef962e67c837 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Sun, 23 Jan 2022 19:34:43 +0000 Subject: [PATCH 042/220] bpo-41403: Improve error message for invalid mock target (GH-30833) (GH-30834) (cherry picked from commit f7955a82e36d4c32ebdd7b7707cdf0e6ffa7a418) --- Lib/unittest/mock.py | 6 +++--- Lib/unittest/test/testmock/testpatch.py | 9 +++++++-- .../Library/2022-01-23-18-04-45.bpo-41403.SgoHqV.rst | 3 +++ 3 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-01-23-18-04-45.bpo-41403.SgoHqV.rst diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 6226bd4bc0c195..7152f86ed96945 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -1602,9 +1602,9 @@ def stop(self): def _get_target(target): try: target, attribute = target.rsplit('.', 1) - except (TypeError, ValueError): - raise TypeError("Need a valid target to patch. You supplied: %r" % - (target,)) + except (TypeError, ValueError, AttributeError): + raise TypeError( + f"Need a valid target to patch. You supplied: {target!r}") getter = lambda: _importer(target) return getter, attribute diff --git a/Lib/unittest/test/testmock/testpatch.py b/Lib/unittest/test/testmock/testpatch.py index 233a5afffaed4a..8ab63a1317d3df 100644 --- a/Lib/unittest/test/testmock/testpatch.py +++ b/Lib/unittest/test/testmock/testpatch.py @@ -1933,8 +1933,13 @@ def test(mock): def test_invalid_target(self): - with self.assertRaises(TypeError): - patch('') + class Foo: + pass + + for target in ['', 12, Foo()]: + with self.subTest(target=target): + with self.assertRaises(TypeError): + patch(target) def test_cant_set_kwargs_when_passing_a_mock(self): diff --git a/Misc/NEWS.d/next/Library/2022-01-23-18-04-45.bpo-41403.SgoHqV.rst b/Misc/NEWS.d/next/Library/2022-01-23-18-04-45.bpo-41403.SgoHqV.rst new file mode 100644 index 00000000000000..ede159b25641f3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-01-23-18-04-45.bpo-41403.SgoHqV.rst @@ -0,0 +1,3 @@ +Make :meth:`mock.patch` raise a :exc:`TypeError` with a relevant error +message on invalid arg. Previously it allowed a cryptic +:exc:`AttributeError` to escape. From b2c7fe1f61c8ec3742635428570bc61d820c7a68 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 23 Jan 2022 14:02:31 -0800 Subject: [PATCH 043/220] Improve grouper() recipe to demonstrate all forms of zip() (GH-30837) (GH-30840) --- Doc/library/itertools.rst | 15 ++++++++++++--- Lib/test/test_itertools.py | 35 +++++++++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 61d8b869711fa4..34667561c3cfe7 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -813,11 +813,20 @@ which incur interpreter overhead. return starmap(func, repeat(args)) return starmap(func, repeat(args, times)) - def grouper(iterable, n, fillvalue=None): + def grouper(iterable, n, *, incomplete='fill', fillvalue=None): "Collect data into non-overlapping fixed-length chunks or blocks" - # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx + # grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx + # grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError + # grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF args = [iter(iterable)] * n - return zip_longest(*args, fillvalue=fillvalue) + if incomplete == 'fill': + return zip_longest(*args, fillvalue=fillvalue) + if incomplete == 'strict': + return zip(*args, strict=True) + if incomplete == 'ignore': + return zip(*args) + else: + raise ValueError('Expected fill, strict, or ignore') def triplewise(iterable): "Return overlapping triplets from an iterable" diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index a12f6f0b9773e4..4c9c597cc4842d 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -2394,6 +2394,21 @@ def test_permutations_sizeof(self): ... else: ... return starmap(func, repeat(args, times)) +>>> def grouper(iterable, n, *, incomplete='fill', fillvalue=None): +... "Collect data into non-overlapping fixed-length chunks or blocks" +... # grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx +... # grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError +... # grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF +... args = [iter(iterable)] * n +... if incomplete == 'fill': +... return zip_longest(*args, fillvalue=fillvalue) +... if incomplete == 'strict': +... return zip(*args, strict=True) +... if incomplete == 'ignore': +... return zip(*args) +... else: +... raise ValueError('Expected fill, strict, or ignore') + >>> def triplewise(iterable): ... "Return overlapping triplets from an iterable" ... # pairwise('ABCDEFG') -> ABC BCD CDE DEF EFG @@ -2411,11 +2426,6 @@ def test_permutations_sizeof(self): ... window.append(x) ... yield tuple(window) ->>> def grouper(n, iterable, fillvalue=None): -... "grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx" -... args = [iter(iterable)] * n -... return zip_longest(*args, fillvalue=fillvalue) - >>> def roundrobin(*iterables): ... "roundrobin('ABC', 'D', 'EF') --> A D E B F C" ... # Recipe credited to George Sakkis @@ -2584,9 +2594,22 @@ def test_permutations_sizeof(self): >>> dotproduct([1,2,3], [4,5,6]) 32 ->>> list(grouper(3, 'abcdefg', 'x')) +>>> list(grouper('abcdefg', 3, fillvalue='x')) [('a', 'b', 'c'), ('d', 'e', 'f'), ('g', 'x', 'x')] +>>> it = grouper('abcdefg', 3, incomplete='strict') +>>> next(it) +('a', 'b', 'c') +>>> next(it) +('d', 'e', 'f') +>>> next(it) +Traceback (most recent call last): + ... +ValueError: zip() argument 2 is shorter than argument 1 + +>>> list(grouper('abcdefg', n=3, incomplete='ignore')) +[('a', 'b', 'c'), ('d', 'e', 'f')] + >>> list(triplewise('ABCDEFG')) [('A', 'B', 'C'), ('B', 'C', 'D'), ('C', 'D', 'E'), ('D', 'E', 'F'), ('E', 'F', 'G')] From eaeb99468045b863d2dd3da3e3d1c3c9c78e1254 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 24 Jan 2022 19:18:38 +0300 Subject: [PATCH 044/220] [3.10] bpo-46416: Allow direct invocation of `Lib/test/test_typing.py` (GH-30641) (GH-30697) Use `__name__` (cherry picked from commit 2792d6d18eab3efeb71e6397f88db86e610541f1) Co-authored-by: Nikita Sobolev --- Lib/test/test_typing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index d6c55ef1de75fe..acad35d18d5f35 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -5067,7 +5067,7 @@ def test_special_attrs2(self): ) self.assertEqual( SpecialAttrsTests.TypeName.__module__, - 'test.test_typing', + __name__, ) # NewTypes are picklable assuming correct qualname information. for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -5081,7 +5081,7 @@ def test_special_attrs2(self): # __qualname__ is unnecessary. self.assertEqual(SpecialAttrsT.__name__, 'SpecialAttrsT') self.assertFalse(hasattr(SpecialAttrsT, '__qualname__')) - self.assertEqual(SpecialAttrsT.__module__, 'test.test_typing') + self.assertEqual(SpecialAttrsT.__module__, __name__) # Module-level type variables are picklable. for proto in range(pickle.HIGHEST_PROTOCOL + 1): s = pickle.dumps(SpecialAttrsT, proto) @@ -5090,7 +5090,7 @@ def test_special_attrs2(self): self.assertEqual(SpecialAttrsP.__name__, 'SpecialAttrsP') self.assertFalse(hasattr(SpecialAttrsP, '__qualname__')) - self.assertEqual(SpecialAttrsP.__module__, 'test.test_typing') + self.assertEqual(SpecialAttrsP.__module__, __name__) # Module-level ParamSpecs are picklable. for proto in range(pickle.HIGHEST_PROTOCOL + 1): s = pickle.dumps(SpecialAttrsP, proto) From 894e8c13484822458d53cc77c9265b7a88450a4b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 24 Jan 2022 19:13:11 -0800 Subject: [PATCH 045/220] bpo-46503: Prevent an assert from firing when parsing some invalid \N sequences in f-strings. (GH-30865) (GH-30866) * bpo-46503: Prevent an assert from firing. Also fix one nearby tiny PEP-7 nit. * Added blurb. (cherry picked from commit 0daf72194bd4e31de7f12020685bb39a14d6f45e) Co-authored-by: Eric V. Smith Co-authored-by: Eric V. Smith --- Lib/test/test_fstring.py | 4 ++++ .../2022-01-24-21-24-41.bpo-46503.4UrPsE.rst | 1 + Parser/string_parser.c | 16 ++++++++++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-01-24-21-24-41.bpo-46503.4UrPsE.rst diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index bd1ca943c7c094..d0b1ade15137ba 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -746,12 +746,16 @@ def test_misformed_unicode_character_name(self): # differently inside f-strings. self.assertAllRaise(SyntaxError, r"\(unicode error\) 'unicodeescape' codec can't decode bytes in position .*: malformed \\N character escape", [r"f'\N'", + r"f'\N '", + r"f'\N '", # See bpo-46503. r"f'\N{'", r"f'\N{GREEK CAPITAL LETTER DELTA'", # Here are the non-f-string versions, # which should give the same errors. r"'\N'", + r"'\N '", + r"'\N '", r"'\N{'", r"'\N{GREEK CAPITAL LETTER DELTA'", ]) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-01-24-21-24-41.bpo-46503.4UrPsE.rst b/Misc/NEWS.d/next/Core and Builtins/2022-01-24-21-24-41.bpo-46503.4UrPsE.rst new file mode 100644 index 00000000000000..e48028d72ca8e9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-01-24-21-24-41.bpo-46503.4UrPsE.rst @@ -0,0 +1 @@ +Fix an assert when parsing some invalid \N escape sequences in f-strings. diff --git a/Parser/string_parser.c b/Parser/string_parser.c index c83e63fc6f8f2b..4c2043521c0cd3 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -442,12 +442,23 @@ fstring_find_literal(Parser *p, const char **str, const char *end, int raw, if (!raw && ch == '\\' && s < end) { ch = *s++; if (ch == 'N') { + /* We need to look at and skip matching braces for "\N{name}" + sequences because otherwise we'll think the opening '{' + starts an expression, which is not the case with "\N". + Keep looking for either a matched '{' '}' pair, or the end + of the string. */ + if (s < end && *s++ == '{') { while (s < end && *s++ != '}') { } continue; } - break; + + /* This is an invalid "\N" sequence, since it's a "\N" not + followed by a "{". Just keep parsing this literal. This + error will be caught later by + decode_unicode_with_escapes(). */ + continue; } if (ch == '{' && warn_invalid_escape_sequence(p, ch, t) < 0) { return -1; @@ -491,7 +502,8 @@ fstring_find_literal(Parser *p, const char **str, const char *end, int raw, *literal = PyUnicode_DecodeUTF8Stateful(literal_start, s - literal_start, NULL, NULL); - } else { + } + else { *literal = decode_unicode_with_escapes(p, literal_start, s - literal_start, t); } From ce79b504a790d02c080449d31356d33a5aaf19dd Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 24 Jan 2022 23:02:02 -0800 Subject: [PATCH 046/220] bpo-41841: update idlelib News up to 3.10.0. (GH-30868) (GH-30870) (cherry picked from commit 9d3c9788a6ccd4f2f53a147dd0026a316c396976) Co-authored-by: Terry Jan Reedy Co-authored-by: Terry Jan Reedy --- Lib/idlelib/NEWS.txt | 32 +++++++++++++++++++++++++++++++- Misc/NEWS.d/3.10.0a7.rst | 2 +- Misc/NEWS.d/3.10.0b1.rst | 3 ++- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index 396820e9117b53..c0f9a10e10b1c3 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -1,12 +1,30 @@ What's New in IDLE 3.10.0 (since 3.9.0) -Released on 2021-10-04? +Released on 2021-10-04 ========================= +bpo-40128: Mostly fix completions on macOS when not using tcl/tk 8.6.11 +(as with 3.9). + +bpo-33962: Move the indent space setting from the Font tab to the new Windows +tab. Patch by Mark Roseman and Terry Jan Reedy. + +bpo-40468: Split the settings dialog General tab into Windows and Shell/Ed +tabs. Move help sources, which extend the Help menu, to the Extensions tab. +Make space for new options and shorten the dialog. The latter makes the +dialog better fit small screens. + +bpo-44010: Highlight the new match statement's soft keywords: match, case, +and _. This highlighting is not perfect and will be incorrect in some rare +cases, especially for some _s in case patterns. + bpo-44026: Include interpreter's typo fix suggestions in message line for NameErrors and AttributeErrors. Patch by E. Paine. +bpo-41611: Avoid occasional uncaught exceptions and freezing when using +completions on macOS. + bpo-37903: Add mouse actions to the shell sidebar. Left click and optional drag selects one or more lines of text, as with the editor line number sidebar. Right click after selecting text lines @@ -14,6 +32,9 @@ displays a context menu with 'copy with prompts'. This zips together prompts from the sidebar with lines from the selected text. This option also appears on the context menu for the text. +bpo-43981: Fix reference leaks in test_sidebar and test_squeezer. +Patches by Terry Jan Reedy and Pablo Galindo + bpo-37892: Change Shell input indents from tabs to spaces. Shell input now 'looks right'. Making this feasible motivated the shell sidebar. @@ -22,6 +43,9 @@ bpo-37903: Move the Shell input prompt to a side bar. bpo-43655: Make window managers on macOS and X Window recognize IDLE dialog windows as dialogs. +bpo-42225: Document that IDLE can fail on Unix either from misconfigured IP +masquerade rules or failure displaying complex colored (non-ascii) characters. + bpo-43283: Document why printing to IDLE's Shell is often slower than printing to a system terminal and that it can be made faster by pre-formatting a single string before printing. @@ -50,6 +74,12 @@ by using inspect.getdoc. bpo-33987: Mostly finish using ttk widgets, mainly for editor, settings, and searches. Some patches by Mark Roseman. +bpo-40511: Stop unnecessary "flashing" when typing opening and closing +parentheses inside the parentheses of a function call. + +bpo-38439: Add a 256x256 pixel IDLE icon to the Windows .ico file. Created by +Andrew Clover. Remove the low-color gif variations from the .ico file. + bpo-41775: Make 'IDLE Shell' the shell title. bpo-35764: Rewrite the Calltips doc section. diff --git a/Misc/NEWS.d/3.10.0a7.rst b/Misc/NEWS.d/3.10.0a7.rst index 6c32e60dc8a469..f62be491d56dd6 100644 --- a/Misc/NEWS.d/3.10.0a7.rst +++ b/Misc/NEWS.d/3.10.0a7.rst @@ -855,7 +855,7 @@ Aasland. .. nonce: iIeiLg .. section: IDLE -Document that IDLE can fail on Unix either from misconfigured IP masquerage +Document that IDLE can fail on Unix either from misconfigured IP masquerade rules or failure displaying complex colored (non-ascii) characters. .. diff --git a/Misc/NEWS.d/3.10.0b1.rst b/Misc/NEWS.d/3.10.0b1.rst index ca524fe16b7cf8..4731dca2e74cf7 100644 --- a/Misc/NEWS.d/3.10.0b1.rst +++ b/Misc/NEWS.d/3.10.0b1.rst @@ -1673,7 +1673,8 @@ zips together prompts from the sidebar with lines from the selected text. .. nonce: 3EFl1H .. section: IDLE -Fix reference leak in test_squeezer. Patch by Pablo Galindo +Fix reference leak in test_sidebar and test_squeezer. +Patches by Terry Jan Reedy and Pablo Galindo .. From 367a37a18c4411c42da9006947dd95b0afbdf200 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Tue, 25 Jan 2022 03:28:29 -0500 Subject: [PATCH 047/220] [3.10] bpo-46496: Update IDLE News to 2021 Jan 24 (GH-30876) Cherry picked from b1a3446f077b7d56b89f55d98dadb8018986a3e --- Lib/idlelib/NEWS.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index c0f9a10e10b1c3..0fe1cd258bfa10 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -1,3 +1,15 @@ +What's New in IDLE 3.10.z +after 3.10.0 until 3.10.? +Released on 2022-05-16 +========================= + + +bpo-45495: Add context keywords 'case' and 'match' to completions list. + +bpo-45296: On Windows, change exit/quit message to suggest Ctrl-D, which +works, instead of , which does not work in IDLE. + + What's New in IDLE 3.10.0 (since 3.9.0) Released on 2021-10-04 From 41e0aead3defa6d0486514e313b6975fbf375998 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 25 Jan 2022 06:38:45 -0800 Subject: [PATCH 048/220] bpo-46491: Allow Annotated on outside of Final/ClassVar (GH-30864) We treat Annotated type arg as class-level annotation. This exempts it from checks against Final and ClassVar in order to allow using them in any nesting order. Automerge-Triggered-By: GH:gvanrossum (cherry picked from commit e1abffca45b60729c460e3e2ad50c8c1946cfd4e) Co-authored-by: Gregory Beauregard --- Lib/test/test_typing.py | 8 ++++++++ Lib/typing.py | 8 ++++---- .../next/Library/2022-01-24-23-55-30.bpo-46491.jmIKHo.rst | 1 + 3 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-01-24-23-55-30.bpo-46491.jmIKHo.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index acad35d18d5f35..a840ffe8daaec5 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4582,6 +4582,14 @@ class C: A.x = 5 self.assertEqual(C.x, 5) + def test_special_form_containment(self): + class C: + classvar: Annotated[ClassVar[int], "a decoration"] = 4 + const: Annotated[Final[int], "Const"] = 4 + + self.assertEqual(get_type_hints(C, globals())['classvar'], ClassVar[int]) + self.assertEqual(get_type_hints(C, globals())['const'], Final[int]) + def test_hash_eq(self): self.assertEqual(len({Annotated[int, 4, 5], Annotated[int, 4, 5]}), 1) self.assertNotEqual(Annotated[int, 4, 5], Annotated[int, 5, 4]) diff --git a/Lib/typing.py b/Lib/typing.py index 25225470afbac2..705331a9a89a07 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -143,7 +143,7 @@ def _type_convert(arg, module=None): return arg -def _type_check(arg, msg, is_argument=True, module=None, *, is_class=False): +def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms=False): """Check that the argument is a type, and return it (internal helper). As a special case, accept None and return type(None) instead. Also wrap strings @@ -156,7 +156,7 @@ def _type_check(arg, msg, is_argument=True, module=None, *, is_class=False): We append the repr() of the actual value (truncated to 100 chars). """ invalid_generic_forms = (Generic, Protocol) - if not is_class: + if not allow_special_forms: invalid_generic_forms += (ClassVar,) if is_argument: invalid_generic_forms += (Final,) @@ -691,7 +691,7 @@ def _evaluate(self, globalns, localns, recursive_guard): eval(self.__forward_code__, globalns, localns), "Forward references must evaluate to types.", is_argument=self.__forward_is_argument__, - is_class=self.__forward_is_class__, + allow_special_forms=self.__forward_is_class__, ) self.__forward_value__ = _eval_type( type_, globalns, localns, recursive_guard | {self.__forward_arg__} @@ -1677,7 +1677,7 @@ def __class_getitem__(cls, params): "with at least two arguments (a type and an " "annotation).") msg = "Annotated[t, ...]: t must be a type." - origin = _type_check(params[0], msg) + origin = _type_check(params[0], msg, allow_special_forms=True) metadata = tuple(params[1:]) return _AnnotatedAlias(origin, metadata) diff --git a/Misc/NEWS.d/next/Library/2022-01-24-23-55-30.bpo-46491.jmIKHo.rst b/Misc/NEWS.d/next/Library/2022-01-24-23-55-30.bpo-46491.jmIKHo.rst new file mode 100644 index 00000000000000..f66e8868f753fb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-01-24-23-55-30.bpo-46491.jmIKHo.rst @@ -0,0 +1 @@ +Allow :data:`typing.Annotated` to wrap :data:`typing.Final` and :data:`typing.ClassVar`. Patch by Gregory Beauregard. From 9a7d01046723a8a0a10f9a26702c5e39e73d4414 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 25 Jan 2022 23:31:12 +0800 Subject: [PATCH 049/220] [3.10] bpo-46445, bpo-46519: Re-import typing.NewType (GH-30886) Partially reverts 65b88d5e01c845c0cfa3ff61bc8b2faec8f67a57. --- Lib/test/test_typing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index a840ffe8daaec5..9b552c422d56d0 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -19,6 +19,7 @@ from typing import is_typeddict from typing import no_type_check, no_type_check_decorator from typing import Type +from typing import NewType from typing import NamedTuple, TypedDict from typing import IO, TextIO, BinaryIO from typing import Pattern, Match From 75d88b91e6b1320ae0511eaf72e860bea913a3eb Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 25 Jan 2022 09:40:13 -0800 Subject: [PATCH 050/220] bpo-41682: fixed flaky test test_sendfile_close_peer_in_the_middle_of_receiving (GH-30845) (#30860) (cherry picked from commit 1c705fda8f9902906edd26d46acb0433b0b098e1) Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> --- Lib/test/test_asyncio/test_sendfile.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index c8bfa892c73fc6..effca6644c062e 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -92,9 +92,13 @@ async def wait_closed(self): class SendfileBase: - # 128 KiB plus small unaligned to buffer chunk - DATA = b"SendfileBaseData" * (1024 * 8 + 1) - + # 256 KiB plus small unaligned to buffer chunk + # Newer versions of Windows seems to have increased its internal + # buffer and tries to send as much of the data as it can as it + # has some form of buffering for this which is less than 256KiB + # on newer server versions and Windows 11. + # So DATA should be larger than 256 KiB to make this test reliable. + DATA = b"x" * (1024 * 256 + 1) # Reduce socket buffer size to test on relative small data sets. BUF_SIZE = 4 * 1024 # 4 KiB @@ -456,8 +460,6 @@ def test_sendfile_ssl_close_peer_after_receiving(self): # themselves). @unittest.skipIf(sys.platform.startswith('sunos'), "Doesn't work on Solaris") - @unittest.skipIf(sys.platform == "win32", - "It is flaky on Windows and needs to be fixed") # TODO: bpo-41682 def test_sendfile_close_peer_in_the_middle_of_receiving(self): srv_proto, cli_proto = self.prepare_sendfile(close_after=1024) with self.assertRaises(ConnectionError): From 4a57fa296b92125e41220ecd201eb2e432b79fb0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 25 Jan 2022 21:20:34 +0100 Subject: [PATCH 051/220] [3.10] bpo-45382: test.pythoninfo logs more Windows versions (GH-30891) Add the following info to test.pythoninfo: * windows.ver: output of the shell "ver" command * windows.version and windows.version_caption: output of the "wmic os get Caption,Version /value" command. (cherry picked from commit b0898f4aa90d9397e23aef98a2d6b82445ee7455) * bpo-45382: test.pythoninfo: set wmic.exe encoding to OEM (GH-30890) (cherry picked from commit cef0a5458f254c2f8536b928dee25862ca90ffa6) --- Lib/test/pythoninfo.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 39ee9e1d769f8d..8c8011b550acdc 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -720,6 +720,48 @@ def collect_windows(info_add): except (ImportError, AttributeError): pass + import subprocess + try: + # When wmic.exe output is redirected to a pipe, + # it uses the OEM code page + proc = subprocess.Popen(["wmic", "os", "get", "Caption,Version", "/value"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="oem", + text=True) + output, stderr = proc.communicate() + if proc.returncode: + output = "" + except OSError: + pass + else: + for line in output.splitlines(): + line = line.strip() + if line.startswith('Caption='): + line = line.removeprefix('Caption=').strip() + if line: + info_add('windows.version_caption', line) + elif line.startswith('Version='): + line = line.removeprefix('Version=').strip() + if line: + info_add('windows.version', line) + + try: + proc = subprocess.Popen(["ver"], shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True) + output = proc.communicate()[0] + if proc.returncode: + output = "" + except OSError: + return + else: + output = output.strip() + line = output.splitlines()[0] + if line: + info_add('windows.ver', line) + def collect_fips(info_add): try: From 3fc8b74ace033a17346a992f661928ba619e61e8 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 25 Jan 2022 22:33:57 +0000 Subject: [PATCH 052/220] [3.10] bpo-46091: Correctly calculate indentation levels for whitespace lines with continuation characters (GH-30130). (GH-30898) (cherry picked from commit a0efc0c1960e2c49e0092694d98395555270914c) Co-authored-by: Pablo Galindo Salgado --- Lib/test/test_ast.py | 3 +- Lib/test/test_syntax.py | 30 ++++++++++++ Lib/test/test_tokenize.py | 2 +- .../2021-12-16-00-24-00.bpo-46091.rJ_e_e.rst | 2 + Parser/tokenizer.c | 46 +++++++++++++------ 5 files changed, 67 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-12-16-00-24-00.bpo-46091.rJ_e_e.rst diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 39fc7e96738164..95af9e2aa690c4 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1045,8 +1045,7 @@ def test_literal_eval_malformed_lineno(self): ast.literal_eval(node) def test_literal_eval_syntax_errors(self): - msg = "unexpected character after line continuation character" - with self.assertRaisesRegex(SyntaxError, msg): + with self.assertRaisesRegex(SyntaxError, "unexpected indent"): ast.literal_eval(r''' \ (\ diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 7aa93a012e1133..ac5a41ce4cc672 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1463,6 +1463,36 @@ def test_empty_line_after_linecont(self): except SyntaxError: self.fail("Empty line after a line continuation character is valid.") + # See issue-46091 + s1 = r"""\ +def fib(n): + \ +'''Print a Fibonacci series up to n.''' + \ +a, b = 0, 1 +""" + s2 = r"""\ +def fib(n): + '''Print a Fibonacci series up to n.''' + a, b = 0, 1 +""" + try: + self.assertEqual(compile(s1, '', 'exec'), compile(s2, '', 'exec')) + except SyntaxError: + self.fail("Indented statement over multiple lines is valid") + + def test_continuation_bad_indentation(self): + # Check that code that breaks indentation across multiple lines raises a syntax error + + code = r"""\ +if x: + y = 1 + \ + foo = 1 + """ + + self.assertRaises(IndentationError, exec, code) + @support.cpython_only def test_nested_named_except_blocks(self): code = "" diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 4bce1ca9c76f7c..127f0a17c95e32 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -6,6 +6,7 @@ NEWLINE) from io import BytesIO, StringIO import unittest +from textwrap import dedent from unittest import TestCase, mock from test.test_grammar import (VALID_UNDERSCORE_LITERALS, INVALID_UNDERSCORE_LITERALS) @@ -45,7 +46,6 @@ def check_tokenize(self, s, expected): # The ENDMARKER and final NEWLINE are omitted. f = BytesIO(s.encode('utf-8')) result = stringify_tokens_from_source(tokenize(f.readline), s) - self.assertEqual(result, [" ENCODING 'utf-8' (0, 0) (0, 0)"] + expected.rstrip().splitlines()) diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-12-16-00-24-00.bpo-46091.rJ_e_e.rst b/Misc/NEWS.d/next/Core and Builtins/2021-12-16-00-24-00.bpo-46091.rJ_e_e.rst new file mode 100644 index 00000000000000..a2eee0f3ebd515 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-12-16-00-24-00.bpo-46091.rJ_e_e.rst @@ -0,0 +1,2 @@ +Correctly calculate indentation levels for lines with whitespace character +that are ended by line continuation characters. Patch by Pablo Galindo diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 8e9c69d0785afd..de5f57649c256a 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1346,6 +1346,24 @@ tok_decimal_tail(struct tok_state *tok) /* Get next token, after space stripping etc. */ +static inline int +tok_continuation_line(struct tok_state *tok) { + int c = tok_nextc(tok); + if (c != '\n') { + tok->done = E_LINECONT; + return -1; + } + c = tok_nextc(tok); + if (c == EOF) { + tok->done = E_EOF; + tok->cur = tok->inp; + return -1; + } else { + tok_backup(tok, c); + } + return c; +} + static int tok_get(struct tok_state *tok, const char **p_start, const char **p_end) { @@ -1362,6 +1380,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) int col = 0; int altcol = 0; tok->atbol = 0; + int cont_line_col = 0; for (;;) { c = tok_nextc(tok); if (c == ' ') { @@ -1374,14 +1393,23 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) else if (c == '\014') {/* Control-L (formfeed) */ col = altcol = 0; /* For Emacs users */ } + else if (c == '\\') { + // Indentation cannot be split over multiple physical lines + // using backslashes. This means that if we found a backslash + // preceded by whitespace, **the first one we find** determines + // the level of indentation of whatever comes next. + cont_line_col = cont_line_col ? cont_line_col : col; + if ((c = tok_continuation_line(tok)) == -1) { + return ERRORTOKEN; + } + } else { break; } } tok_backup(tok, c); - if (c == '#' || c == '\n' || c == '\\') { + if (c == '#' || c == '\n') { /* Lines with only whitespace and/or comments - and/or a line continuation character shouldn't affect the indentation and are not passed to the parser as NEWLINE tokens, except *totally* empty lines in interactive @@ -1402,6 +1430,8 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) may need to skip to the end of a comment */ } if (!blankline && tok->level == 0) { + col = cont_line_col ? cont_line_col : col; + altcol = cont_line_col ? cont_line_col : altcol; if (col == tok->indstack[tok->indent]) { /* No change */ if (altcol != tok->altindstack[tok->indent]) { @@ -1963,19 +1993,9 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) /* Line continuation */ if (c == '\\') { - c = tok_nextc(tok); - if (c != '\n') { - tok->done = E_LINECONT; + if ((c = tok_continuation_line(tok)) == -1) { return ERRORTOKEN; } - c = tok_nextc(tok); - if (c == EOF) { - tok->done = E_EOF; - tok->cur = tok->inp; - return ERRORTOKEN; - } else { - tok_backup(tok, c); - } tok->cont_line = 1; goto again; /* Read next line */ } From 8356f6aac2fc41cab44159574f5d8fd5fdf95a63 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 25 Jan 2022 21:10:53 -0800 Subject: [PATCH 053/220] bpo-41844: Update IDLE part of What's New 3.9 to 20228 (GH-30905) (cherry picked from commit 7cf285d82ec722d4225297366013e924805171f2) Co-authored-by: Terry Jan Reedy --- Doc/whatsnew/3.9.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 296c64d737d3ec..a6f83a91f5e148 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -484,8 +484,22 @@ Najera in :issue:`38944`.) Added keywords to module name completion list. (Contributed by Terry J. Reedy in :issue:`37765`.) +New in 3.9 maintenance releases + +Make IDLE invoke :func:`sys.excepthook` (when started without '-n'). +User hooks were previously ignored. (Contributed by Ken Hilton in +:issue:`43008`.) + The changes above have been backported to 3.8 maintenance releases. +Rearrange the settings dialog. Split the General tab into Windows +and Shell/Ed tabs. Move help sources, which extend the Help menu, to the +Extensions tab. Make space for new options and shorten the dialog. The +latter makes the dialog better fit small screens. (Contributed by Terry Jan +Reedy in :issue:`40468`.) Move the indent space setting from the Font tab to +the new Windows tab. (Contributed by Mark Roseman and Terry Jan Reedy in +:issue:`33962`.) + imaplib ------- From c1254c44e2e0e807fa1b8a0b589732996d2a9c2e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 25 Jan 2022 23:04:52 -0800 Subject: [PATCH 054/220] bpo-48146: Update IDLE part of What's New 3.10 to 2022 (GH-30906) (cherry picked from commit 4a49fa6ca66664383d406dbf6f6c28289ffeeeaa) Co-authored-by: Terry Jan Reedy --- Doc/whatsnew/3.10.rst | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index c07523c49163bf..6edc09ff6cef6d 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1137,10 +1137,18 @@ IDLE and idlelib ---------------- Make IDLE invoke :func:`sys.excepthook` (when started without '-n'). -User hooks were previously ignored. (Patch by Ken Hilton in +User hooks were previously ignored. (Contributed by Ken Hilton in :issue:`43008`.) -This change was backported to a 3.9 maintenance release. +Rearrange the settings dialog. Split the General tab into Windows +and Shell/Ed tabs. Move help sources, which extend the Help menu, to the +Extensions tab. Make space for new options and shorten the dialog. The +latter makes the dialog better fit small screens. (Contributed by Terry Jan +Reedy in :issue:`40468`.) Move the indent space setting from the Font tab to +the new Windows tab. (Contributed by Mark Roseman and Terry Jan Reedy in +:issue:`33962`.) + +These changes were backported to a 3.9 maintenance release. Add a Shell sidebar. Move the primary prompt ('>>>') to the sidebar. Add secondary prompts ('...') to the sidebar. Left click and optional @@ -1153,12 +1161,9 @@ in :issue:`37903`.) Use spaces instead of tabs to indent interactive code. This makes interactive code entries 'look right'. Making this feasible was a -major motivation for adding the shell sidebar. Contributed by +major motivation for adding the shell sidebar. (Contributed by Terry Jan Reedy in :issue:`37892`.) -We expect to backport these shell changes to a future 3.9 maintenance -release. - Highlight the new :ref:`soft keywords ` :keyword:`match`, :keyword:`case `, and :keyword:`_ ` in pattern-matching statements. However, this highlighting is not perfect From c730342005edf67333c37b575b419e2fc67d232b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 26 Jan 2022 02:39:40 -0800 Subject: [PATCH 055/220] bpo-46529: increase coverage of `typing.Union.__repr__` method (GH-30911) (cherry picked from commit d0c690b5f85c679de6059cf353fe0524e905530e) Co-authored-by: Nikita Sobolev --- Lib/test/test_typing.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 9b552c422d56d0..d4068242da6da1 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -318,6 +318,15 @@ def test_repr(self): u = Union[int | float] self.assertEqual(repr(u), 'typing.Union[int, float]') + u = Union[None, str] + self.assertEqual(repr(u), 'typing.Optional[str]') + u = Union[str, None] + self.assertEqual(repr(u), 'typing.Optional[str]') + u = Union[None, str, int] + self.assertEqual(repr(u), 'typing.Union[NoneType, str, int]') + u = Optional[str] + self.assertEqual(repr(u), 'typing.Optional[str]') + def test_cannot_subclass(self): with self.assertRaises(TypeError): class C(Union): From 4371fbd4328781496f5f2c6938c4d9a84049b187 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 26 Jan 2022 13:20:31 +0200 Subject: [PATCH 056/220] [3.10] bpo-46513: Remove AC_C_CHAR_UNSIGNED / __CHAR_UNSIGNED__ (GH-30851) (GH-30914) Co-authored-by: Christian Heimes --- .../2022-01-25-12-32-37.bpo-46513.mPm9B4.rst | 2 ++ Modules/audioop.c | 7 ---- configure | 33 ------------------- configure.ac | 1 - pyconfig.h.in | 5 --- 5 files changed, 2 insertions(+), 46 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2022-01-25-12-32-37.bpo-46513.mPm9B4.rst diff --git a/Misc/NEWS.d/next/Build/2022-01-25-12-32-37.bpo-46513.mPm9B4.rst b/Misc/NEWS.d/next/Build/2022-01-25-12-32-37.bpo-46513.mPm9B4.rst new file mode 100644 index 00000000000000..b8986ae31a3400 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-01-25-12-32-37.bpo-46513.mPm9B4.rst @@ -0,0 +1,2 @@ +:program:`configure` no longer uses ``AC_C_CHAR_UNSIGNED`` macro and +``pyconfig.h`` no longer defines reserved symbol ``__CHAR_UNSIGNED__``. diff --git a/Modules/audioop.c b/Modules/audioop.c index 3aeb6f04f13cbb..2a5d805c053c7d 100644 --- a/Modules/audioop.c +++ b/Modules/audioop.c @@ -5,13 +5,6 @@ #include "Python.h" -#if defined(__CHAR_UNSIGNED__) -#if defined(signed) -/* This module currently does not work on systems where only unsigned - characters are available. Take it out of Setup. Sorry. */ -#endif -#endif - static const int maxvals[] = {0, 0x7F, 0x7FFF, 0x7FFFFF, 0x7FFFFFFF}; /* -1 trick is needed on Windows to support -0x80000000 without a warning */ static const int minvals[] = {0, -0x80, -0x8000, -0x800000, -0x7FFFFFFF-1}; diff --git a/configure b/configure index a7d2975f1f5e81..e68e00b0b338b0 100755 --- a/configure +++ b/configure @@ -14183,39 +14183,6 @@ fi # checks for compiler characteristics -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether char is unsigned" >&5 -$as_echo_n "checking whether char is unsigned... " >&6; } -if ${ac_cv_c_char_unsigned+:} false; then : - $as_echo_n "(cached) " >&6 -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -$ac_includes_default -int -main () -{ -static int test_array [1 - 2 * !(((char) -1) < 0)]; -test_array [0] = 0; -return test_array [0]; - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - ac_cv_c_char_unsigned=no -else - ac_cv_c_char_unsigned=yes -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_char_unsigned" >&5 -$as_echo "$ac_cv_c_char_unsigned" >&6; } -if test $ac_cv_c_char_unsigned = yes && test "$GCC" != yes; then - $as_echo "#define __CHAR_UNSIGNED__ 1" >>confdefs.h - -fi - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for an ANSI C-conforming const" >&5 $as_echo_n "checking for an ANSI C-conforming const... " >&6; } if ${ac_cv_c_const+:} false; then : diff --git a/configure.ac b/configure.ac index 5aa91cbad35551..0efeb8f585d097 100644 --- a/configure.ac +++ b/configure.ac @@ -4322,7 +4322,6 @@ fi # checks for compiler characteristics -AC_C_CHAR_UNSIGNED AC_C_CONST works=no diff --git a/pyconfig.h.in b/pyconfig.h.in index b97b8f8bf8b4ac..8a4aeda646923b 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1656,11 +1656,6 @@ /* Define on FreeBSD to activate all library features */ #undef __BSD_VISIBLE -/* Define to 1 if type `char' is unsigned and you are not using gcc. */ -#ifndef __CHAR_UNSIGNED__ -# undef __CHAR_UNSIGNED__ -#endif - /* Define to 'long' if doesn't define. */ #undef clock_t From a57ec7a4feaf24f470a9d1e5b1b3f2cb1b062af7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 26 Jan 2022 04:42:26 -0800 Subject: [PATCH 057/220] bpo-43698: do not use `...` as argument name in docs (GH-30502) (cherry picked from commit b9d8980d89bfaa4bf16d60f0488adcc9d2cbf5ef) Co-authored-by: Nikita Sobolev --- Doc/faq/design.rst | 19 ++++++++----------- Doc/glossary.rst | 4 ++-- Doc/library/abc.rst | 10 +++++----- Doc/library/functions.rst | 2 +- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 0437b59d55da6e..ff83a1b8134b77 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -266,12 +266,9 @@ For cases where you need to choose from a very large number of possibilities, you can create a dictionary mapping case values to functions to call. For example:: - def function_1(...): - ... - functions = {'a': function_1, 'b': function_2, - 'c': self.method_1, ...} + 'c': self.method_1} func = functions[value] func() @@ -279,14 +276,14 @@ example:: For calling methods on objects, you can simplify yet further by using the :func:`getattr` built-in to retrieve methods with a particular name:: - def visit_a(self, ...): - ... - ... + class MyVisitor: + def visit_a(self): + ... - def dispatch(self, value): - method_name = 'visit_' + str(value) - method = getattr(self, method_name) - method() + def dispatch(self, value): + method_name = 'visit_' + str(value) + method = getattr(self, method_name) + method() It's suggested that you use a prefix for the method names, such as ``visit_`` in this example. Without such a prefix, if values are coming from an untrusted diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 1bbd05a5bcecd0..ddf085b8c1a6c6 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -292,12 +292,12 @@ Glossary The decorator syntax is merely syntactic sugar, the following two function definitions are semantically equivalent:: - def f(...): + def f(arg): ... f = staticmethod(f) @staticmethod - def f(...): + def f(arg): ... The same concept exists for classes, but is less commonly used there. See diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 1a6ed474ff21da..3b74622e7ff46c 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -186,15 +186,15 @@ The :mod:`abc` module also provides the following decorator: class C(ABC): @abstractmethod - def my_abstract_method(self, ...): + def my_abstract_method(self, arg1): ... @classmethod @abstractmethod - def my_abstract_classmethod(cls, ...): + def my_abstract_classmethod(cls, arg2): ... @staticmethod @abstractmethod - def my_abstract_staticmethod(...): + def my_abstract_staticmethod(arg3): ... @property @@ -255,7 +255,7 @@ The :mod:`abc` module also supports the following legacy decorators: class C(ABC): @classmethod @abstractmethod - def my_abstract_classmethod(cls, ...): + def my_abstract_classmethod(cls, arg): ... @@ -276,7 +276,7 @@ The :mod:`abc` module also supports the following legacy decorators: class C(ABC): @staticmethod @abstractmethod - def my_abstract_staticmethod(...): + def my_abstract_staticmethod(arg): ... diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 9a9c87e32013c8..95231363a5dc2e 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -248,7 +248,7 @@ are always available. They are listed here in alphabetical order. class C: @classmethod - def f(cls, arg1, arg2, ...): ... + def f(cls, arg1, arg2): ... The ``@classmethod`` form is a function :term:`decorator` -- see :ref:`function` for details. From 171fdf2162130bc8c748173bc8eef184b21f5a08 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 26 Jan 2022 15:49:53 -0800 Subject: [PATCH 058/220] bpo-38472: setup.py uses LC_ALL=C to check the C compiler (GH-30929) Fix GCC detection in setup.py when cross-compiling. The C compiler is now run with LC_ALL=C. Previously, the detection failed with a German locale. (cherry picked from commit a9503ac39474a9cb1b1935ddf159c0d9672b04b6) Co-authored-by: Victor Stinner --- .../next/Build/2022-01-26-22-59-12.bpo-38472.RxfLho.rst | 2 ++ setup.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Build/2022-01-26-22-59-12.bpo-38472.RxfLho.rst diff --git a/Misc/NEWS.d/next/Build/2022-01-26-22-59-12.bpo-38472.RxfLho.rst b/Misc/NEWS.d/next/Build/2022-01-26-22-59-12.bpo-38472.RxfLho.rst new file mode 100644 index 00000000000000..4e0ee70bdc5137 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-01-26-22-59-12.bpo-38472.RxfLho.rst @@ -0,0 +1,2 @@ +Fix GCC detection in setup.py when cross-compiling. The C compiler is now +run with LC_ALL=C. Previously, the detection failed with a German locale. diff --git a/setup.py b/setup.py index 43e807f20d9895..e74a275edbf2d0 100644 --- a/setup.py +++ b/setup.py @@ -757,7 +757,9 @@ def add_cross_compiling_paths(self): tmpfile = os.path.join(self.build_temp, 'ccpaths') if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) - ret = run_command('%s -E -v - %s 1>/dev/null' % (CC, tmpfile)) + # bpo-38472: With a German locale, GCC returns "gcc-Version 9.1.0 + # (GCC)", whereas it returns "gcc version 9.1.0" with the C locale. + ret = run_command('LC_ALL=C %s -E -v - %s 1>/dev/null' % (CC, tmpfile)) is_gcc = False is_clang = False in_incdirs = False From c7af838805ddf52320bce3d5978bfdd37eed1b3a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 27 Jan 2022 00:16:50 +0000 Subject: [PATCH 059/220] [3.10] bpo-46502: Remove "How do I tell incomplete input" from FAQ (GH-30925) (GH-30933) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since, - Py_CompileString no longer allows to distinguish "incomplete input" from "invalid input" - there is no alternative solution available from the Python C API due to how the new parser works (rewritten in 3.9) - the only supported way is to manually import the codeop module from C and use its API as IDLE does, and accept its own complications it is desirable to remove this Q&A from the official FAQ.. (cherry picked from commit f0a648152f2d8011f47cc49873438ebaf01d3f82) Co-authored-by: Mateusz Łoskot Co-authored-by: Mateusz Łoskot --- Doc/faq/extending.rst | 125 ------------------------------------------ 1 file changed, 125 deletions(-) diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 3379e41d9de072..1d2aca6f4c8d97 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -254,7 +254,6 @@ For Red Hat, install the python-devel RPM to get the necessary files. For Debian, run ``apt-get install python-dev``. - How do I tell "incomplete input" from "invalid input"? ------------------------------------------------------ @@ -273,130 +272,6 @@ you. You can also set the :c:func:`PyOS_ReadlineFunctionPointer` to point at you custom input function. See ``Modules/readline.c`` and ``Parser/myreadline.c`` for more hints. -However sometimes you have to run the embedded Python interpreter in the same -thread as your rest application and you can't allow the -:c:func:`PyRun_InteractiveLoop` to stop while waiting for user input. -A solution is trying to compile the received string with -:c:func:`Py_CompileString`. If it compiles without errors, try to execute the -returned code object by calling :c:func:`PyEval_EvalCode`. Otherwise save the -input for later. If the compilation fails, find out if it's an error or just -more input is required - by extracting the message string from the exception -tuple and comparing it to the string "unexpected EOF while parsing". Here is a -complete example using the GNU readline library (you may want to ignore -**SIGINT** while calling readline()):: - - #include - #include - - #define PY_SSIZE_T_CLEAN - #include - #include - #include - #include - - int main (int argc, char* argv[]) - { - int i, j, done = 0; /* lengths of line, code */ - char ps1[] = ">>> "; - char ps2[] = "... "; - char *prompt = ps1; - char *msg, *line, *code = NULL; - PyObject *src, *glb, *loc; - PyObject *exc, *val, *trb, *obj, *dum; - - Py_Initialize (); - loc = PyDict_New (); - glb = PyDict_New (); - PyDict_SetItemString (glb, "__builtins__", PyEval_GetBuiltins ()); - - while (!done) - { - line = readline (prompt); - - if (NULL == line) /* Ctrl-D pressed */ - { - done = 1; - } - else - { - i = strlen (line); - - if (i > 0) - add_history (line); /* save non-empty lines */ - - if (NULL == code) /* nothing in code yet */ - j = 0; - else - j = strlen (code); - - code = realloc (code, i + j + 2); - if (NULL == code) /* out of memory */ - exit (1); - - if (0 == j) /* code was empty, so */ - code[0] = '\0'; /* keep strncat happy */ - - strncat (code, line, i); /* append line to code */ - code[i + j] = '\n'; /* append '\n' to code */ - code[i + j + 1] = '\0'; - - src = Py_CompileString (code, "", Py_single_input); - - if (NULL != src) /* compiled just fine - */ - { - if (ps1 == prompt || /* ">>> " or */ - '\n' == code[i + j - 1]) /* "... " and double '\n' */ - { /* so execute it */ - dum = PyEval_EvalCode (src, glb, loc); - Py_XDECREF (dum); - Py_XDECREF (src); - free (code); - code = NULL; - if (PyErr_Occurred ()) - PyErr_Print (); - prompt = ps1; - } - } /* syntax error or E_EOF? */ - else if (PyErr_ExceptionMatches (PyExc_SyntaxError)) - { - PyErr_Fetch (&exc, &val, &trb); /* clears exception! */ - - if (PyArg_ParseTuple (val, "sO", &msg, &obj) && - !strcmp (msg, "unexpected EOF while parsing")) /* E_EOF */ - { - Py_XDECREF (exc); - Py_XDECREF (val); - Py_XDECREF (trb); - prompt = ps2; - } - else /* some other syntax error */ - { - PyErr_Restore (exc, val, trb); - PyErr_Print (); - free (code); - code = NULL; - prompt = ps1; - } - } - else /* some non-syntax error */ - { - PyErr_Print (); - free (code); - code = NULL; - prompt = ps1; - } - - free (line); - } - } - - Py_XDECREF(glb); - Py_XDECREF(loc); - Py_Finalize(); - exit(0); - } - - How do I find undefined g++ symbols __builtin_new or __pure_virtual? -------------------------------------------------------------------- From 5acaad0b3033fde6f21de6ac73681cd6cf64b1f7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 26 Jan 2022 19:55:25 -0800 Subject: [PATCH 060/220] bpo-45296: Clarify close, quit, and exit in IDLE (GH-30936) (GH-30944) In the File menu, 'Close' and 'Exit' are now 'Close Window' (the current one) and 'Exit' is now 'Exit IDLE' (by closing all windows). In Shell, 'quit()' and 'exit()' mean 'close Shell'. If there are no other windows, this also exits IDLE. (cherry picked from commit fcde0bc10ddd836b62d0a8e893d80b8c55e0ba3f) Co-authored-by: Terry Jan Reedy Co-authored-by: Terry Jan Reedy --- Doc/library/idle.rst | 10 ++++++---- Lib/idlelib/help.html | 20 ++++++++++--------- Lib/idlelib/mainmenu.py | 4 ++-- .../2022-01-26-19-33-55.bpo-45296.LzZKdU.rst | 4 ++++ 4 files changed, 23 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2022-01-26-19-33-55.bpo-45296.LzZKdU.rst diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index d740973af9124e..d6021042c61166 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -96,11 +96,13 @@ Save Copy As... Print Window Print the current window to the default printer. -Close - Close the current window (ask to save if unsaved). +Close Window + Close the current window (if an unsaved editor, ask to save; if an unsaved + Shell, ask to quit execution). Calling ``exit()`` or ``close()`` in the Shell + window also closes Shell. If this is the only window, also exit IDLE. -Exit - Close all windows and quit IDLE (ask to save unsaved windows). +Exit IDLE + Close all windows and quit IDLE (ask to save unsaved edit windows). Edit menu (Shell and Editor) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index 2468afa7148b9f..41626ec5abb561 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -5,7 +5,7 @@ - IDLE — Python 3.11.0a0 documentation + IDLE — Python 3.11.0a4 documentation @@ -18,7 +18,7 @@ @@ -71,7 +71,7 @@

Navigation

  • - 3.11.0a0 Documentation » + 3.11.0a4 Documentation »
  • @@ -163,9 +163,11 @@

    File menu (Shell and Editor)exit() or close() in the Shell +window also closes Shell. If this is the only window, also exit IDLE.

    -
    Exit

    Close all windows and quit IDLE (ask to save unsaved windows).

    +
    Exit IDLE

    Close all windows and quit IDLE (ask to save unsaved edit windows).

    @@ -976,7 +978,7 @@

    Navigation

  • - 3.11.0a0 Documentation » + 3.11.0a4 Documentation »
  • @@ -1000,7 +1002,7 @@

    Navigation