From 1e26ec55e058f9d8cabcb27f4f13844788461e76 Mon Sep 17 00:00:00 2001
From: Thomas Wouters
Date: Tue, 1 Oct 2024 07:12:34 +0200
Subject: [PATCH 001/269] Post 3.12.7
---
Include/patchlevel.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Include/patchlevel.h b/Include/patchlevel.h
index 4f0c8fb1aa4e48..a1bbc45127b601 100644
--- a/Include/patchlevel.h
+++ b/Include/patchlevel.h
@@ -23,7 +23,7 @@
#define PY_RELEASE_SERIAL 0
/* Version as a string */
-#define PY_VERSION "3.12.7"
+#define PY_VERSION "3.12.7+"
/*--end constants--*/
/* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2.
From 9547d2a36616525e7e57ef2064cabcfbf5f9b61b Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 1 Oct 2024 11:50:59 +0200
Subject: [PATCH 002/269] [3.12] Doc: Fix archive filenames for standard builds
(GH-124826) (#124837)
Doc: Fix archive filenames for standard builds (GH-124826)
(cherry picked from commit 91e64be731fe42e6b252b95d79d900251388bfc6)
Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
---
Doc/Makefile | 6 ++++--
Doc/tools/extensions/patchlevel.py | 6 +++++-
2 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/Doc/Makefile b/Doc/Makefile
index 70ad703ac77e82..a090ee5ba92705 100644
--- a/Doc/Makefile
+++ b/Doc/Makefile
@@ -305,13 +305,15 @@ serve:
# for development releases: always build
.PHONY: autobuild-dev
+autobuild-dev: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short)
autobuild-dev:
- $(MAKE) dist-no-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1'
+ $(MAKE) dist-no-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION)
# for HTML-only rebuilds
.PHONY: autobuild-dev-html
+autobuild-dev-html: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short)
autobuild-dev-html:
- $(MAKE) dist-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1'
+ $(MAKE) dist-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION)
# for stable releases: only build if not in pre-release stage (alpha, beta)
# release candidate downloads are okay, since the stable tree can be in that stage
diff --git a/Doc/tools/extensions/patchlevel.py b/Doc/tools/extensions/patchlevel.py
index 53ea1bf47b8fd3..9ccaec3dd5ce0f 100644
--- a/Doc/tools/extensions/patchlevel.py
+++ b/Doc/tools/extensions/patchlevel.py
@@ -74,4 +74,8 @@ def get_version_info():
if __name__ == "__main__":
- print(format_version_info(get_header_version_info())[0])
+ short_ver, full_ver = format_version_info(get_header_version_info())
+ if sys.argv[1:2] == ["--short"]:
+ print(short_ver)
+ else:
+ print(full_ver)
From 79746b31ff1c2658d5f3fb3ce68e12733cf837d4 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 1 Oct 2024 18:22:05 +0200
Subject: [PATCH 003/269] [3.12] gh-124842: Fix
test.support.import_helper.make_legacy_pyc() (GH-124843) (GH-124854)
For source file "path/to/file.py" it created file with incorrect path
"/absolute/path/to/path/to/file.pyc" instead of "path/to/file.pyc".
(cherry picked from commit 60ff67d010078eca15a74b1429caf779ac4f9c74)
Co-authored-by: Serhiy Storchaka
---
Lib/test/support/import_helper.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py
index 29c6f535b40342..bb957046364bd4 100644
--- a/Lib/test/support/import_helper.py
+++ b/Lib/test/support/import_helper.py
@@ -58,8 +58,8 @@ def make_legacy_pyc(source):
:return: The file system path to the legacy pyc file.
"""
pyc_file = importlib.util.cache_from_source(source)
- up_one = os.path.dirname(os.path.abspath(source))
- legacy_pyc = os.path.join(up_one, source + 'c')
+ assert source.endswith('.py')
+ legacy_pyc = source + 'c'
shutil.move(pyc_file, legacy_pyc)
return legacy_pyc
From b338a6122bd4f9a8dfb294981fd4c0b6cfc1b744 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 2 Oct 2024 06:57:07 +0200
Subject: [PATCH 004/269] [3.12] Highlight `datetime.timedelta.seconds` vs
`.total_seconds()` in docs. (GH-124811) (GH-124863)
Highlight `datetime.timedelta.seconds` vs `.total_seconds()` in docs. (GH-124811)
Thanks to the reviewers for suggesting the use of a "caution" section instead of "warning" or "note".
(cherry picked from commit d150e4abcfc13770c2d239878ed337fb53e51de5)
Co-authored-by: Gregory P. Smith
---
Doc/library/datetime.rst | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst
index e3ddd8ca82edb6..8f5ea47da0db25 100644
--- a/Doc/library/datetime.rst
+++ b/Doc/library/datetime.rst
@@ -295,6 +295,20 @@ Instance attributes (read-only):
Between 0 and 86,399 inclusive.
+ .. caution::
+
+ It is a somewhat common bug for code to unintentionally use this attribute
+ when it is actually intended to get a :meth:`~timedelta.total_seconds`
+ value instead:
+
+ .. doctest::
+
+ >>> from datetime import timedelta
+ >>> duration = timedelta(seconds=11235813)
+ >>> duration.days, duration.seconds
+ (130, 3813)
+ >>> duration.total_seconds()
+ 11235813.0
.. attribute:: timedelta.microseconds
@@ -351,7 +365,7 @@ Supported operations:
| | same value. (2) |
+--------------------------------+-----------------------------------------------+
| ``-t1`` | Equivalent to ``timedelta(-t1.days, |
-| | -t1.seconds*, -t1.microseconds)``, |
+| | -t1.seconds, -t1.microseconds)``, |
| | and to ``t1 * -1``. (1)(4) |
+--------------------------------+-----------------------------------------------+
| ``abs(t)`` | Equivalent to ``+t`` when ``t.days >= 0``, |
From 72bbebdfd9263dcf11423518aaeb61128dc4b43f Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 2 Oct 2024 11:07:38 +0200
Subject: [PATCH 005/269] [3.12] gh-58282: Fix support of tuple metavar for
positional arguments in argparse (GH-124782) (GH-124881)
Previously, formatting help output or error message for positional argument
with a tuple metavar raised exception.
(cherry picked from commit 9b31a2d83fa7cb0fe4d75ce7cf6a2c9ea2ce0728)
Co-authored-by: Serhiy Storchaka
Co-authored-by: Cyker Way
---
Lib/argparse.py | 13 ++-
Lib/test/test_argparse.py | 98 ++++++++++++++++++-
.../2018-12-04-07-36-27.bpo-14074.fMLKCu.rst | 2 +
3 files changed, 109 insertions(+), 4 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2018-12-04-07-36-27.bpo-14074.fMLKCu.rst
diff --git a/Lib/argparse.py b/Lib/argparse.py
index 3d01415fcf2742..bd5c757197fcf8 100644
--- a/Lib/argparse.py
+++ b/Lib/argparse.py
@@ -564,8 +564,7 @@ def _format_action(self, action):
def _format_action_invocation(self, action):
if not action.option_strings:
default = self._get_default_metavar_for_positional(action)
- metavar, = self._metavar_formatter(action, default)(1)
- return metavar
+ return ' '.join(self._metavar_formatter(action, default)(1))
else:
parts = []
@@ -752,7 +751,15 @@ def _get_action_name(argument):
elif argument.option_strings:
return '/'.join(argument.option_strings)
elif argument.metavar not in (None, SUPPRESS):
- return argument.metavar
+ metavar = argument.metavar
+ if not isinstance(metavar, tuple):
+ return metavar
+ if argument.nargs == ZERO_OR_MORE and len(metavar) == 2:
+ return '%s[, %s]' % metavar
+ elif argument.nargs == ONE_OR_MORE:
+ return '%s[, %s]' % metavar
+ else:
+ return ', '.join(metavar)
elif argument.dest not in (None, SUPPRESS):
return argument.dest
elif argument.choices:
diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py
index cecba6872d2907..1ba958c10addfa 100644
--- a/Lib/test/test_argparse.py
+++ b/Lib/test/test_argparse.py
@@ -4674,7 +4674,7 @@ class TestHelpNone(HelpTestCase):
version = ''
-class TestHelpTupleMetavar(HelpTestCase):
+class TestHelpTupleMetavarOptional(HelpTestCase):
"""Test specifying metavar as a tuple"""
parser_signature = Sig(prog='PROG')
@@ -4701,6 +4701,34 @@ class TestHelpTupleMetavar(HelpTestCase):
version = ''
+class TestHelpTupleMetavarPositional(HelpTestCase):
+ """Test specifying metavar on a Positional as a tuple"""
+
+ parser_signature = Sig(prog='PROG')
+ argument_signatures = [
+ Sig('w', help='w help', nargs='+', metavar=('W1', 'W2')),
+ Sig('x', help='x help', nargs='*', metavar=('X1', 'X2')),
+ Sig('y', help='y help', nargs=3, metavar=('Y1', 'Y2', 'Y3')),
+ Sig('z', help='z help', nargs='?', metavar=('Z1',)),
+ ]
+ argument_group_signatures = []
+ usage = '''\
+ usage: PROG [-h] W1 [W2 ...] [X1 [X2 ...]] Y1 Y2 Y3 [Z1]
+ '''
+ help = usage + '''\
+
+ positional arguments:
+ W1 W2 w help
+ X1 X2 x help
+ Y1 Y2 Y3 y help
+ Z1 z help
+
+ options:
+ -h, --help show this help message and exit
+ '''
+ version = ''
+
+
class TestHelpRawText(HelpTestCase):
"""Test the RawTextHelpFormatter"""
@@ -6146,6 +6174,27 @@ def test_required_args(self):
'the following arguments are required: bar, baz$',
self.parser.parse_args, [])
+ def test_required_args_with_metavar(self):
+ self.parser.add_argument('bar')
+ self.parser.add_argument('baz', metavar='BaZ')
+ self.assertRaisesRegex(argparse.ArgumentError,
+ 'the following arguments are required: bar, BaZ$',
+ self.parser.parse_args, [])
+
+ def test_required_args_n(self):
+ self.parser.add_argument('bar')
+ self.parser.add_argument('baz', nargs=3)
+ self.assertRaisesRegex(argparse.ArgumentError,
+ 'the following arguments are required: bar, baz$',
+ self.parser.parse_args, [])
+
+ def test_required_args_n_with_metavar(self):
+ self.parser.add_argument('bar')
+ self.parser.add_argument('baz', nargs=3, metavar=('B', 'A', 'Z'))
+ self.assertRaisesRegex(argparse.ArgumentError,
+ 'the following arguments are required: bar, B, A, Z$',
+ self.parser.parse_args, [])
+
def test_required_args_optional(self):
self.parser.add_argument('bar')
self.parser.add_argument('baz', nargs='?')
@@ -6160,6 +6209,20 @@ def test_required_args_zero_or_more(self):
'the following arguments are required: bar$',
self.parser.parse_args, [])
+ def test_required_args_one_or_more(self):
+ self.parser.add_argument('bar')
+ self.parser.add_argument('baz', nargs='+')
+ self.assertRaisesRegex(argparse.ArgumentError,
+ 'the following arguments are required: bar, baz$',
+ self.parser.parse_args, [])
+
+ def test_required_args_one_or_more_with_metavar(self):
+ self.parser.add_argument('bar')
+ self.parser.add_argument('baz', nargs='+', metavar=('BaZ1', 'BaZ2'))
+ self.assertRaisesRegex(argparse.ArgumentError,
+ r'the following arguments are required: bar, BaZ1\[, BaZ2]$',
+ self.parser.parse_args, [])
+
def test_required_args_remainder(self):
self.parser.add_argument('bar')
self.parser.add_argument('baz', nargs='...')
@@ -6175,6 +6238,39 @@ def test_required_mutually_exclusive_args(self):
'one of the arguments --bar --baz is required',
self.parser.parse_args, [])
+ def test_conflicting_mutually_exclusive_args_optional_with_metavar(self):
+ group = self.parser.add_mutually_exclusive_group()
+ group.add_argument('--bar')
+ group.add_argument('baz', nargs='?', metavar='BaZ')
+ self.assertRaisesRegex(argparse.ArgumentError,
+ 'argument BaZ: not allowed with argument --bar$',
+ self.parser.parse_args, ['--bar', 'a', 'b'])
+ self.assertRaisesRegex(argparse.ArgumentError,
+ 'argument --bar: not allowed with argument BaZ$',
+ self.parser.parse_args, ['a', '--bar', 'b'])
+
+ def test_conflicting_mutually_exclusive_args_zero_or_more_with_metavar1(self):
+ group = self.parser.add_mutually_exclusive_group()
+ group.add_argument('--bar')
+ group.add_argument('baz', nargs='*', metavar=('BAZ1',))
+ self.assertRaisesRegex(argparse.ArgumentError,
+ 'argument BAZ1: not allowed with argument --bar$',
+ self.parser.parse_args, ['--bar', 'a', 'b'])
+ self.assertRaisesRegex(argparse.ArgumentError,
+ 'argument --bar: not allowed with argument BAZ1$',
+ self.parser.parse_args, ['a', '--bar', 'b'])
+
+ def test_conflicting_mutually_exclusive_args_zero_or_more_with_metavar2(self):
+ group = self.parser.add_mutually_exclusive_group()
+ group.add_argument('--bar')
+ group.add_argument('baz', nargs='*', metavar=('BAZ1', 'BAZ2'))
+ self.assertRaisesRegex(argparse.ArgumentError,
+ r'argument BAZ1\[, BAZ2]: not allowed with argument --bar$',
+ self.parser.parse_args, ['--bar', 'a', 'b'])
+ self.assertRaisesRegex(argparse.ArgumentError,
+ r'argument --bar: not allowed with argument BAZ1\[, BAZ2]$',
+ self.parser.parse_args, ['a', '--bar', 'b'])
+
def test_ambiguous_option(self):
self.parser.add_argument('--foobaz')
self.parser.add_argument('--fooble', action='store_true')
diff --git a/Misc/NEWS.d/next/Library/2018-12-04-07-36-27.bpo-14074.fMLKCu.rst b/Misc/NEWS.d/next/Library/2018-12-04-07-36-27.bpo-14074.fMLKCu.rst
new file mode 100644
index 00000000000000..221c8e05fa98aa
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-12-04-07-36-27.bpo-14074.fMLKCu.rst
@@ -0,0 +1,2 @@
+Fix :mod:`argparse` metavar processing to allow positional arguments to have a
+tuple metavar.
From cceb25c5033bce5d19311d63f6943215a1a27283 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 2 Oct 2024 12:36:54 +0200
Subject: [PATCH 006/269] [3.12] gh-122864: Fix a
``test_funcattrs.test___builtins__`` when executing directly (GH-124845)
(#124885)
gh-122864: Fix a ``test_funcattrs.test___builtins__`` when executing directly (GH-124845)
Previously when executing ``test_functattrs.test___builtins__`` directly, it failed because the fact, that ``__builtins__`` is refers to the built-in module ``builtins`` while it's expects a ``__builtins__.__dict__``. But when this test is being run from another module, then ``__builtins__`` is refers to ``builtins.__dict__``. Now this part of the behaviour is covered.
---------
(cherry picked from commit 8fbf10d6cfd9c69ffcc1f80fa0c5f33785197af7)
Co-authored-by: Mark Byrne <31762852+mbyrnepr2@users.noreply.github.com>
Co-authored-by: Victor Stinner
---
Lib/test/test_funcattrs.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/Lib/test/test_funcattrs.py b/Lib/test/test_funcattrs.py
index 35b473d5e9a0b4..7ab1b0b5f7014c 100644
--- a/Lib/test/test_funcattrs.py
+++ b/Lib/test/test_funcattrs.py
@@ -76,7 +76,12 @@ def test___globals__(self):
(AttributeError, TypeError))
def test___builtins__(self):
- self.assertIs(self.b.__builtins__, __builtins__)
+ if __name__ == "__main__":
+ builtins_dict = __builtins__.__dict__
+ else:
+ builtins_dict = __builtins__
+
+ self.assertIs(self.b.__builtins__, builtins_dict)
self.cannot_set_attr(self.b, '__builtins__', 2,
(AttributeError, TypeError))
@@ -86,7 +91,7 @@ def func(s): return len(s)
ns = {}
func2 = type(func)(func.__code__, ns)
self.assertIs(func2.__globals__, ns)
- self.assertIs(func2.__builtins__, __builtins__)
+ self.assertIs(func2.__builtins__, builtins_dict)
# Make sure that the function actually works.
self.assertEqual(func2("abc"), 3)
From 6660d2974558d2b01b611e8954f9013448d29f0c Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 2 Oct 2024 16:31:20 +0200
Subject: [PATCH 007/269] [3.12] gh-85935: Improve tests for invalid arguments
in test_argparse (GH-124891) (GH-124898)
Check also specific error messages.
(cherry picked from commit 2c050d4bc28bffd2990b5a0bd03fb6fc56b13656)
Co-authored-by: Serhiy Storchaka
---
Lib/test/test_argparse.py | 77 +++++++++++++++++++++------------------
1 file changed, 42 insertions(+), 35 deletions(-)
diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py
index 1ba958c10addfa..13d15fbf075861 100644
--- a/Lib/test/test_argparse.py
+++ b/Lib/test/test_argparse.py
@@ -5069,15 +5069,15 @@ def custom_formatter(prog):
class TestInvalidArgumentConstructors(TestCase):
"""Test a bunch of invalid Argument constructors"""
- def assertTypeError(self, *args, **kwargs):
+ def assertTypeError(self, *args, errmsg=None, **kwargs):
parser = argparse.ArgumentParser()
- self.assertRaises(TypeError, parser.add_argument,
- *args, **kwargs)
+ self.assertRaisesRegex(TypeError, errmsg, parser.add_argument,
+ *args, **kwargs)
- def assertValueError(self, *args, **kwargs):
+ def assertValueError(self, *args, errmsg=None, **kwargs):
parser = argparse.ArgumentParser()
- self.assertRaises(ValueError, parser.add_argument,
- *args, **kwargs)
+ self.assertRaisesRegex(ValueError, errmsg, parser.add_argument,
+ *args, **kwargs)
def test_invalid_keyword_arguments(self):
self.assertTypeError('-x', bar=None)
@@ -5087,8 +5087,9 @@ def test_invalid_keyword_arguments(self):
def test_missing_destination(self):
self.assertTypeError()
- for action in ['append', 'store']:
- self.assertTypeError(action=action)
+ for action in ['store', 'append', 'extend']:
+ with self.subTest(action=action):
+ self.assertTypeError(action=action)
def test_invalid_option_strings(self):
self.assertValueError('--')
@@ -5102,10 +5103,8 @@ def test_invalid_action(self):
self.assertValueError('-x', action='foo')
self.assertValueError('foo', action='baz')
self.assertValueError('--foo', action=('store', 'append'))
- parser = argparse.ArgumentParser()
- with self.assertRaises(ValueError) as cm:
- parser.add_argument("--foo", action="store-true")
- self.assertIn('unknown action', str(cm.exception))
+ self.assertValueError('--foo', action="store-true",
+ errmsg='unknown action')
def test_multiple_dest(self):
parser = argparse.ArgumentParser()
@@ -5118,39 +5117,47 @@ def test_multiple_dest(self):
def test_no_argument_actions(self):
for action in ['store_const', 'store_true', 'store_false',
'append_const', 'count']:
- for attrs in [dict(type=int), dict(nargs='+'),
- dict(choices=['a', 'b'])]:
- self.assertTypeError('-x', action=action, **attrs)
+ with self.subTest(action=action):
+ for attrs in [dict(type=int), dict(nargs='+'),
+ dict(choices=['a', 'b'])]:
+ with self.subTest(attrs=attrs):
+ self.assertTypeError('-x', action=action, **attrs)
+ self.assertTypeError('x', action=action, **attrs)
+ self.assertTypeError('-x', action=action, nargs=0)
+ self.assertTypeError('x', action=action, nargs=0)
def test_no_argument_no_const_actions(self):
# options with zero arguments
for action in ['store_true', 'store_false', 'count']:
+ with self.subTest(action=action):
+ # const is always disallowed
+ self.assertTypeError('-x', const='foo', action=action)
- # const is always disallowed
- self.assertTypeError('-x', const='foo', action=action)
-
- # nargs is always disallowed
- self.assertTypeError('-x', nargs='*', action=action)
+ # nargs is always disallowed
+ self.assertTypeError('-x', nargs='*', action=action)
def test_more_than_one_argument_actions(self):
- for action in ['store', 'append']:
-
- # nargs=0 is disallowed
- self.assertValueError('-x', nargs=0, action=action)
- self.assertValueError('spam', nargs=0, action=action)
-
- # const is disallowed with non-optional arguments
- for nargs in [1, '*', '+']:
- self.assertValueError('-x', const='foo',
- nargs=nargs, action=action)
- self.assertValueError('spam', const='foo',
- nargs=nargs, action=action)
+ for action in ['store', 'append', 'extend']:
+ with self.subTest(action=action):
+ # nargs=0 is disallowed
+ action_name = 'append' if action == 'extend' else action
+ self.assertValueError('-x', nargs=0, action=action,
+ errmsg=f'nargs for {action_name} actions must be != 0')
+ self.assertValueError('spam', nargs=0, action=action,
+ errmsg=f'nargs for {action_name} actions must be != 0')
+
+ # const is disallowed with non-optional arguments
+ for nargs in [1, '*', '+']:
+ self.assertValueError('-x', const='foo',
+ nargs=nargs, action=action)
+ self.assertValueError('spam', const='foo',
+ nargs=nargs, action=action)
def test_required_const_actions(self):
for action in ['store_const', 'append_const']:
-
- # nargs is always disallowed
- self.assertTypeError('-x', nargs='+', action=action)
+ with self.subTest(action=action):
+ # nargs is always disallowed
+ self.assertTypeError('-x', nargs='+', action=action)
def test_parsers_action_missing_params(self):
self.assertTypeError('command', action='parsers')
From 829cdf06d4f40b0774a5a13f9e6023f3e29bf44d Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka
Date: Wed, 2 Oct 2024 21:02:14 +0300
Subject: [PATCH 008/269] [3.12] gh-123978: Remove broken time.thread_time() on
NetBSD (GH-124116) (GH-124427)
(cherry picked from commit e670a113b5e1fcc3ce3bb9b5b4b1f126264ae21b)
---
...-09-16-12-31-48.gh-issue-123978.z3smEu.rst | 1 +
Modules/timemodule.c | 19 ++++++++++++++++---
2 files changed, 17 insertions(+), 3 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-09-16-12-31-48.gh-issue-123978.z3smEu.rst
diff --git a/Misc/NEWS.d/next/Library/2024-09-16-12-31-48.gh-issue-123978.z3smEu.rst b/Misc/NEWS.d/next/Library/2024-09-16-12-31-48.gh-issue-123978.z3smEu.rst
new file mode 100644
index 00000000000000..e5b3229122b509
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-09-16-12-31-48.gh-issue-123978.z3smEu.rst
@@ -0,0 +1 @@
+Remove broken :func:`time.thread_time` and :func:`time.thread_time_ns` on NetBSD.
diff --git a/Modules/timemodule.c b/Modules/timemodule.c
index 8613fccfe02249..b10f8c7c939550 100644
--- a/Modules/timemodule.c
+++ b/Modules/timemodule.c
@@ -1292,8 +1292,14 @@ _PyTime_GetProcessTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info)
#else
/* clock_gettime */
+// gh-115714: Don't use CLOCK_PROCESS_CPUTIME_ID on WASI.
+/* CLOCK_PROF is defined on NetBSD, but not supported.
+ * CLOCK_PROCESS_CPUTIME_ID is broken on NetBSD for the same reason as
+ * CLOCK_THREAD_CPUTIME_ID (see comment below).
+ */
#if defined(HAVE_CLOCK_GETTIME) \
- && (defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_PROF))
+ && (defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_PROF)) \
+ && !defined(__NetBSD__)
struct timespec ts;
if (HAVE_CLOCK_GETTIME_RUNTIME) {
@@ -1499,9 +1505,16 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info)
return 0;
}
+/* CLOCK_THREAD_CPUTIME_ID is broken on NetBSD: the result of clock_gettime()
+ * includes the sleeping time, that defeats the purpose of the clock.
+ * Also, clock_getres() does not support it.
+ * https://github.com/python/cpython/issues/123978
+ * https://gnats.netbsd.org/57512
+ */
#elif defined(HAVE_CLOCK_GETTIME) && \
- defined(CLOCK_PROCESS_CPUTIME_ID) && \
- !defined(__EMSCRIPTEN__) && !defined(__wasi__)
+ defined(CLOCK_THREAD_CPUTIME_ID) && \
+ !defined(__EMSCRIPTEN__) && !defined(__wasi__) && \
+ !defined(__NetBSD__)
#define HAVE_THREAD_TIME
#if defined(__APPLE__) && defined(__has_attribute) && __has_attribute(availability)
From 8dce4918ab3ad6c6faa25f9e3fbbfe022b4882b5 Mon Sep 17 00:00:00 2001
From: Peter Bierma
Date: Wed, 2 Oct 2024 17:40:49 -0400
Subject: [PATCH 009/269] [3.12] gh-120378: Fix crash caused by integer
overflow in `curses` (GH-124555) (#124911)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This is actually an upstream problem in curses, and has been reported
to them already:
https://lists.gnu.org/archive/html/bug-ncurses/2024-09/msg00101.html
This is a nice workaround in the meantime to prevent the segfault.
(cherry picked from commit c2ba931318280796a6dcc33d1a5c5c02ad4d035b)
Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
---
Lib/test/test_curses.py | 16 +++
...-09-25-18-07-51.gh-issue-120378.NlBSz_.rst | 2 +
Modules/_cursesmodule.c | 16 +--
Modules/clinic/_cursesmodule.c.h | 98 +++++++++++++++----
4 files changed, 105 insertions(+), 27 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-09-25-18-07-51.gh-issue-120378.NlBSz_.rst
diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py
index 83d10dd8579074..cc3aa561cd4c42 100644
--- a/Lib/test/test_curses.py
+++ b/Lib/test/test_curses.py
@@ -1081,6 +1081,14 @@ def test_resize_term(self):
self.assertEqual(curses.LINES, lines)
self.assertEqual(curses.COLS, cols)
+ with self.assertRaises(OverflowError):
+ curses.resize_term(35000, 1)
+ with self.assertRaises(OverflowError):
+ curses.resize_term(1, 35000)
+ # GH-120378: Overflow failure in resize_term() causes refresh to fail
+ tmp = curses.initscr()
+ tmp.erase()
+
@requires_curses_func('resizeterm')
def test_resizeterm(self):
curses.update_lines_cols()
@@ -1095,6 +1103,14 @@ def test_resizeterm(self):
self.assertEqual(curses.LINES, lines)
self.assertEqual(curses.COLS, cols)
+ with self.assertRaises(OverflowError):
+ curses.resizeterm(35000, 1)
+ with self.assertRaises(OverflowError):
+ curses.resizeterm(1, 35000)
+ # GH-120378: Overflow failure in resizeterm() causes refresh to fail
+ tmp = curses.initscr()
+ tmp.erase()
+
def test_ungetch(self):
curses.ungetch(b'A')
self.assertEqual(self.stdscr.getkey(), 'A')
diff --git a/Misc/NEWS.d/next/Library/2024-09-25-18-07-51.gh-issue-120378.NlBSz_.rst b/Misc/NEWS.d/next/Library/2024-09-25-18-07-51.gh-issue-120378.NlBSz_.rst
new file mode 100644
index 00000000000000..1a8c1427b6b9b9
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-09-25-18-07-51.gh-issue-120378.NlBSz_.rst
@@ -0,0 +1,2 @@
+Fix a crash related to an integer overflow in :func:`curses.resizeterm`
+and :func:`curses.resize_term`.
diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c
index 743b9e37fbcb04..b099959ea409b6 100644
--- a/Modules/_cursesmodule.c
+++ b/Modules/_cursesmodule.c
@@ -4071,9 +4071,9 @@ NoArgNoReturnFunctionBody(resetty)
/*[clinic input]
_curses.resizeterm
- nlines: int
+ nlines: short
Height.
- ncols: int
+ ncols: short
Width.
/
@@ -4084,8 +4084,8 @@ window dimensions (in particular the SIGWINCH handler).
[clinic start generated code]*/
static PyObject *
-_curses_resizeterm_impl(PyObject *module, int nlines, int ncols)
-/*[clinic end generated code: output=56d6bcc5194ad055 input=0fca02ebad5ffa82]*/
+_curses_resizeterm_impl(PyObject *module, short nlines, short ncols)
+/*[clinic end generated code: output=4de3abab50c67f02 input=414e92a63e3e9899]*/
{
PyObject *result;
@@ -4107,9 +4107,9 @@ _curses_resizeterm_impl(PyObject *module, int nlines, int ncols)
/*[clinic input]
_curses.resize_term
- nlines: int
+ nlines: short
Height.
- ncols: int
+ ncols: short
Width.
/
@@ -4123,8 +4123,8 @@ without additional interaction with the application.
[clinic start generated code]*/
static PyObject *
-_curses_resize_term_impl(PyObject *module, int nlines, int ncols)
-/*[clinic end generated code: output=9e26d8b9ea311ed2 input=2197edd05b049ed4]*/
+_curses_resize_term_impl(PyObject *module, short nlines, short ncols)
+/*[clinic end generated code: output=46c6d749fa291dbd input=276afa43d8ea7091]*/
{
PyObject *result;
diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h
index 9d99d41af5d2d9..1d716bec898d0a 100644
--- a/Modules/clinic/_cursesmodule.c.h
+++ b/Modules/clinic/_cursesmodule.c.h
@@ -3688,25 +3688,55 @@ PyDoc_STRVAR(_curses_resizeterm__doc__,
{"resizeterm", _PyCFunction_CAST(_curses_resizeterm), METH_FASTCALL, _curses_resizeterm__doc__},
static PyObject *
-_curses_resizeterm_impl(PyObject *module, int nlines, int ncols);
+_curses_resizeterm_impl(PyObject *module, short nlines, short ncols);
static PyObject *
_curses_resizeterm(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
{
PyObject *return_value = NULL;
- int nlines;
- int ncols;
+ short nlines;
+ short ncols;
if (!_PyArg_CheckPositional("resizeterm", nargs, 2, 2)) {
goto exit;
}
- nlines = _PyLong_AsInt(args[0]);
- if (nlines == -1 && PyErr_Occurred()) {
- goto exit;
+ {
+ long ival = PyLong_AsLong(args[0]);
+ if (ival == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ else if (ival < SHRT_MIN) {
+ PyErr_SetString(PyExc_OverflowError,
+ "signed short integer is less than minimum");
+ goto exit;
+ }
+ else if (ival > SHRT_MAX) {
+ PyErr_SetString(PyExc_OverflowError,
+ "signed short integer is greater than maximum");
+ goto exit;
+ }
+ else {
+ nlines = (short) ival;
+ }
}
- ncols = _PyLong_AsInt(args[1]);
- if (ncols == -1 && PyErr_Occurred()) {
- goto exit;
+ {
+ long ival = PyLong_AsLong(args[1]);
+ if (ival == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ else if (ival < SHRT_MIN) {
+ PyErr_SetString(PyExc_OverflowError,
+ "signed short integer is less than minimum");
+ goto exit;
+ }
+ else if (ival > SHRT_MAX) {
+ PyErr_SetString(PyExc_OverflowError,
+ "signed short integer is greater than maximum");
+ goto exit;
+ }
+ else {
+ ncols = (short) ival;
+ }
}
return_value = _curses_resizeterm_impl(module, nlines, ncols);
@@ -3739,25 +3769,55 @@ PyDoc_STRVAR(_curses_resize_term__doc__,
{"resize_term", _PyCFunction_CAST(_curses_resize_term), METH_FASTCALL, _curses_resize_term__doc__},
static PyObject *
-_curses_resize_term_impl(PyObject *module, int nlines, int ncols);
+_curses_resize_term_impl(PyObject *module, short nlines, short ncols);
static PyObject *
_curses_resize_term(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
{
PyObject *return_value = NULL;
- int nlines;
- int ncols;
+ short nlines;
+ short ncols;
if (!_PyArg_CheckPositional("resize_term", nargs, 2, 2)) {
goto exit;
}
- nlines = _PyLong_AsInt(args[0]);
- if (nlines == -1 && PyErr_Occurred()) {
- goto exit;
+ {
+ long ival = PyLong_AsLong(args[0]);
+ if (ival == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ else if (ival < SHRT_MIN) {
+ PyErr_SetString(PyExc_OverflowError,
+ "signed short integer is less than minimum");
+ goto exit;
+ }
+ else if (ival > SHRT_MAX) {
+ PyErr_SetString(PyExc_OverflowError,
+ "signed short integer is greater than maximum");
+ goto exit;
+ }
+ else {
+ nlines = (short) ival;
+ }
}
- ncols = _PyLong_AsInt(args[1]);
- if (ncols == -1 && PyErr_Occurred()) {
- goto exit;
+ {
+ long ival = PyLong_AsLong(args[1]);
+ if (ival == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ else if (ival < SHRT_MIN) {
+ PyErr_SetString(PyExc_OverflowError,
+ "signed short integer is less than minimum");
+ goto exit;
+ }
+ else if (ival > SHRT_MAX) {
+ PyErr_SetString(PyExc_OverflowError,
+ "signed short integer is greater than maximum");
+ goto exit;
+ }
+ else {
+ ncols = (short) ival;
+ }
}
return_value = _curses_resize_term_impl(module, nlines, ncols);
@@ -4313,4 +4373,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored
#ifndef _CURSES_USE_DEFAULT_COLORS_METHODDEF
#define _CURSES_USE_DEFAULT_COLORS_METHODDEF
#endif /* !defined(_CURSES_USE_DEFAULT_COLORS_METHODDEF) */
-/*[clinic end generated code: output=27a2364193b503c1 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=764ee4c154c6d4a8 input=a9049054013a1b77]*/
From c13bb996f63f042ce00292e741e7a5a870a49da3 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 3 Oct 2024 10:31:54 +0200
Subject: [PATCH 010/269] [3.12] gh-121982: ``csv``: Add a test case for
invalid ``quoting`` constant. (GH-121983) (#124925)
gh-121982: ``csv``: Add a test case for invalid ``quoting`` constant. (GH-121983)
Test invalid quoting constant
(cherry picked from commit 656b7a3c83c79f99beac950b59c47575562ea729)
Co-authored-by: Tomas R
Co-authored-by: Kirill Podoprigora
---
Lib/test/test_csv.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py
index 57db59185d97f7..47ff707ffe6230 100644
--- a/Lib/test/test_csv.py
+++ b/Lib/test/test_csv.py
@@ -1071,6 +1071,12 @@ class mydialect(csv.Dialect):
mydialect.quoting = None
self.assertRaises(csv.Error, mydialect)
+ mydialect.quoting = 42
+ with self.assertRaises(csv.Error) as cm:
+ mydialect()
+ self.assertEqual(str(cm.exception),
+ 'bad "quoting" value')
+
mydialect.doublequote = True
mydialect.quoting = csv.QUOTE_ALL
mydialect.quotechar = '"'
From 58f7763b88d4e81662b9d212f6beddcb42437fe3 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 3 Oct 2024 12:45:25 +0200
Subject: [PATCH 011/269] [3.12] gh-115145: Update documentation about
``PyThreadState_DeleteCurrent`` (gh-124920) (gh-124931)
gh-115145: Update documentation about ``PyThreadState_DeleteCurrent`` (gh-124920)
(cherry picked from commit 9eeb21bf761070649bf8d78976a62dabb6d67a99)
Co-authored-by: Donghee Na
---
Doc/c-api/init.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
index 8b7b28ae319200..bec075038f06e0 100644
--- a/Doc/c-api/init.rst
+++ b/Doc/c-api/init.rst
@@ -1269,7 +1269,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
.. c:function:: void PyThreadState_DeleteCurrent(void)
Destroy the current thread state and release the global interpreter lock.
- Like :c:func:`PyThreadState_Delete`, the global interpreter lock need not
+ Like :c:func:`PyThreadState_Delete`, the global interpreter lock must
be held. The thread state must have been reset with a previous call
to :c:func:`PyThreadState_Clear`.
From b3e2c0291595edddc968680689bec7707d27d2d1 Mon Sep 17 00:00:00 2001
From: Petr Viktorin
Date: Fri, 4 Oct 2024 16:50:34 +0200
Subject: [PATCH 012/269] [3.12] gh-113993: For string interning, do not rely
on (or assert) _Py_IsImmortal (GH-121358) (GH-124938)
gh-113993: For string interning, do not rely on (or assert) _Py_IsImmortal (GH-121358)
Older stable ABI extensions are allowed to make immortal objects mortal.
Instead, use `_PyUnicode_STATE` (`interned` and `statically_allocated`).
(cherry picked from commit 956270d08d5c23f59937e2f29f8e0b7f63d68afd)
Co-authored-by: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com>
---
...024-07-04-13-23-27.gh-issue-113601.K3RLqp.rst | 2 ++
Objects/unicodeobject.c | 16 ++++++++++------
2 files changed, 12 insertions(+), 6 deletions(-)
create mode 100644 Misc/NEWS.d/next/C API/2024-07-04-13-23-27.gh-issue-113601.K3RLqp.rst
diff --git a/Misc/NEWS.d/next/C API/2024-07-04-13-23-27.gh-issue-113601.K3RLqp.rst b/Misc/NEWS.d/next/C API/2024-07-04-13-23-27.gh-issue-113601.K3RLqp.rst
new file mode 100644
index 00000000000000..009cc2bf017180
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2024-07-04-13-23-27.gh-issue-113601.K3RLqp.rst
@@ -0,0 +1,2 @@
+Removed debug build assertions related to interning strings, which were
+falsely triggered by stable ABI extensions.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 815747e1b1ed9c..7bd4b221c83cf7 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -257,7 +257,8 @@ _PyUnicode_InternedSize_Immortal(void)
// value, to help detect bugs in optimizations.
while (PyDict_Next(dict, &pos, &key, &value)) {
- if (_Py_IsImmortal(key)) {
+ assert(PyUnicode_CHECK_INTERNED(key) != SSTATE_INTERNED_IMMORTAL_STATIC);
+ if (PyUnicode_CHECK_INTERNED(key) == SSTATE_INTERNED_IMMORTAL) {
count++;
}
}
@@ -691,10 +692,14 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content)
/* Check interning state */
#ifdef Py_DEBUG
+ // Note that we do not check `_Py_IsImmortal(op)`, since stable ABI
+ // extensions can make immortal strings mortal (but with a high enough
+ // refcount).
+ // The other way is extremely unlikely (worth a potential failed assertion
+ // in a debug build), so we do check `!_Py_IsImmortal(op)`.
switch (PyUnicode_CHECK_INTERNED(op)) {
case SSTATE_NOT_INTERNED:
if (ascii->state.statically_allocated) {
- CHECK(_Py_IsImmortal(op));
// This state is for two exceptions:
// - strings are currently checked before they're interned
// - the 256 one-latin1-character strings
@@ -710,11 +715,9 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content)
break;
case SSTATE_INTERNED_IMMORTAL:
CHECK(!ascii->state.statically_allocated);
- CHECK(_Py_IsImmortal(op));
break;
case SSTATE_INTERNED_IMMORTAL_STATIC:
CHECK(ascii->state.statically_allocated);
- CHECK(_Py_IsImmortal(op));
break;
default:
Py_UNREACHABLE();
@@ -1899,7 +1902,8 @@ unicode_write_cstr(PyObject *unicode, Py_ssize_t index,
static PyObject*
get_latin1_char(Py_UCS1 ch)
{
- return Py_NewRef(LATIN1(ch));
+ PyObject *o = LATIN1(ch);
+ return o;
}
static PyObject*
@@ -15015,7 +15019,7 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */,
{
PyObject *r = (PyObject *)_Py_hashtable_get(INTERNED_STRINGS, s);
if (r != NULL) {
- assert(_Py_IsImmortal(r));
+ assert(_PyUnicode_STATE(r).statically_allocated);
assert(r != s); // r must be statically_allocated; s is not
Py_DECREF(s);
return Py_NewRef(r);
From db4b382b1faad575a0f630d71462c610661235ea Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sun, 6 Oct 2024 15:34:24 +0200
Subject: [PATCH 013/269] [3.12] gh-125008: Fix `tokenize.untokenize` roundtrip
for `\n{{` (GH-125013) (#125021)
---
Lib/test/test_tokenize.py | 20 +++++++++++++++++++
Lib/tokenize.py | 2 +-
...-10-05-23-53-06.gh-issue-125008.ETANpd.rst | 2 ++
3 files changed, 23 insertions(+), 1 deletion(-)
create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-05-23-53-06.gh-issue-125008.ETANpd.rst
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 84741e308bffae..2c4e7b960f7273 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1916,6 +1916,26 @@ def test_roundtrip(self):
self.check_roundtrip(r"f'\\\\N{{'")
self.check_roundtrip(r"f'\\\\\\N{{'")
self.check_roundtrip(r"f'\\\\\\\\N{{'")
+
+ self.check_roundtrip(r"f'\n{{foo}}'")
+ self.check_roundtrip(r"f'\\n{{foo}}'")
+ self.check_roundtrip(r"f'\\\n{{foo}}'")
+ self.check_roundtrip(r"f'\\\\n{{foo}}'")
+
+ self.check_roundtrip(r"f'\t{{foo}}'")
+ self.check_roundtrip(r"f'\\t{{foo}}'")
+ self.check_roundtrip(r"f'\\\t{{foo}}'")
+ self.check_roundtrip(r"f'\\\\t{{foo}}'")
+
+ self.check_roundtrip(r"rf'\t{{foo}}'")
+ self.check_roundtrip(r"rf'\\t{{foo}}'")
+ self.check_roundtrip(r"rf'\\\t{{foo}}'")
+ self.check_roundtrip(r"rf'\\\\t{{foo}}'")
+
+ self.check_roundtrip(r"rf'\{{foo}}'")
+ self.check_roundtrip(r"f'\\{{foo}}'")
+ self.check_roundtrip(r"rf'\\\{{foo}}'")
+ self.check_roundtrip(r"f'\\\\{{foo}}'")
cases = [
"""
if 1:
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 7af7a5cc1cd680..b2dff8e6967094 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -202,7 +202,7 @@ def escape_brackets(self, token):
characters[-2::-1]
)
)
- if n_backslashes % 2 == 0:
+ if n_backslashes % 2 == 0 or characters[-1] != "N":
characters.append(character)
else:
consume_until_next_bracket = True
diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-05-23-53-06.gh-issue-125008.ETANpd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-05-23-53-06.gh-issue-125008.ETANpd.rst
new file mode 100644
index 00000000000000..8971e052860225
--- /dev/null
+++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-05-23-53-06.gh-issue-125008.ETANpd.rst
@@ -0,0 +1,2 @@
+Fix :func:`tokenize.untokenize` producing invalid syntax for
+double braces preceded by certain escape characters.
From 73d7393780b6d92909782148f50ec03f0db5892d Mon Sep 17 00:00:00 2001
From: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
Date: Sun, 6 Oct 2024 23:42:11 +0100
Subject: [PATCH 014/269] [3.12] Doc: Simplify the definition of 'soft
deprecated' (GH-124988) (#125030)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
(cherry picked from commit feca4cf64e9742b9c002d5533ced47e68b34a880)
Co-authored-by: Andrés Delfino
Co-authored-by: Sergey B Kirpichev
Co-authored-by: Carol Willing
---
Doc/glossary.rst | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index f7870155930793..b233967adc9e6a 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -1134,6 +1134,17 @@ Glossary
when several are given, such as in ``variable_name[1:3:5]``. The bracket
(subscript) notation uses :class:`slice` objects internally.
+ soft deprecated
+ A soft deprecated API should not be used in new code,
+ but it is safe for already existing code to use it.
+ The API remains documented and tested, but will not be enhanced further.
+
+ Soft deprecation, unlike normal deprecation, does not plan on removing the API
+ and will not emit warnings.
+
+ See `PEP 387: Soft Deprecation
+ `_.
+
special method
.. index:: pair: special; method
From 36f9bbbdab905563ce0ca5e4f2b6bab1af5cb729 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 7 Oct 2024 01:03:30 +0200
Subject: [PATCH 015/269] [3.12] gh-125025: `_thread` docs: fix/update the
*caveats* list (GH-125026) (GH-125031)
gh-125025: `_thread` docs: fix/update the *caveats* list (GH-125026)
(cherry picked from commit 1e098dc766ba4f29a63da4f188fb214af7623365)
Co-authored-by: Jan Kaliszewski
---
Doc/library/_thread.rst | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst
index 35053702e6db2b..d82f63834dd2d1 100644
--- a/Doc/library/_thread.rst
+++ b/Doc/library/_thread.rst
@@ -210,9 +210,8 @@ In addition to these methods, lock objects can also be used via the
.. index:: pair: module; signal
-* Threads interact strangely with interrupts: the :exc:`KeyboardInterrupt`
- exception will be received by an arbitrary thread. (When the :mod:`signal`
- module is available, interrupts always go to the main thread.)
+* Interrupts always go to the main thread (the :exc:`KeyboardInterrupt`
+ exception will be received by that thread.)
* Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is
equivalent to calling :func:`_thread.exit`.
@@ -226,7 +225,3 @@ In addition to these methods, lock objects can also be used via the
:keyword:`try` ... :keyword:`finally` clauses or executing object
destructors.
-* When the main thread exits, it does not do any of its usual cleanup (except
- that :keyword:`try` ... :keyword:`finally` clauses are honored), and the
- standard I/O files are not flushed.
-
From fa32f007f0ee702fbad970961a0258e8a3e92de8 Mon Sep 17 00:00:00 2001
From: Alyssa Coghlan
Date: Tue, 8 Oct 2024 00:24:45 +1000
Subject: [PATCH 016/269] [3.12] gh-125018: Add importlib.metadata semantic
link targets (GH-125027) (#125048)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This allows direct intersphinx references to APIs via references
like `` :func:`importlib.metadata.version` ``.
---------
(cherry picked from commit cda3b5a576412a8671bbe4c68bb792ec14f1a4b1)
Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко)
Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
---
Doc/library/importlib.metadata.rst | 188 ++++++++++++++----
...-10-07-00-31-17.gh-issue-125018.yKnymn.rst | 4 +
2 files changed, 148 insertions(+), 44 deletions(-)
create mode 100644 Misc/NEWS.d/next/Documentation/2024-10-07-00-31-17.gh-issue-125018.yKnymn.rst
diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst
index 3f407ba48a4d8e..d85b3c68626a89 100644
--- a/Doc/library/importlib.metadata.rst
+++ b/Doc/library/importlib.metadata.rst
@@ -100,6 +100,13 @@ You can also get a :ref:`distribution's version number `, list its
:ref:`requirements`.
+.. exception:: PackageNotFoundError
+
+ Subclass of :class:`ModuleNotFoundError` raised by several functions in this
+ module when queried for a distribution package which is not installed in the
+ current Python environment.
+
+
Functional API
==============
@@ -111,31 +118,53 @@ This package provides the following functionality via its public API.
Entry points
------------
-The ``entry_points()`` function returns a collection of entry points.
-Entry points are represented by ``EntryPoint`` instances;
-each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and
-a ``.load()`` method to resolve the value. There are also ``.module``,
-``.attr``, and ``.extras`` attributes for getting the components of the
-``.value`` attribute.
+.. function:: entry_points(**select_params)
+
+ Returns a :class:`EntryPoints` instance describing entry points for the
+ current environment. Any given keyword parameters are passed to the
+ :meth:`!~EntryPoints.select` method for comparison to the attributes of
+ the individual entry point definitions.
+
+ Note: it is not currently possible to query for entry points based on
+ their :attr:`!EntryPoint.dist` attribute (as different :class:`!Distribution`
+ instances do not currently compare equal, even if they have the same attributes)
+
+.. class:: EntryPoints
+
+ Details of a collection of installed entry points.
+
+ Also provides a ``.groups`` attribute that reports all identifed entry
+ point groups, and a ``.names`` attribute that reports all identified entry
+ point names.
+
+.. class:: EntryPoint
+
+ Details of an installed entry point.
+
+ Each :class:`!EntryPoint` instance has ``.name``, ``.group``, and ``.value``
+ attributes and a ``.load()`` method to resolve the value. There are also
+ ``.module``, ``.attr``, and ``.extras`` attributes for getting the
+ components of the ``.value`` attribute, and ``.dist`` for obtaining
+ information regarding the distribution package that provides the entry point.
Query all entry points::
>>> eps = entry_points() # doctest: +SKIP
-The ``entry_points()`` function returns an ``EntryPoints`` object,
-a collection of all ``EntryPoint`` objects with ``names`` and ``groups``
+The :func:`!entry_points` function returns a :class:`!EntryPoints` object,
+a collection of all :class:`!EntryPoint` objects with ``names`` and ``groups``
attributes for convenience::
>>> sorted(eps.groups) # doctest: +SKIP
['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation']
-``EntryPoints`` has a ``select`` method to select entry points
+:class:`!EntryPoints` has a :meth:`!~EntryPoints.select` method to select entry points
matching specific properties. Select entry points in the
``console_scripts`` group::
>>> scripts = eps.select(group='console_scripts') # doctest: +SKIP
-Equivalently, since ``entry_points`` passes keyword arguments
+Equivalently, since :func:`!entry_points` passes keyword arguments
through to select::
>>> scripts = entry_points(group='console_scripts') # doctest: +SKIP
@@ -187,31 +216,41 @@ for compatibility options.
Distribution metadata
---------------------
-Every `Distribution Package `_ includes some metadata,
-which you can extract using the
-``metadata()`` function::
+.. function:: metadata(distribution_name)
+
+ Return the distribution metadata corresponding to the named
+ distribution package as a :class:`PackageMetadata` instance.
+
+ Raises :exc:`PackageNotFoundError` if the named distribution
+ package is not installed in the current Python environment.
+
+.. class:: PackageMetadata
+
+ A concrete implementation of the
+ `PackageMetadata protocol `_.
+
+ In addition to providing the defined protocol methods and attributes, subscripting
+ the instance is equivalent to calling the :meth:`!~PackageMetadata.get` method.
+
+Every `Distribution Package `_
+includes some metadata, which you can extract using the :func:`!metadata` function::
>>> wheel_metadata = metadata('wheel') # doctest: +SKIP
-The keys of the returned data structure, a ``PackageMetadata``,
-name the metadata keywords, and
+The keys of the returned data structure name the metadata keywords, and
the values are returned unparsed from the distribution metadata::
>>> wheel_metadata['Requires-Python'] # doctest: +SKIP
'>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*'
-``PackageMetadata`` also presents a ``json`` attribute that returns
+:class:`PackageMetadata` also presents a :attr:`!~PackageMetadata.json` attribute that returns
all the metadata in a JSON-compatible form per :PEP:`566`::
>>> wheel_metadata.json['requires_python']
'>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*'
-.. note::
-
- The actual type of the object returned by ``metadata()`` is an
- implementation detail and should be accessed only through the interface
- described by the
- `PackageMetadata protocol `_.
+The full set of available metadata is not described here.
+See the PyPA `Core metadata specification `_ for additional details.
.. versionchanged:: 3.10
The ``Description`` is now included in the metadata when presented
@@ -225,7 +264,15 @@ all the metadata in a JSON-compatible form per :PEP:`566`::
Distribution versions
---------------------
-The ``version()`` function is the quickest way to get a
+.. function:: version(distribution_name)
+
+ Return the installed distribution package version for the named
+ distribution package.
+
+ Raises :exc:`PackageNotFoundError` if the named distribution
+ package is not installed in the current Python environment.
+
+The :func:`!version` function is the quickest way to get a
`Distribution Package `_'s version
number, as a string::
@@ -238,12 +285,28 @@ number, as a string::
Distribution files
------------------
-You can also get the full set of files contained within a distribution. The
-``files()`` function takes a `Distribution Package `_ name
-and returns all of the
-files installed by this distribution. Each file object returned is a
-``PackagePath``, a :class:`pathlib.PurePath` derived object with additional ``dist``,
-``size``, and ``hash`` properties as indicated by the metadata. For example::
+.. function:: files(distribution_name)
+
+ Return the full set of files contained within the named
+ distribution package.
+
+ Raises :exc:`PackageNotFoundError` if the named distribution
+ package is not installed in the current Python environment.
+
+ Returns :const:`None` if the distribution is found but the installation
+ database records reporting the files associated with the distribuion package
+ are missing.
+
+.. class:: PackagePath
+
+ A :class:`pathlib.PurePath` derived object with additional ``dist``,
+ ``size``, and ``hash`` properties corresponding to the distribution
+ package's installation metadata for that file.
+
+The :func:`!files` function takes a
+`Distribution Package `_
+name and returns all of the files installed by this distribution. Each file is reported
+as a :class:`PackagePath` instance. For example::
>>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP
>>> util # doctest: +SKIP
@@ -266,16 +329,16 @@ Once you have the file, you can also read its contents::
return s.encode('utf-8')
return s
-You can also use the ``locate`` method to get a the absolute path to the
-file::
+You can also use the :meth:`!~PackagePath.locate` method to get the absolute
+path to the file::
>>> util.locate() # doctest: +SKIP
PosixPath('/home/gustav/example/lib/site-packages/wheel/util.py')
In the case where the metadata file listing files
-(RECORD or SOURCES.txt) is missing, ``files()`` will
-return ``None``. The caller may wish to wrap calls to
-``files()`` in `always_iterable
+(``RECORD`` or ``SOURCES.txt``) is missing, :func:`!files` will
+return :const:`None`. The caller may wish to wrap calls to
+:func:`!files` in `always_iterable
`_
or otherwise guard against this condition if the target
distribution is not known to have the metadata present.
@@ -285,8 +348,16 @@ distribution is not known to have the metadata present.
Distribution requirements
-------------------------
+.. function:: requires(distribution_name)
+
+ Return the declared dependency specifiers for the named
+ distribution package.
+
+ Raises :exc:`PackageNotFoundError` if the named distribution
+ package is not installed in the current Python environment.
+
To get the full set of requirements for a `Distribution Package `_,
-use the ``requires()``
+use the :func:`!requires`
function::
>>> requires('wheel') # doctest: +SKIP
@@ -299,6 +370,16 @@ function::
Mapping import to distribution packages
---------------------------------------
+.. function:: packages_distributions()
+
+ Return a mapping from the top level module and import package
+ names found via :attr:`sys.meta_path` to the names of the distribution
+ packages (if any) that provide the corresponding files.
+
+ To allow for namespace packages (which may have members provided by
+ multiple distribution packages), each top level import name maps to a
+ list of distribution names rather than mapping directly to a single name.
+
A convenience method to resolve the `Distribution Package `_
name (or names, in the case of a namespace package)
that provide each importable top-level
@@ -318,23 +399,42 @@ function is not reliable with such installs.
Distributions
=============
-While the above API is the most common and convenient usage, you can get all
-of that information from the ``Distribution`` class. A ``Distribution`` is an
-abstract object that represents the metadata for
-a Python `Distribution Package `_. You can
-get the ``Distribution`` instance::
+.. function:: distribution(distribution_name)
+
+ Return a :class:`Distribution` instance describing the named
+ distribution package.
+
+ Raises :exc:`PackageNotFoundError` if the named distribution
+ package is not installed in the current Python environment.
+
+.. class:: Distribution
+
+ Details of an installed distribution package.
+
+ Note: different :class:`!Distribution` instances do not currently compare
+ equal, even if they relate to the same installed distribution and
+ accordingly have the same attributes.
+
+While the module level API described above is the most common and convenient usage,
+you can get all of that information from the :class:`!Distribution` class.
+:class:`!Distribution` is an abstract object that represents the metadata for
+a Python `Distribution Package `_.
+You can get the concreate :class:`!Distribution` subclass instance for an installed
+distribution package by calling the :func:`distribution` function::
>>> from importlib.metadata import distribution # doctest: +SKIP
>>> dist = distribution('wheel') # doctest: +SKIP
+ >>> type(dist) # doctest: +SKIP
+
Thus, an alternative way to get the version number is through the
-``Distribution`` instance::
+:class:`!Distribution` instance::
>>> dist.version # doctest: +SKIP
'0.32.3'
-There are all kinds of additional metadata available on the ``Distribution``
-instance::
+There are all kinds of additional metadata available on :class:`!Distribution`
+instances::
>>> dist.metadata['Requires-Python'] # doctest: +SKIP
'>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*'
@@ -342,7 +442,7 @@ instance::
'MIT'
The full set of available metadata is not described here.
-See the `Core metadata specifications `_ for additional details.
+See the PyPA `Core metadata specification `_ for additional details.
Distribution Discovery
diff --git a/Misc/NEWS.d/next/Documentation/2024-10-07-00-31-17.gh-issue-125018.yKnymn.rst b/Misc/NEWS.d/next/Documentation/2024-10-07-00-31-17.gh-issue-125018.yKnymn.rst
new file mode 100644
index 00000000000000..e910da5b879ba5
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2024-10-07-00-31-17.gh-issue-125018.yKnymn.rst
@@ -0,0 +1,4 @@
+The :mod:`importlib.metadata` documentation now includes semantic
+cross-reference targets for the significant documented APIs. This means
+intersphinx references like :func:`importlib.metadata.version` will
+now work as expected.
From d2b307171ce899f6bd17f6ba570f5172ff9a390a Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 7 Oct 2024 18:58:47 +0200
Subject: [PATCH 017/269] [3.12] gh-125018: Fix role syntax (GH-125050)
(#125055)
gh-125018: Fix role syntax (GH-125050)
(cherry picked from commit 10094a533a947b72d01ed8195dcf540f2e7820ea)
Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
---
Doc/library/importlib.metadata.rst | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst
index d85b3c68626a89..14414d973f20c1 100644
--- a/Doc/library/importlib.metadata.rst
+++ b/Doc/library/importlib.metadata.rst
@@ -122,7 +122,7 @@ Entry points
Returns a :class:`EntryPoints` instance describing entry points for the
current environment. Any given keyword parameters are passed to the
- :meth:`!~EntryPoints.select` method for comparison to the attributes of
+ :meth:`!select` method for comparison to the attributes of
the individual entry point definitions.
Note: it is not currently possible to query for entry points based on
@@ -158,7 +158,7 @@ attributes for convenience::
>>> sorted(eps.groups) # doctest: +SKIP
['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation']
-:class:`!EntryPoints` has a :meth:`!~EntryPoints.select` method to select entry points
+:class:`!EntryPoints` has a :meth:`!select` method to select entry points
matching specific properties. Select entry points in the
``console_scripts`` group::
@@ -230,7 +230,7 @@ Distribution metadata
`PackageMetadata protocol `_.
In addition to providing the defined protocol methods and attributes, subscripting
- the instance is equivalent to calling the :meth:`!~PackageMetadata.get` method.
+ the instance is equivalent to calling the :meth:`!get` method.
Every `Distribution Package `_
includes some metadata, which you can extract using the :func:`!metadata` function::
@@ -243,7 +243,7 @@ the values are returned unparsed from the distribution metadata::
>>> wheel_metadata['Requires-Python'] # doctest: +SKIP
'>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*'
-:class:`PackageMetadata` also presents a :attr:`!~PackageMetadata.json` attribute that returns
+:class:`PackageMetadata` also presents a :attr:`!json` attribute that returns
all the metadata in a JSON-compatible form per :PEP:`566`::
>>> wheel_metadata.json['requires_python']
@@ -329,7 +329,7 @@ Once you have the file, you can also read its contents::
return s.encode('utf-8')
return s
-You can also use the :meth:`!~PackagePath.locate` method to get the absolute
+You can also use the :meth:`!locate` method to get the absolute
path to the file::
>>> util.locate() # doctest: +SKIP
From a62d2d3318fcd5a9ac5c062360f73e511a4375e1 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 7 Oct 2024 19:59:31 +0200
Subject: [PATCH 018/269] [3.12] gh-124182: Explain naming rules for struct
sequence types (GH-124335) (#125056)
gh-124182: Explain naming rules for struct sequence types (GH-124335)
(cherry picked from commit 3287c834e5370294e310450115290979aac06efa)
Co-authored-by: ffelixg <142172984+ffelixg@users.noreply.github.com>
---
Doc/c-api/tuple.rst | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst
index 82ef4bcd14784d..e20c5a66189be0 100644
--- a/Doc/c-api/tuple.rst
+++ b/Doc/c-api/tuple.rst
@@ -158,7 +158,8 @@ type.
.. c:member:: const char *name
- Name of the struct sequence type.
+ Fully qualified name of the type; null-terminated UTF-8 encoded.
+ The name must contain the module name.
.. c:member:: const char *doc
From a01970d6a2add5a07af58b0aa3ca280bad62b9fd Mon Sep 17 00:00:00 2001
From: "T. Wouters"
Date: Mon, 7 Oct 2024 11:51:07 -0700
Subject: [PATCH 019/269] [3.12] GH-109975: Announce final release in What's
New in Python 3.13 (GH-125007) (#125034)
Prepare What's New in Python 3.13 for final release
(cherry picked from commit 31516c98dd7097047ba10da8dcf728c3d580f3d6)
Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
---
Doc/whatsnew/3.12.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst
index 09f47b7041e9e8..5c36e2d4bcfb49 100644
--- a/Doc/whatsnew/3.12.rst
+++ b/Doc/whatsnew/3.12.rst
@@ -59,7 +59,7 @@ Summary -- Release highlights
.. This section singles out the most important changes in Python 3.12.
Brevity is key.
-Python 3.12 is the latest stable release of the Python programming language,
+Python 3.12 is a stable release of the Python programming language,
with a mix of changes to the language and the standard library.
The library changes focus on cleaning up deprecated APIs, usability, and correctness.
Of note, the :mod:`!distutils` package has been removed from the standard library.
From 79b4c9d99a070a6cdcb810148e514864d8958bc1 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 7 Oct 2024 21:23:08 +0200
Subject: [PATCH 020/269] [3.12] gh-122392: IDLE - Fix overlapping lines in
browsers (GH-122392) (GH-124975) (#125062)
gh-122392: IDLE - Fix overlapping lines in browsers (GH-122392) (GH-124975)
Increase currently inadequate vertical spacing for the IDLE browsers (path,
module, and stack) on high-resolution monitors.
---------
(cherry picked from commit c5df1cb7bde7e86f046196b0e34a0b90f8fc11de)
Co-authored-by: Zhikang Yan <2951256653@qq.com>
Co-authored-by: Terry Jan Reedy
---
Lib/idlelib/tree.py | 14 ++++++++++----
.../2024-10-04-15-34-34.gh-issue-122392.V8K3w2.rst | 2 ++
2 files changed, 12 insertions(+), 4 deletions(-)
create mode 100644 Misc/NEWS.d/next/IDLE/2024-10-04-15-34-34.gh-issue-122392.V8K3w2.rst
diff --git a/Lib/idlelib/tree.py b/Lib/idlelib/tree.py
index 0726d7e23660f6..182ce7189614da 100644
--- a/Lib/idlelib/tree.py
+++ b/Lib/idlelib/tree.py
@@ -83,6 +83,8 @@ def wheel_event(event, widget=None):
class TreeNode:
+ dy = 0
+
def __init__(self, canvas, parent, item):
self.canvas = canvas
self.parent = parent
@@ -199,23 +201,22 @@ def update(self):
def draw(self, x, y):
# XXX This hard-codes too many geometry constants!
- dy = 20
self.x, self.y = x, y
self.drawicon()
self.drawtext()
if self.state != 'expanded':
- return y + dy
+ return y + TreeNode.dy
# draw children
if not self.children:
sublist = self.item._GetSubList()
if not sublist:
# _IsExpandable() was mistaken; that's allowed
- return y+17
+ return y + TreeNode.dy
for item in sublist:
child = self.__class__(self.canvas, self, item)
self.children.append(child)
cx = x+20
- cy = y + dy
+ cy = y + TreeNode.dy
cylast = 0
for child in self.children:
cylast = cy
@@ -289,6 +290,11 @@ def drawtext(self):
self.label.bind("", lambda e: wheel_event(e, self.canvas))
self.label.bind("", lambda e: wheel_event(e, self.canvas))
self.text_id = id
+ if TreeNode.dy == 0:
+ # The first row doesn't matter what the dy is, just measure its
+ # size to get the value of the subsequent dy
+ coords = self.canvas.bbox(id)
+ TreeNode.dy = max(20, coords[3] - coords[1] - 3)
def select_or_edit(self, event=None):
if self.selected and self.item.IsEditable():
diff --git a/Misc/NEWS.d/next/IDLE/2024-10-04-15-34-34.gh-issue-122392.V8K3w2.rst b/Misc/NEWS.d/next/IDLE/2024-10-04-15-34-34.gh-issue-122392.V8K3w2.rst
new file mode 100644
index 00000000000000..541f6212794ef2
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2024-10-04-15-34-34.gh-issue-122392.V8K3w2.rst
@@ -0,0 +1,2 @@
+Increase currently inadequate vertical spacing for the IDLE browsers (path,
+module, and stack) on high-resolution monitors.
From bc237ed9a8d631675a4962d6627d0571dfa4c04f Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 8 Oct 2024 08:24:09 +0200
Subject: [PATCH 021/269] [3.12] gh-124653: Relax (again) detection of queue
API for logging handlers (GH-124897) (GH-125060)
(cherry picked from commit 7ffe94fb242fd51bb07c7f0d31e94efeea3619d4)
---
Doc/library/logging.config.rst | 11 +-
Lib/logging/config.py | 14 +--
Lib/test/test_logging.py | 108 +++++++++++-------
...-10-02-15-05-45.gh-issue-124653.tqsTu9.rst | 2 +
4 files changed, 79 insertions(+), 56 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-02-15-05-45.gh-issue-124653.tqsTu9.rst
diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst
index 50ffff60250ee0..bc85f4301f9ba0 100644
--- a/Doc/library/logging.config.rst
+++ b/Doc/library/logging.config.rst
@@ -752,16 +752,17 @@ The ``queue`` and ``listener`` keys are optional.
If the ``queue`` key is present, the corresponding value can be one of the following:
-* An object implementing the :class:`queue.Queue` public API. For instance,
- this may be an actual instance of :class:`queue.Queue` or a subclass thereof,
- or a proxy obtained by :meth:`multiprocessing.managers.SyncManager.Queue`.
+* An object implementing the :meth:`Queue.put_nowait `
+ and :meth:`Queue.get ` public API. For instance, this may be
+ an actual instance of :class:`queue.Queue` or a subclass thereof, or a proxy
+ obtained by :meth:`multiprocessing.managers.SyncManager.Queue`.
This is of course only possible if you are constructing or modifying
the configuration dictionary in code.
* A string that resolves to a callable which, when called with no arguments, returns
- the :class:`queue.Queue` instance to use. That callable could be a
- :class:`queue.Queue` subclass or a function which returns a suitable queue instance,
+ the queue instance to use. That callable could be a :class:`queue.Queue` subclass
+ or a function which returns a suitable queue instance,
such as ``my.module.queue_factory()``.
* A dict with a ``'()'`` key which is constructed in the usual way as discussed in
diff --git a/Lib/logging/config.py b/Lib/logging/config.py
index ac90b537d8a396..c128bc7a61e34e 100644
--- a/Lib/logging/config.py
+++ b/Lib/logging/config.py
@@ -502,7 +502,7 @@ def as_tuple(self, value):
def _is_queue_like_object(obj):
"""Check that *obj* implements the Queue API."""
- if isinstance(obj, queue.Queue):
+ if isinstance(obj, (queue.Queue, queue.SimpleQueue)):
return True
# defer importing multiprocessing as much as possible
from multiprocessing.queues import Queue as MPQueue
@@ -519,13 +519,13 @@ def _is_queue_like_object(obj):
# Ideally, we would have wanted to simply use strict type checking
# instead of a protocol-based type checking since the latter does
# not check the method signatures.
- queue_interface = [
- 'empty', 'full', 'get', 'get_nowait',
- 'put', 'put_nowait', 'join', 'qsize',
- 'task_done',
- ]
+ #
+ # Note that only 'put_nowait' and 'get' are required by the logging
+ # queue handler and queue listener (see gh-124653) and that other
+ # methods are either optional or unused.
+ minimal_queue_interface = ['put_nowait', 'get']
return all(callable(getattr(obj, method, None))
- for method in queue_interface)
+ for method in minimal_queue_interface)
class DictConfigurator(BaseConfigurator):
"""
diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
index 78bcd065ad5d72..5112c2e7d60f0b 100644
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -2391,16 +2391,22 @@ def __getattr__(self, attribute):
return getattr(queue, attribute)
class CustomQueueFakeProtocol(CustomQueueProtocol):
- # An object implementing the Queue API (incorrect signatures).
+ # An object implementing the minimial Queue API for
+ # the logging module but with incorrect signatures.
+ #
# The object will be considered a valid queue class since we
# do not check the signatures (only callability of methods)
# but will NOT be usable in production since a TypeError will
- # be raised due to a missing argument.
- def empty(self, x):
+ # be raised due to the extra argument in 'put_nowait'.
+ def put_nowait(self):
pass
class CustomQueueWrongProtocol(CustomQueueProtocol):
- empty = None
+ put_nowait = None
+
+class MinimalQueueProtocol:
+ def put_nowait(self, x): pass
+ def get(self): pass
def queueMaker():
return queue.Queue()
@@ -3914,56 +3920,70 @@ def test_config_queue_handler(self):
msg = str(ctx.exception)
self.assertEqual(msg, "Unable to configure handler 'ah'")
+ def _apply_simple_queue_listener_configuration(self, qspec):
+ self.apply_config({
+ "version": 1,
+ "handlers": {
+ "queue_listener": {
+ "class": "logging.handlers.QueueHandler",
+ "queue": qspec,
+ },
+ },
+ })
+
@threading_helper.requires_working_threading()
@support.requires_subprocess()
@patch("multiprocessing.Manager")
def test_config_queue_handler_does_not_create_multiprocessing_manager(self, manager):
- # gh-120868, gh-121723
-
- from multiprocessing import Queue as MQ
-
- q1 = {"()": "queue.Queue", "maxsize": -1}
- q2 = MQ()
- q3 = queue.Queue()
- # CustomQueueFakeProtocol passes the checks but will not be usable
- # since the signatures are incompatible. Checking the Queue API
- # without testing the type of the actual queue is a trade-off
- # between usability and the work we need to do in order to safely
- # check that the queue object correctly implements the API.
- q4 = CustomQueueFakeProtocol()
-
- for qspec in (q1, q2, q3, q4):
- self.apply_config(
- {
- "version": 1,
- "handlers": {
- "queue_listener": {
- "class": "logging.handlers.QueueHandler",
- "queue": qspec,
- },
- },
- }
- )
- manager.assert_not_called()
+ # gh-120868, gh-121723, gh-124653
+
+ for qspec in [
+ {"()": "queue.Queue", "maxsize": -1},
+ queue.Queue(),
+ # queue.SimpleQueue does not inherit from queue.Queue
+ queue.SimpleQueue(),
+ # CustomQueueFakeProtocol passes the checks but will not be usable
+ # since the signatures are incompatible. Checking the Queue API
+ # without testing the type of the actual queue is a trade-off
+ # between usability and the work we need to do in order to safely
+ # check that the queue object correctly implements the API.
+ CustomQueueFakeProtocol(),
+ MinimalQueueProtocol(),
+ ]:
+ with self.subTest(qspec=qspec):
+ self._apply_simple_queue_listener_configuration(qspec)
+ manager.assert_not_called()
@patch("multiprocessing.Manager")
def test_config_queue_handler_invalid_config_does_not_create_multiprocessing_manager(self, manager):
# gh-120868, gh-121723
for qspec in [object(), CustomQueueWrongProtocol()]:
- with self.assertRaises(ValueError):
- self.apply_config(
- {
- "version": 1,
- "handlers": {
- "queue_listener": {
- "class": "logging.handlers.QueueHandler",
- "queue": qspec,
- },
- },
- }
- )
- manager.assert_not_called()
+ with self.subTest(qspec=qspec), self.assertRaises(ValueError):
+ self._apply_simple_queue_listener_configuration(qspec)
+ manager.assert_not_called()
+
+ @skip_if_tsan_fork
+ @support.requires_subprocess()
+ @unittest.skipUnless(support.Py_DEBUG, "requires a debug build for testing"
+ " assertions in multiprocessing")
+ def test_config_reject_simple_queue_handler_multiprocessing_context(self):
+ # multiprocessing.SimpleQueue does not implement 'put_nowait'
+ # and thus cannot be used as a queue-like object (gh-124653)
+
+ import multiprocessing
+
+ if support.MS_WINDOWS:
+ start_methods = ['spawn']
+ else:
+ start_methods = ['spawn', 'fork', 'forkserver']
+
+ for start_method in start_methods:
+ with self.subTest(start_method=start_method):
+ ctx = multiprocessing.get_context(start_method)
+ qspec = ctx.SimpleQueue()
+ with self.assertRaises(ValueError):
+ self._apply_simple_queue_listener_configuration(qspec)
@skip_if_tsan_fork
@support.requires_subprocess()
diff --git a/Misc/NEWS.d/next/Library/2024-10-02-15-05-45.gh-issue-124653.tqsTu9.rst b/Misc/NEWS.d/next/Library/2024-10-02-15-05-45.gh-issue-124653.tqsTu9.rst
new file mode 100644
index 00000000000000..6f5ad12d2c2981
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-02-15-05-45.gh-issue-124653.tqsTu9.rst
@@ -0,0 +1,2 @@
+Fix detection of the minimal Queue API needed by the :mod:`logging` module.
+Patch by Bénédikt Tran.
From 82257374b9d10bc3df2a466fe4450a5dc576842d Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka
Date: Tue, 8 Oct 2024 12:47:37 +0300
Subject: [PATCH 022/269] [3.12] gh-53203: Improve tests for strptime()
(GH-125090) (GH-125093)
Run them with different locales and different date and time.
Add the @run_with_locales() decorator to run the test with multiple
locales.
Improve the run_with_locale() context manager/decorator -- it now
catches only expected exceptions and reports the test as skipped if no
appropriate locale is available.
(cherry picked from commit 19984fe024bfd90649f1c36b78c9abf3ed72b27d)
---
Lib/test/pickletester.py | 4 +-
Lib/test/support/__init__.py | 53 +++++++++-
Lib/test/test_codecs.py | 9 +-
Lib/test/test_decimal.py | 2 +-
Lib/test/test_float.py | 2 +-
Lib/test/test_imaplib.py | 2 +-
Lib/test/test_strptime.py | 197 +++++++++++++++++++++++++----------
Lib/test/test_time.py | 12 +--
Lib/test/test_types.py | 4 +-
Lib/test/test_unicode.py | 2 +-
10 files changed, 201 insertions(+), 86 deletions(-)
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index b0968afcf683f3..dd7466e2c150d6 100644
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -26,7 +26,7 @@
from test import support
from test.support import os_helper
from test.support import (
- TestFailed, run_with_locale, no_tracing,
+ TestFailed, run_with_locales, no_tracing,
_2G, _4G, bigmemtest
)
from test.support.import_helper import forget
@@ -2591,7 +2591,7 @@ def test_float(self):
got = self.loads(pickle)
self.assert_is_copy(value, got)
- @run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
+ @run_with_locales('LC_ALL', 'de_DE', 'fr_FR', '')
def test_float_format(self):
# make sure that floats are formatted locale independent with proto 0
self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.')
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 8519fedf8dbc9d..9c3dcbc1d2bc29 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -854,8 +854,8 @@ def check_sizeof(test, o, size):
test.assertEqual(result, size, msg)
#=======================================================================
-# Decorator for running a function in a different locale, correctly resetting
-# it afterwards.
+# Decorator/context manager for running a code in a different locale,
+# correctly resetting it afterwards.
@contextlib.contextmanager
def run_with_locale(catstr, *locales):
@@ -866,16 +866,21 @@ def run_with_locale(catstr, *locales):
except AttributeError:
# if the test author gives us an invalid category string
raise
- except:
+ except Exception:
# cannot retrieve original locale, so do nothing
locale = orig_locale = None
+ if '' not in locales:
+ raise unittest.SkipTest('no locales')
else:
for loc in locales:
try:
locale.setlocale(category, loc)
break
- except:
+ except locale.Error:
pass
+ else:
+ if '' not in locales:
+ raise unittest.SkipTest(f'no locales {locales}')
try:
yield
@@ -883,6 +888,46 @@ def run_with_locale(catstr, *locales):
if locale and orig_locale:
locale.setlocale(category, orig_locale)
+#=======================================================================
+# Decorator for running a function in multiple locales (if they are
+# availasble) and resetting the original locale afterwards.
+
+def run_with_locales(catstr, *locales):
+ def deco(func):
+ @functools.wraps(func)
+ def wrapper(self, /, *args, **kwargs):
+ dry_run = '' in locales
+ try:
+ import locale
+ category = getattr(locale, catstr)
+ orig_locale = locale.setlocale(category)
+ except AttributeError:
+ # if the test author gives us an invalid category string
+ raise
+ except Exception:
+ # cannot retrieve original locale, so do nothing
+ pass
+ else:
+ try:
+ for loc in locales:
+ with self.subTest(locale=loc):
+ try:
+ locale.setlocale(category, loc)
+ except locale.Error:
+ self.skipTest(f'no locale {loc!r}')
+ else:
+ dry_run = False
+ func(self, *args, **kwargs)
+ finally:
+ locale.setlocale(category, orig_locale)
+ if dry_run:
+ # no locales available, so just run the test
+ # with the current locale
+ with self.subTest(locale=None):
+ func(self, *args, **kwargs)
+ return wrapper
+ return deco
+
#=======================================================================
# Decorator for running a function in a specific timezone, correctly
# resetting it afterwards.
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index 5dc5b1acf10f74..f683f069ae1397 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -2,7 +2,6 @@
import contextlib
import copy
import io
-import locale
import pickle
import sys
import unittest
@@ -1700,16 +1699,10 @@ def test_getwriter(self):
self.assertRaises(TypeError, codecs.getwriter)
self.assertRaises(LookupError, codecs.getwriter, "__spam__")
+ @support.run_with_locale('LC_CTYPE', 'tr_TR')
def test_lookup_issue1813(self):
# Issue #1813: under Turkish locales, lookup of some codecs failed
# because 'I' is lowercased as "ı" (dotless i)
- oldlocale = locale.setlocale(locale.LC_CTYPE)
- self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale)
- try:
- locale.setlocale(locale.LC_CTYPE, 'tr_TR')
- except locale.Error:
- # Unsupported locale on this system
- self.skipTest('test needs Turkish locale')
c = codecs.lookup('ASCII')
self.assertEqual(c.name, 'ascii')
diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py
index eab4881e2cba62..2ec12439d7fee4 100644
--- a/Lib/test/test_decimal.py
+++ b/Lib/test/test_decimal.py
@@ -1242,7 +1242,7 @@ def get_fmt(x, override=None, fmt='n'):
self.assertEqual(get_fmt(Decimal('-1.5'), dotsep_wide, '020n'),
'-0\u00b4000\u00b4000\u00b4000\u00b4001\u00bf5')
- @run_with_locale('LC_ALL', 'ps_AF')
+ @run_with_locale('LC_ALL', 'ps_AF', '')
def test_wide_char_separator_decimal_point(self):
# locale with wide char separator and decimal point
Decimal = self.decimal.Decimal
diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py
index 9afaa55765e708..0c1adabdd58852 100644
--- a/Lib/test/test_float.py
+++ b/Lib/test/test_float.py
@@ -153,7 +153,7 @@ def check(s):
# non-UTF-8 byte string
check(b'123\xa0')
- @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE')
+ @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE', '')
def test_float_with_comma(self):
# set locale to something that doesn't use '.' for the decimal point
# float must not accept the locale specific decimal point but
diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py
index 69f995f483c354..1d701281b839e8 100644
--- a/Lib/test/test_imaplib.py
+++ b/Lib/test/test_imaplib.py
@@ -57,7 +57,7 @@ def timevalues(self):
timezone(timedelta(0, 2 * 60 * 60))),
'"18-May-2033 05:33:20 +0200"']
- @run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
+ @run_with_locale('LC_ALL', 'de_DE', 'fr_FR', '')
# DST rules included to work around quirk where the Gnu C library may not
# otherwise restore the previous time zone
@run_with_tz('STD-1DST,M3.2.0,M11.1.0')
diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py
index 05c8afc907ad3c..66a9815e69b9c2 100644
--- a/Lib/test/test_strptime.py
+++ b/Lib/test/test_strptime.py
@@ -7,7 +7,7 @@
import os
import sys
from test import support
-from test.support import skip_if_buggy_ucrt_strfptime
+from test.support import skip_if_buggy_ucrt_strfptime, run_with_locales
from datetime import date as datetime_date
import _strptime
@@ -206,8 +206,8 @@ class StrptimeTests(unittest.TestCase):
"""Tests for _strptime.strptime."""
def setUp(self):
- """Create testing time tuple."""
- self.time_tuple = time.gmtime()
+ """Create testing time tuples."""
+ self.time_tuple = time.localtime()
def test_ValueError(self):
# Make sure ValueError is raised when match fails or format is bad
@@ -288,53 +288,67 @@ def test_unconverteddata(self):
# Check ValueError is raised when there is unconverted data
self.assertRaises(ValueError, _strptime._strptime_time, "10 12", "%m")
- def helper(self, directive, position):
+ def roundtrip(self, fmt, position, time_tuple=None):
"""Helper fxn in testing."""
- strf_output = time.strftime("%" + directive, self.time_tuple)
- strp_output = _strptime._strptime_time(strf_output, "%" + directive)
- self.assertTrue(strp_output[position] == self.time_tuple[position],
- "testing of '%s' directive failed; '%s' -> %s != %s" %
- (directive, strf_output, strp_output[position],
- self.time_tuple[position]))
+ if time_tuple is None:
+ time_tuple = self.time_tuple
+ strf_output = time.strftime(fmt, time_tuple)
+ strp_output = _strptime._strptime_time(strf_output, fmt)
+ self.assertEqual(strp_output[position], time_tuple[position],
+ "testing of %r format failed; %r -> %r != %r" %
+ (fmt, strf_output, strp_output[position],
+ time_tuple[position]))
+ if support.verbose >= 3:
+ print("testing of %r format: %r -> %r" %
+ (fmt, strf_output, strp_output[position]))
def test_year(self):
# Test that the year is handled properly
- for directive in ('y', 'Y'):
- self.helper(directive, 0)
+ self.roundtrip('%Y', 0)
+ self.roundtrip('%y', 0)
+ self.roundtrip('%Y', 0, (1900, 1, 1, 0, 0, 0, 0, 1, 0))
+
# Must also make sure %y values are correct for bounds set by Open Group
- for century, bounds in ((1900, ('69', '99')), (2000, ('00', '68'))):
- for bound in bounds:
- strp_output = _strptime._strptime_time(bound, '%y')
- expected_result = century + int(bound)
- self.assertTrue(strp_output[0] == expected_result,
- "'y' test failed; passed in '%s' "
- "and returned '%s'" % (bound, strp_output[0]))
+ strptime = _strptime._strptime_time
+ self.assertEqual(strptime('00', '%y')[0], 2000)
+ self.assertEqual(strptime('68', '%y')[0], 2068)
+ self.assertEqual(strptime('69', '%y')[0], 1969)
+ self.assertEqual(strptime('99', '%y')[0], 1999)
def test_month(self):
# Test for month directives
- for directive in ('B', 'b', 'm'):
- self.helper(directive, 1)
+ self.roundtrip('%m', 1)
+
+ @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', 'he_IL', '')
+ def test_month_locale(self):
+ # Test for month directives
+ self.roundtrip('%B', 1)
+ self.roundtrip('%b', 1)
+ for m in range(1, 13):
+ self.roundtrip('%B', 1, (1900, m, 1, 0, 0, 0, 0, 1, 0))
+ self.roundtrip('%b', 1, (1900, m, 1, 0, 0, 0, 0, 1, 0))
def test_day(self):
# Test for day directives
- self.helper('d', 2)
+ self.roundtrip('%d %Y', 2)
def test_hour(self):
# Test hour directives
- self.helper('H', 3)
- strf_output = time.strftime("%I %p", self.time_tuple)
- strp_output = _strptime._strptime_time(strf_output, "%I %p")
- self.assertTrue(strp_output[3] == self.time_tuple[3],
- "testing of '%%I %%p' directive failed; '%s' -> %s != %s" %
- (strf_output, strp_output[3], self.time_tuple[3]))
+ self.roundtrip('%H', 3)
+
+ # NB: Only works on locales with AM/PM
+ @run_with_locales('LC_TIME', 'C', 'en_US', 'ja_JP')
+ def test_hour_locale(self):
+ # Test hour directives
+ self.roundtrip('%I %p', 3)
def test_minute(self):
# Test minute directives
- self.helper('M', 4)
+ self.roundtrip('%M', 4)
def test_second(self):
# Test second directives
- self.helper('S', 5)
+ self.roundtrip('%S', 5)
def test_fraction(self):
# Test microseconds
@@ -345,12 +359,18 @@ def test_fraction(self):
def test_weekday(self):
# Test weekday directives
- for directive in ('A', 'a', 'w', 'u'):
- self.helper(directive,6)
+ self.roundtrip('%w', 6)
+ self.roundtrip('%u', 6)
+
+ @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', '')
+ def test_weekday_locale(self):
+ # Test weekday directives
+ self.roundtrip('%A', 6)
+ self.roundtrip('%a', 6)
def test_julian(self):
# Test julian directives
- self.helper('j', 7)
+ self.roundtrip('%j', 7)
def test_offset(self):
one_hour = 60 * 60
@@ -447,20 +467,96 @@ def test_bad_timezone(self):
"time.daylight set to %s and passing in %s" %
(time.tzname, tz_value, time.daylight, tz_name))
- def test_date_time(self):
+ # NB: Does not roundtrip in some locales due to the ambiguity of
+ # the date and time representation (bugs in locales?):
+ # * Seconds are not included: bem_ZM, bokmal, ff_SN, nb_NO, nn_NO,
+ # no_NO, norwegian, nynorsk.
+ # * Hours are in 12-hour notation without AM/PM indication: hy_AM,
+ # id_ID, ms_MY.
+ # * Year is not included: ha_NG.
+ # * Use non-Gregorian calendar: lo_LA, thai, th_TH.
+ #
+ # BUG: Generates invalid regexp for br_FR, csb_PL, Arabic.
+ # BUG: Generates regexp that does not match the current date and time
+ # for fa_IR, gez_ER, gez_ET, lzh_TW, my_MM, or_IN, shn_MM, yo_NG.
+ # BUG: Generates regexp that does not match the current date and time
+ # for fa_IR, gez_ER, gez_ET, lzh_TW, my_MM, or_IN, shn_MM, yo_NG,
+ # fr_FR, ja_JP, he_IL, ko_KR, zh_CN, etc.
+ @run_with_locales('LC_TIME', 'C', 'en_US', 'de_DE',
+ 'eu_ES', 'mfe_MU')
+ def test_date_time_locale(self):
# Test %c directive
- for position in range(6):
- self.helper('c', position)
-
- def test_date(self):
+ now = time.time()
+ self.roundtrip('%c', slice(0, 6), time.localtime(now))
+ # 1 hour 20 minutes 30 seconds ago
+ self.roundtrip('%c', slice(0, 6), time.localtime(now - 4830))
+ # 12 hours ago
+ self.roundtrip('%c', slice(0, 6), time.localtime(now - 12*3600))
+ # different days of the week
+ for i in range(1, 7):
+ self.roundtrip('%c', slice(0, 6), time.localtime(now - i*24*3600))
+ # different months
+ for i in range(1, 12):
+ self.roundtrip('%c', slice(0, 6), time.localtime(now - i*30*24*3600))
+ # different year
+ self.roundtrip('%c', slice(0, 6), time.localtime(now - 366*24*3600))
+
+ # NB: Dates before 1969 do not roundtrip on some locales:
+ # bo_CN, bo_IN, dz_BT, eu_ES, eu_FR.
+ @run_with_locales('LC_TIME', 'C', 'en_US', 'de_DE', 'ja_JP')
+ def test_date_time_locale2(self):
+ # Test %c directive
+ self.roundtrip('%c', slice(0, 6), (1900, 1, 1, 0, 0, 0, 0, 1, 0))
+
+ # NB: Does not roundtrip because use non-Gregorian calendar:
+ # lo_LA, thai, th_TH.
+ # BUG: Generates regexp that does not match the current date
+ # for az_IR, fa_IR, lzh_TW, my_MM, or_IN, shn_MM,
+ # Arabic, ja_JP, ko_KR, zh_CN, etc.
+ @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE',
+ 'he_IL', 'eu_ES')
+ def test_date_locale(self):
# Test %x directive
- for position in range(0,3):
- self.helper('x', position)
-
- def test_time(self):
+ now = time.time()
+ self.roundtrip('%x', slice(0, 3), time.localtime(now))
+ # different days of the week
+ for i in range(1, 7):
+ self.roundtrip('%x', slice(0, 3), time.localtime(now - i*24*3600))
+ # different months
+ for i in range(1, 12):
+ self.roundtrip('%x', slice(0, 3), time.localtime(now - i*30*24*3600))
+ # different year
+ self.roundtrip('%x', slice(0, 3), time.localtime(now - 366*24*3600))
+
+ # NB: Dates before 1969 do not roundtrip on many locales, including C.
+ @unittest.skipIf(
+ support.is_emscripten or support.is_wasi,
+ "musl libc issue on Emscripten, bpo-46390"
+ )
+ @run_with_locales('LC_TIME', 'en_US', 'fr_FR', 'de_DE', 'ja_JP')
+ def test_date_locale2(self):
+ # Test %x directive
+ self.roundtrip('%x', slice(0, 3), (1900, 1, 1, 0, 0, 0, 0, 1, 0))
+
+ # NB: Does not roundtrip in some locales due to the ambiguity of
+ # the time representation (bugs in locales?):
+ # * Seconds are not included: bokmal, ff_SN, nb_NO, nn_NO, no_NO,
+ # norwegian, nynorsk.
+ # * Hours are in 12-hour notation without AM/PM indication: hy_AM,
+ # ms_MY, sm_WS.
+ # BUG: Generates regexp that does not match the current time for
+ # aa_DJ, aa_ER, aa_ET, am_ET, az_IR, byn_ER, fa_IR, gez_ER, gez_ET,
+ # lzh_TW, my_MM, om_ET, om_KE, or_IN, shn_MM, sid_ET, so_DJ, so_ET,
+ # so_SO, ti_ER, ti_ET, tig_ER, wal_ET.
+ @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP')
+ def test_time_locale(self):
# Test %X directive
- for position in range(3,6):
- self.helper('X', position)
+ now = time.time()
+ self.roundtrip('%X', slice(3, 6), time.localtime(now))
+ # 1 hour 20 minutes 30 seconds ago
+ self.roundtrip('%X', slice(3, 6), time.localtime(now - 4830))
+ # 12 hours ago
+ self.roundtrip('%X', slice(3, 6), time.localtime(now - 12*3600))
def test_percent(self):
# Make sure % signs are handled properly
@@ -710,13 +806,8 @@ def test_new_localetime(self):
def test_TimeRE_recreation_locale(self):
# The TimeRE instance should be recreated upon changing the locale.
- locale_info = locale.getlocale(locale.LC_TIME)
- try:
- locale.setlocale(locale.LC_TIME, ('en_US', 'UTF8'))
- except locale.Error:
- self.skipTest('test needs en_US.UTF8 locale')
- try:
- _strptime._strptime_time('10', '%d')
+ with support.run_with_locale('LC_TIME', 'en_US.UTF8'):
+ _strptime._strptime_time('10 2004', '%d %Y')
# Get id of current cache object.
first_time_re = _strptime._TimeRE_cache
try:
@@ -732,10 +823,6 @@ def test_TimeRE_recreation_locale(self):
# to the resetting to the original locale.
except locale.Error:
self.skipTest('test needs de_DE.UTF8 locale')
- # Make sure we don't trample on the locale setting once we leave the
- # test.
- finally:
- locale.setlocale(locale.LC_TIME, locale_info)
@support.run_with_tz('STD-1DST,M4.1.0,M10.1.0')
def test_TimeRE_recreation_timezone(self):
diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py
index 02cc3f43a66a67..921e4eea649d6b 100644
--- a/Lib/test/test_time.py
+++ b/Lib/test/test_time.py
@@ -2,7 +2,6 @@
from test.support import warnings_helper
import decimal
import enum
-import locale
import math
import platform
import sys
@@ -588,17 +587,8 @@ def test_get_clock_info(self):
class TestLocale(unittest.TestCase):
- def setUp(self):
- self.oldloc = locale.setlocale(locale.LC_ALL)
-
- def tearDown(self):
- locale.setlocale(locale.LC_ALL, self.oldloc)
-
+ @support.run_with_locale('LC_ALL', 'fr_FR', '')
def test_bug_3061(self):
- try:
- tmp = locale.setlocale(locale.LC_ALL, "fr_FR")
- except locale.Error:
- self.skipTest('could not set locale.LC_ALL to fr_FR')
# This should not cause an exception
time.strftime("%B", (2009,2,1,0,0,0,0,0,0))
diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py
index 43949770b16637..9b2a5728e9f534 100644
--- a/Lib/test/test_types.py
+++ b/Lib/test/test_types.py
@@ -397,7 +397,7 @@ def test(i, format_spec, result):
test(123456, "1=20", '11111111111111123456')
test(123456, "*=20", '**************123456')
- @run_with_locale('LC_NUMERIC', 'en_US.UTF8')
+ @run_with_locale('LC_NUMERIC', 'en_US.UTF8', '')
def test_float__format__locale(self):
# test locale support for __format__ code 'n'
@@ -406,7 +406,7 @@ def test_float__format__locale(self):
self.assertEqual(locale.format_string('%g', x, grouping=True), format(x, 'n'))
self.assertEqual(locale.format_string('%.10g', x, grouping=True), format(x, '.10n'))
- @run_with_locale('LC_NUMERIC', 'en_US.UTF8')
+ @run_with_locale('LC_NUMERIC', 'en_US.UTF8', '')
def test_int__format__locale(self):
# test locale support for __format__ code 'n' for integers
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
index e43be3578836ef..19556eecc3c646 100644
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -1676,7 +1676,7 @@ def test_startswith_endswith_errors(self):
self.assertIn('str', exc)
self.assertIn('tuple', exc)
- @support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
+ @support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR', '')
def test_format_float(self):
# should not format with a comma, but always with C locale
self.assertEqual('1.0', '%.1f' % 1.0)
From d3b437cb7816d0163dbc3495889e87ebc22e56a0 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 8 Oct 2024 13:56:18 +0200
Subject: [PATCH 023/269] [3.12] gh-123378: fix a crash in
`UnicodeError.__str__` (GH-124935) (#125098)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
gh-123378: fix a crash in `UnicodeError.__str__` (GH-124935)
(cherry picked from commit ba14dfafd97d1fd03938ac8ddec4ca5b2f12985d)
Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
---
Lib/test/test_exceptions.py | 24 ++++
...-10-03-14-39-41.gh-issue-123378.dCxANf.rst | 3 +
Objects/exceptions.c | 111 +++++++++++-------
3 files changed, 93 insertions(+), 45 deletions(-)
create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-03-14-39-41.gh-issue-123378.dCxANf.rst
diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index 5b0334f34652d2..c5f4b892efb50f 100644
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -8,6 +8,7 @@
import weakref
import errno
from codecs import BOM_UTF8
+from itertools import product
from textwrap import dedent
from test.support import (captured_stderr, check_impl_detail,
@@ -1333,6 +1334,29 @@ def test_unicode_errors_no_object(self):
for klass in klasses:
self.assertEqual(str(klass.__new__(klass)), "")
+ def test_unicode_error_str_does_not_crash(self):
+ # Test that str(UnicodeError(...)) does not crash.
+ # See https://github.com/python/cpython/issues/123378.
+
+ for start, end, objlen in product(
+ range(-5, 5),
+ range(-5, 5),
+ range(7),
+ ):
+ obj = 'a' * objlen
+ with self.subTest('encode', objlen=objlen, start=start, end=end):
+ exc = UnicodeEncodeError('utf-8', obj, start, end, '')
+ self.assertIsInstance(str(exc), str)
+
+ with self.subTest('translate', objlen=objlen, start=start, end=end):
+ exc = UnicodeTranslateError(obj, start, end, '')
+ self.assertIsInstance(str(exc), str)
+
+ encoded = obj.encode()
+ with self.subTest('decode', objlen=objlen, start=start, end=end):
+ exc = UnicodeDecodeError('utf-8', encoded, start, end, '')
+ self.assertIsInstance(str(exc), str)
+
@no_tracing
def test_badisinstance(self):
# Bug #2542: if issubclass(e, MyException) raises an exception,
diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-03-14-39-41.gh-issue-123378.dCxANf.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-03-14-39-41.gh-issue-123378.dCxANf.rst
new file mode 100644
index 00000000000000..5cd34535d674d3
--- /dev/null
+++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-03-14-39-41.gh-issue-123378.dCxANf.rst
@@ -0,0 +1,3 @@
+Fix a crash in the :meth:`~object.__str__` method of :exc:`UnicodeError`
+objects when the :attr:`UnicodeError.start` and :attr:`UnicodeError.end`
+values are invalid or out-of-range. Patch by Bénédikt Tran.
diff --git a/Objects/exceptions.c b/Objects/exceptions.c
index 4f2153b19358d2..c579563db75275 100644
--- a/Objects/exceptions.c
+++ b/Objects/exceptions.c
@@ -2961,46 +2961,55 @@ UnicodeEncodeError_init(PyObject *self, PyObject *args, PyObject *kwds)
static PyObject *
UnicodeEncodeError_str(PyObject *self)
{
- PyUnicodeErrorObject *uself = (PyUnicodeErrorObject *)self;
+ PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self;
PyObject *result = NULL;
PyObject *reason_str = NULL;
PyObject *encoding_str = NULL;
- if (!uself->object)
+ if (exc->object == NULL) {
/* Not properly initialized. */
return PyUnicode_FromString("");
+ }
/* Get reason and encoding as strings, which they might not be if
they've been modified after we were constructed. */
- reason_str = PyObject_Str(uself->reason);
- if (reason_str == NULL)
+ reason_str = PyObject_Str(exc->reason);
+ if (reason_str == NULL) {
goto done;
- encoding_str = PyObject_Str(uself->encoding);
- if (encoding_str == NULL)
+ }
+ encoding_str = PyObject_Str(exc->encoding);
+ if (encoding_str == NULL) {
goto done;
+ }
+
+ Py_ssize_t len = PyUnicode_GET_LENGTH(exc->object);
+ Py_ssize_t start = exc->start, end = exc->end;
- if (uself->start < PyUnicode_GET_LENGTH(uself->object) && uself->end == uself->start+1) {
- Py_UCS4 badchar = PyUnicode_ReadChar(uself->object, uself->start);
+ if ((start >= 0 && start < len) && (end >= 0 && end <= len) && end == start + 1) {
+ Py_UCS4 badchar = PyUnicode_ReadChar(exc->object, start);
const char *fmt;
- if (badchar <= 0xff)
+ if (badchar <= 0xff) {
fmt = "'%U' codec can't encode character '\\x%02x' in position %zd: %U";
- else if (badchar <= 0xffff)
+ }
+ else if (badchar <= 0xffff) {
fmt = "'%U' codec can't encode character '\\u%04x' in position %zd: %U";
- else
+ }
+ else {
fmt = "'%U' codec can't encode character '\\U%08x' in position %zd: %U";
+ }
result = PyUnicode_FromFormat(
fmt,
encoding_str,
(int)badchar,
- uself->start,
+ start,
reason_str);
}
else {
result = PyUnicode_FromFormat(
"'%U' codec can't encode characters in position %zd-%zd: %U",
encoding_str,
- uself->start,
- uself->end-1,
+ start,
+ end - 1,
reason_str);
}
done:
@@ -3074,41 +3083,46 @@ UnicodeDecodeError_init(PyObject *self, PyObject *args, PyObject *kwds)
static PyObject *
UnicodeDecodeError_str(PyObject *self)
{
- PyUnicodeErrorObject *uself = (PyUnicodeErrorObject *)self;
+ PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self;
PyObject *result = NULL;
PyObject *reason_str = NULL;
PyObject *encoding_str = NULL;
- if (!uself->object)
+ if (exc->object == NULL) {
/* Not properly initialized. */
return PyUnicode_FromString("");
+ }
/* Get reason and encoding as strings, which they might not be if
they've been modified after we were constructed. */
- reason_str = PyObject_Str(uself->reason);
- if (reason_str == NULL)
+ reason_str = PyObject_Str(exc->reason);
+ if (reason_str == NULL) {
goto done;
- encoding_str = PyObject_Str(uself->encoding);
- if (encoding_str == NULL)
+ }
+ encoding_str = PyObject_Str(exc->encoding);
+ if (encoding_str == NULL) {
goto done;
+ }
+
+ Py_ssize_t len = PyBytes_GET_SIZE(exc->object);
+ Py_ssize_t start = exc->start, end = exc->end;
- if (uself->start < PyBytes_GET_SIZE(uself->object) && uself->end == uself->start+1) {
- int byte = (int)(PyBytes_AS_STRING(((PyUnicodeErrorObject *)self)->object)[uself->start]&0xff);
+ if ((start >= 0 && start < len) && (end >= 0 && end <= len) && end == start + 1) {
+ int badbyte = (int)(PyBytes_AS_STRING(exc->object)[start] & 0xff);
result = PyUnicode_FromFormat(
"'%U' codec can't decode byte 0x%02x in position %zd: %U",
encoding_str,
- byte,
- uself->start,
+ badbyte,
+ start,
reason_str);
}
else {
result = PyUnicode_FromFormat(
"'%U' codec can't decode bytes in position %zd-%zd: %U",
encoding_str,
- uself->start,
- uself->end-1,
- reason_str
- );
+ start,
+ end - 1,
+ reason_str);
}
done:
Py_XDECREF(reason_str);
@@ -3171,42 +3185,49 @@ UnicodeTranslateError_init(PyUnicodeErrorObject *self, PyObject *args,
static PyObject *
UnicodeTranslateError_str(PyObject *self)
{
- PyUnicodeErrorObject *uself = (PyUnicodeErrorObject *)self;
+ PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self;
PyObject *result = NULL;
PyObject *reason_str = NULL;
- if (!uself->object)
+ if (exc->object == NULL) {
/* Not properly initialized. */
return PyUnicode_FromString("");
+ }
/* Get reason as a string, which it might not be if it's been
modified after we were constructed. */
- reason_str = PyObject_Str(uself->reason);
- if (reason_str == NULL)
+ reason_str = PyObject_Str(exc->reason);
+ if (reason_str == NULL) {
goto done;
+ }
+
+ Py_ssize_t len = PyUnicode_GET_LENGTH(exc->object);
+ Py_ssize_t start = exc->start, end = exc->end;
- if (uself->start < PyUnicode_GET_LENGTH(uself->object) && uself->end == uself->start+1) {
- Py_UCS4 badchar = PyUnicode_ReadChar(uself->object, uself->start);
+ if ((start >= 0 && start < len) && (end >= 0 && end <= len) && end == start + 1) {
+ Py_UCS4 badchar = PyUnicode_ReadChar(exc->object, start);
const char *fmt;
- if (badchar <= 0xff)
+ if (badchar <= 0xff) {
fmt = "can't translate character '\\x%02x' in position %zd: %U";
- else if (badchar <= 0xffff)
+ }
+ else if (badchar <= 0xffff) {
fmt = "can't translate character '\\u%04x' in position %zd: %U";
- else
+ }
+ else {
fmt = "can't translate character '\\U%08x' in position %zd: %U";
+ }
result = PyUnicode_FromFormat(
fmt,
(int)badchar,
- uself->start,
- reason_str
- );
- } else {
+ start,
+ reason_str);
+ }
+ else {
result = PyUnicode_FromFormat(
"can't translate characters in position %zd-%zd: %U",
- uself->start,
- uself->end-1,
- reason_str
- );
+ start,
+ end - 1,
+ reason_str);
}
done:
Py_XDECREF(reason_str);
From 65272a382b9536e1c53484a58f0f9c392e2ca872 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 8 Oct 2024 14:32:04 +0200
Subject: [PATCH 024/269] [3.12] Doc: Improve description of ``GET_LEN`` opcode
(GH-114583) (#125103)
Doc: Improve description of ``GET_LEN`` opcode (GH-114583)
(cherry picked from commit e8773e59a835d23b9648271e0eb79c1651581564)
Co-authored-by: Kirill Podoprigora
---
Doc/library/dis.rst | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst
index 82b4aa28857f41..f9f82d25b874e8 100644
--- a/Doc/library/dis.rst
+++ b/Doc/library/dis.rst
@@ -850,7 +850,8 @@ iterations of the loop.
.. opcode:: GET_LEN
- Perform ``STACK.append(len(STACK[-1]))``.
+ Perform ``STACK.append(len(STACK[-1]))``. Used in :keyword:`match` statements where
+ comparison with structure of pattern is needed.
.. versionadded:: 3.10
From 382ee1c7bd292f964f5d97384b93dfbdf7b64061 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 8 Oct 2024 20:27:50 +0200
Subject: [PATCH 025/269] [3.12] gh-124832: Add a note to indicate that
`datetime.now` may return the same instant (GH-124834) (#125146)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
gh-124832: Add a note to indicate that `datetime.now` may return the same instant (GH-124834)
* Update datetime.rst
* Update datetime.rst
replace warning with note
* Update Doc/library/datetime.rst
* Update Doc/library/datetime.rst
---------
(cherry picked from commit 760b1e103a0aa696cdf448e0d500cd1bac2213fa)
Co-authored-by: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com>
Co-authored-by: Victor Stinner
Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
---
Doc/library/datetime.rst | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst
index 8f5ea47da0db25..671554f2cf3d0d 100644
--- a/Doc/library/datetime.rst
+++ b/Doc/library/datetime.rst
@@ -896,6 +896,10 @@ Other constructors, all class methods:
This function is preferred over :meth:`today` and :meth:`utcnow`.
+ .. note::
+
+ Subsequent calls to :meth:`!datetime.now` may return the same
+ instant depending on the precision of the underlying clock.
.. classmethod:: datetime.utcnow()
From cf2532b39d099e004d1c07b2d0fcc46567b68e75 Mon Sep 17 00:00:00 2001
From: Raymond Hettinger
Date: Tue, 8 Oct 2024 15:16:18 -0500
Subject: [PATCH 026/269] [3.12] Tee of tee was not producing n independent
iterators (gh-123884) (gh-125153)
---
Doc/library/itertools.rst | 49 +++--
Lib/test/test_itertools.py | 171 +++++++++++++++++-
...-09-24-22-38-51.gh-issue-123884.iEPTK4.rst | 4 +
Modules/itertoolsmodule.c | 37 +---
4 files changed, 214 insertions(+), 47 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-09-24-22-38-51.gh-issue-123884.iEPTK4.rst
diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst
index 3fab46c3c0a5b4..047d805eda628a 100644
--- a/Doc/library/itertools.rst
+++ b/Doc/library/itertools.rst
@@ -676,24 +676,37 @@ loops that truncate the stream.
Roughly equivalent to::
def tee(iterable, n=2):
- iterator = iter(iterable)
- shared_link = [None, None]
- return tuple(_tee(iterator, shared_link) for _ in range(n))
-
- def _tee(iterator, link):
- try:
- while True:
- if link[1] is None:
- link[0] = next(iterator)
- link[1] = [None, None]
- value, link = link
- yield value
- except StopIteration:
- return
-
- Once a :func:`tee` has been created, the original *iterable* should not be
- used anywhere else; otherwise, the *iterable* could get advanced without
- the tee objects being informed.
+ if n < 0:
+ raise ValueError
+ if n == 0:
+ return ()
+ iterator = _tee(iterable)
+ result = [iterator]
+ for _ in range(n - 1):
+ result.append(_tee(iterator))
+ return tuple(result)
+
+ class _tee:
+
+ def __init__(self, iterable):
+ it = iter(iterable)
+ if isinstance(it, _tee):
+ self.iterator = it.iterator
+ self.link = it.link
+ else:
+ self.iterator = it
+ self.link = [None, None]
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ link = self.link
+ if link[1] is None:
+ link[0] = next(self.iterator)
+ link[1] = [None, None]
+ value, self.link = link
+ return value
``tee`` iterators are not threadsafe. A :exc:`RuntimeError` may be
raised when simultaneously using iterators returned by the same :func:`tee`
diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py
index 3d20e70fc1b63f..b6404f4366ca0e 100644
--- a/Lib/test/test_itertools.py
+++ b/Lib/test/test_itertools.py
@@ -1612,10 +1612,11 @@ def test_tee(self):
self.assertEqual(len(result), n)
self.assertEqual([list(x) for x in result], [list('abc')]*n)
- # tee pass-through to copyable iterator
+ # tee objects are independent (see bug gh-123884)
a, b = tee('abc')
c, d = tee(a)
- self.assertTrue(a is c)
+ e, f = tee(c)
+ self.assertTrue(len({a, b, c, d, e, f}) == 6)
# test tee_new
t1, t2 = tee('abc')
@@ -2029,6 +2030,172 @@ def test_islice_recipe(self):
self.assertEqual(next(c), 3)
+ def test_tee_recipe(self):
+
+ # Begin tee() recipe ###########################################
+
+ def tee(iterable, n=2):
+ if n < 0:
+ raise ValueError
+ if n == 0:
+ return ()
+ iterator = _tee(iterable)
+ result = [iterator]
+ for _ in range(n - 1):
+ result.append(_tee(iterator))
+ return tuple(result)
+
+ class _tee:
+
+ def __init__(self, iterable):
+ it = iter(iterable)
+ if isinstance(it, _tee):
+ self.iterator = it.iterator
+ self.link = it.link
+ else:
+ self.iterator = it
+ self.link = [None, None]
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ link = self.link
+ if link[1] is None:
+ link[0] = next(self.iterator)
+ link[1] = [None, None]
+ value, self.link = link
+ return value
+
+ # End tee() recipe #############################################
+
+ n = 200
+
+ a, b = tee([]) # test empty iterator
+ self.assertEqual(list(a), [])
+ self.assertEqual(list(b), [])
+
+ a, b = tee(irange(n)) # test 100% interleaved
+ self.assertEqual(lzip(a,b), lzip(range(n), range(n)))
+
+ a, b = tee(irange(n)) # test 0% interleaved
+ self.assertEqual(list(a), list(range(n)))
+ self.assertEqual(list(b), list(range(n)))
+
+ a, b = tee(irange(n)) # test dealloc of leading iterator
+ for i in range(100):
+ self.assertEqual(next(a), i)
+ del a
+ self.assertEqual(list(b), list(range(n)))
+
+ a, b = tee(irange(n)) # test dealloc of trailing iterator
+ for i in range(100):
+ self.assertEqual(next(a), i)
+ del b
+ self.assertEqual(list(a), list(range(100, n)))
+
+ for j in range(5): # test randomly interleaved
+ order = [0]*n + [1]*n
+ random.shuffle(order)
+ lists = ([], [])
+ its = tee(irange(n))
+ for i in order:
+ value = next(its[i])
+ lists[i].append(value)
+ self.assertEqual(lists[0], list(range(n)))
+ self.assertEqual(lists[1], list(range(n)))
+
+ # test argument format checking
+ self.assertRaises(TypeError, tee)
+ self.assertRaises(TypeError, tee, 3)
+ self.assertRaises(TypeError, tee, [1,2], 'x')
+ self.assertRaises(TypeError, tee, [1,2], 3, 'x')
+
+ # tee object should be instantiable
+ a, b = tee('abc')
+ c = type(a)('def')
+ self.assertEqual(list(c), list('def'))
+
+ # test long-lagged and multi-way split
+ a, b, c = tee(range(2000), 3)
+ for i in range(100):
+ self.assertEqual(next(a), i)
+ self.assertEqual(list(b), list(range(2000)))
+ self.assertEqual([next(c), next(c)], list(range(2)))
+ self.assertEqual(list(a), list(range(100,2000)))
+ self.assertEqual(list(c), list(range(2,2000)))
+
+ # test invalid values of n
+ self.assertRaises(TypeError, tee, 'abc', 'invalid')
+ self.assertRaises(ValueError, tee, [], -1)
+
+ for n in range(5):
+ result = tee('abc', n)
+ self.assertEqual(type(result), tuple)
+ self.assertEqual(len(result), n)
+ self.assertEqual([list(x) for x in result], [list('abc')]*n)
+
+ # tee objects are independent (see bug gh-123884)
+ a, b = tee('abc')
+ c, d = tee(a)
+ e, f = tee(c)
+ self.assertTrue(len({a, b, c, d, e, f}) == 6)
+
+ # test tee_new
+ t1, t2 = tee('abc')
+ tnew = type(t1)
+ self.assertRaises(TypeError, tnew)
+ self.assertRaises(TypeError, tnew, 10)
+ t3 = tnew(t1)
+ self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc'))
+
+ # test that tee objects are weak referencable
+ a, b = tee(range(10))
+ p = weakref.proxy(a)
+ self.assertEqual(getattr(p, '__class__'), type(b))
+ del a
+ gc.collect() # For PyPy or other GCs.
+ self.assertRaises(ReferenceError, getattr, p, '__class__')
+
+ ans = list('abc')
+ long_ans = list(range(10000))
+
+ # Tests not applicable to the tee() recipe
+ if False:
+ # check copy
+ a, b = tee('abc')
+ self.assertEqual(list(copy.copy(a)), ans)
+ self.assertEqual(list(copy.copy(b)), ans)
+ a, b = tee(list(range(10000)))
+ self.assertEqual(list(copy.copy(a)), long_ans)
+ self.assertEqual(list(copy.copy(b)), long_ans)
+
+ # check partially consumed copy
+ a, b = tee('abc')
+ take(2, a)
+ take(1, b)
+ self.assertEqual(list(copy.copy(a)), ans[2:])
+ self.assertEqual(list(copy.copy(b)), ans[1:])
+ self.assertEqual(list(a), ans[2:])
+ self.assertEqual(list(b), ans[1:])
+ a, b = tee(range(10000))
+ take(100, a)
+ take(60, b)
+ self.assertEqual(list(copy.copy(a)), long_ans[100:])
+ self.assertEqual(list(copy.copy(b)), long_ans[60:])
+ self.assertEqual(list(a), long_ans[100:])
+ self.assertEqual(list(b), long_ans[60:])
+
+ # Issue 13454: Crash when deleting backward iterator from tee()
+ forward, backward = tee(repeat(None, 2000)) # 20000000
+ try:
+ any(forward) # exhaust the iterator
+ del backward
+ except:
+ del forward, backward
+ raise
+
+
class TestGC(unittest.TestCase):
def makecycle(self, iterator, container):
diff --git a/Misc/NEWS.d/next/Library/2024-09-24-22-38-51.gh-issue-123884.iEPTK4.rst b/Misc/NEWS.d/next/Library/2024-09-24-22-38-51.gh-issue-123884.iEPTK4.rst
new file mode 100644
index 00000000000000..55f1d4b41125c3
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-09-24-22-38-51.gh-issue-123884.iEPTK4.rst
@@ -0,0 +1,4 @@
+Fixed bug in itertools.tee() handling of other tee inputs (a tee in a tee).
+The output now has the promised *n* independent new iterators. Formerly,
+the first iterator was identical (not independent) to the input iterator.
+This would sometimes give surprising results.
diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c
index d42f9dd0768658..e87c753113563f 100644
--- a/Modules/itertoolsmodule.c
+++ b/Modules/itertoolsmodule.c
@@ -1137,7 +1137,7 @@ itertools_tee_impl(PyObject *module, PyObject *iterable, Py_ssize_t n)
/*[clinic end generated code: output=1c64519cd859c2f0 input=c99a1472c425d66d]*/
{
Py_ssize_t i;
- PyObject *it, *copyable, *copyfunc, *result;
+ PyObject *it, *to, *result;
if (n < 0) {
PyErr_SetString(PyExc_ValueError, "n must be >= 0");
@@ -1154,41 +1154,24 @@ itertools_tee_impl(PyObject *module, PyObject *iterable, Py_ssize_t n)
return NULL;
}
- if (_PyObject_LookupAttr(it, &_Py_ID(__copy__), ©func) < 0) {
- Py_DECREF(it);
+ (void)&_Py_ID(__copy__); // Retain a reference to __copy__
+ itertools_state *state = get_module_state(module);
+ to = tee_fromiterable(state, it);
+ Py_DECREF(it);
+ if (to == NULL) {
Py_DECREF(result);
return NULL;
}
- if (copyfunc != NULL) {
- copyable = it;
- }
- else {
- itertools_state *state = get_module_state(module);
- copyable = tee_fromiterable(state, it);
- Py_DECREF(it);
- if (copyable == NULL) {
- Py_DECREF(result);
- return NULL;
- }
- copyfunc = PyObject_GetAttr(copyable, &_Py_ID(__copy__));
- if (copyfunc == NULL) {
- Py_DECREF(copyable);
- Py_DECREF(result);
- return NULL;
- }
- }
- PyTuple_SET_ITEM(result, 0, copyable);
+ PyTuple_SET_ITEM(result, 0, to);
for (i = 1; i < n; i++) {
- copyable = _PyObject_CallNoArgs(copyfunc);
- if (copyable == NULL) {
- Py_DECREF(copyfunc);
+ to = tee_copy((teeobject *)to, NULL);
+ if (to == NULL) {
Py_DECREF(result);
return NULL;
}
- PyTuple_SET_ITEM(result, i, copyable);
+ PyTuple_SET_ITEM(result, i, to);
}
- Py_DECREF(copyfunc);
return result;
}
From 0dbad1d8b723e1a6a46784ce720ea474f693127e Mon Sep 17 00:00:00 2001
From: Savannah Ostrowski
Date: Tue, 8 Oct 2024 17:20:49 -0600
Subject: [PATCH 027/269] [3.12] GH-124478: Cleanup argparse documentation
(GH-124877) (#125164)
(cherry picked from commit 37228bd16e3ef97d32da08848552f7ef016d68ab)
Co-authored-by: Jelle Zijlstra
Co-authored-by: Tomas R
---
Doc/howto/argparse-optparse.rst | 55 ++++
Doc/library/argparse.rst | 548 +++++++++-----------------------
2 files changed, 201 insertions(+), 402 deletions(-)
create mode 100644 Doc/howto/argparse-optparse.rst
diff --git a/Doc/howto/argparse-optparse.rst b/Doc/howto/argparse-optparse.rst
new file mode 100644
index 00000000000000..cef2d893b28a62
--- /dev/null
+++ b/Doc/howto/argparse-optparse.rst
@@ -0,0 +1,55 @@
+.. currentmodule:: argparse
+
+.. _upgrading-optparse-code:
+
+==========================
+Upgrading optparse code
+==========================
+
+Originally, the :mod:`argparse` module had attempted to maintain compatibility
+with :mod:`optparse`. However, :mod:`optparse` was difficult to extend
+transparently, particularly with the changes required to support
+``nargs=`` specifiers and better usage messages. When most everything in
+:mod:`optparse` had either been copy-pasted over or monkey-patched, it no
+longer seemed practical to try to maintain the backwards compatibility.
+
+The :mod:`argparse` module improves on the :mod:`optparse`
+module in a number of ways including:
+
+* Handling positional arguments.
+* Supporting subcommands.
+* Allowing alternative option prefixes like ``+`` and ``/``.
+* Handling zero-or-more and one-or-more style arguments.
+* Producing more informative usage messages.
+* Providing a much simpler interface for custom ``type`` and ``action``.
+
+A partial upgrade path from :mod:`optparse` to :mod:`argparse`:
+
+* Replace all :meth:`optparse.OptionParser.add_option` calls with
+ :meth:`ArgumentParser.add_argument` calls.
+
+* Replace ``(options, args) = parser.parse_args()`` with ``args =
+ parser.parse_args()`` and add additional :meth:`ArgumentParser.add_argument`
+ calls for the positional arguments. Keep in mind that what was previously
+ called ``options``, now in the :mod:`argparse` context is called ``args``.
+
+* Replace :meth:`optparse.OptionParser.disable_interspersed_args`
+ by using :meth:`~ArgumentParser.parse_intermixed_args` instead of
+ :meth:`~ArgumentParser.parse_args`.
+
+* Replace callback actions and the ``callback_*`` keyword arguments with
+ ``type`` or ``action`` arguments.
+
+* Replace string names for ``type`` keyword arguments with the corresponding
+ type objects (e.g. int, float, complex, etc).
+
+* Replace :class:`optparse.Values` with :class:`Namespace` and
+ :exc:`optparse.OptionError` and :exc:`optparse.OptionValueError` with
+ :exc:`ArgumentError`.
+
+* Replace strings with implicit arguments such as ``%default`` or ``%prog`` with
+ the standard Python syntax to use dictionaries to format strings, that is,
+ ``%(default)s`` and ``%(prog)s``.
+
+* Replace the OptionParser constructor ``version`` argument with a call to
+ ``parser.add_argument('--version', action='version', version='')``.
diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index 632185b3f183f5..abe20d7d611695 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -1,4 +1,4 @@
-:mod:`!argparse` --- Parser for command-line options, arguments and sub-commands
+:mod:`!argparse` --- Parser for command-line options, arguments and subcommands
================================================================================
.. module:: argparse
@@ -19,17 +19,13 @@
introduction to Python command-line parsing, have a look at the
:ref:`argparse tutorial `.
-The :mod:`argparse` module makes it easy to write user-friendly command-line
-interfaces. The program defines what arguments it requires, and :mod:`argparse`
-will figure out how to parse those out of :data:`sys.argv`. The :mod:`argparse`
+The :mod:`!argparse` module makes it easy to write user-friendly command-line
+interfaces. The program defines what arguments it requires, and :mod:`!argparse`
+will figure out how to parse those out of :data:`sys.argv`. The :mod:`!argparse`
module also automatically generates help and usage messages. The module
will also issue errors when users give the program invalid arguments.
-
-Core Functionality
-------------------
-
-The :mod:`argparse` module's support for command-line interfaces is built
+The :mod:`!argparse` module's support for command-line interfaces is built
around an instance of :class:`argparse.ArgumentParser`. It is a container for
argument specifications and has options that apply to the parser as whole::
@@ -53,133 +49,9 @@ the extracted data in a :class:`argparse.Namespace` object::
args = parser.parse_args()
print(args.filename, args.count, args.verbose)
-
-Quick Links for add_argument()
-------------------------------
-
-============================ =========================================================== ==========================================================================================================================
-Name Description Values
-============================ =========================================================== ==========================================================================================================================
-action_ Specify how an argument should be handled ``'store'``, ``'store_const'``, ``'store_true'``, ``'append'``, ``'append_const'``, ``'count'``, ``'help'``, ``'version'``
-choices_ Limit values to a specific set of choices ``['foo', 'bar']``, ``range(1, 10)``, or :class:`~collections.abc.Container` instance
-const_ Store a constant value
-default_ Default value used when an argument is not provided Defaults to ``None``
-dest_ Specify the attribute name used in the result namespace
-help_ Help message for an argument
-metavar_ Alternate display name for the argument as shown in help
-nargs_ Number of times the argument can be used :class:`int`, ``'?'``, ``'*'``, or ``'+'``
-required_ Indicate whether an argument is required or optional ``True`` or ``False``
-:ref:`type ` Automatically convert an argument to the given type :class:`int`, :class:`float`, ``argparse.FileType('w')``, or callable function
-============================ =========================================================== ==========================================================================================================================
-
-
-Example
--------
-
-The following code is a Python program that takes a list of integers and
-produces either the sum or the max::
-
- import argparse
-
- parser = argparse.ArgumentParser(description='Process some integers.')
- parser.add_argument('integers', metavar='N', type=int, nargs='+',
- help='an integer for the accumulator')
- parser.add_argument('--sum', dest='accumulate', action='store_const',
- const=sum, default=max,
- help='sum the integers (default: find the max)')
-
- args = parser.parse_args()
- print(args.accumulate(args.integers))
-
-Assuming the above Python code is saved into a file called ``prog.py``, it can
-be run at the command line and it provides useful help messages:
-
-.. code-block:: shell-session
-
- $ python prog.py -h
- usage: prog.py [-h] [--sum] N [N ...]
-
- Process some integers.
-
- positional arguments:
- N an integer for the accumulator
-
- options:
- -h, --help show this help message and exit
- --sum sum the integers (default: find the max)
-
-When run with the appropriate arguments, it prints either the sum or the max of
-the command-line integers:
-
-.. code-block:: shell-session
-
- $ python prog.py 1 2 3 4
- 4
-
- $ python prog.py 1 2 3 4 --sum
- 10
-
-If invalid arguments are passed in, an error will be displayed:
-
-.. code-block:: shell-session
-
- $ python prog.py a b c
- usage: prog.py [-h] [--sum] N [N ...]
- prog.py: error: argument N: invalid int value: 'a'
-
-The following sections walk you through this example.
-
-
-Creating a parser
-^^^^^^^^^^^^^^^^^
-
-The first step in using the :mod:`argparse` is creating an
-:class:`ArgumentParser` object::
-
- >>> parser = argparse.ArgumentParser(description='Process some integers.')
-
-The :class:`ArgumentParser` object will hold all the information necessary to
-parse the command line into Python data types.
-
-
-Adding arguments
-^^^^^^^^^^^^^^^^
-
-Filling an :class:`ArgumentParser` with information about program arguments is
-done by making calls to the :meth:`~ArgumentParser.add_argument` method.
-Generally, these calls tell the :class:`ArgumentParser` how to take the strings
-on the command line and turn them into objects. This information is stored and
-used when :meth:`~ArgumentParser.parse_args` is called. For example::
-
- >>> parser.add_argument('integers', metavar='N', type=int, nargs='+',
- ... help='an integer for the accumulator')
- >>> parser.add_argument('--sum', dest='accumulate', action='store_const',
- ... const=sum, default=max,
- ... help='sum the integers (default: find the max)')
-
-Later, calling :meth:`~ArgumentParser.parse_args` will return an object with
-two attributes, ``integers`` and ``accumulate``. The ``integers`` attribute
-will be a list of one or more integers, and the ``accumulate`` attribute will be
-either the :func:`sum` function, if ``--sum`` was specified at the command line,
-or the :func:`max` function if it was not.
-
-
-Parsing arguments
-^^^^^^^^^^^^^^^^^
-
-:class:`ArgumentParser` parses arguments through the
-:meth:`~ArgumentParser.parse_args` method. This will inspect the command line,
-convert each argument to the appropriate type and then invoke the appropriate action.
-In most cases, this means a simple :class:`Namespace` object will be built up from
-attributes parsed out of the command line::
-
- >>> parser.parse_args(['--sum', '7', '-1', '42'])
- Namespace(accumulate=, integers=[7, -1, 42])
-
-In a script, :meth:`~ArgumentParser.parse_args` will typically be called with no
-arguments, and the :class:`ArgumentParser` will automatically determine the
-command-line arguments from :data:`sys.argv`.
-
+.. note::
+ If you're looking a guide about how to upgrade optparse code
+ to argparse, see :ref:`Upgrading Optparse Code `.
ArgumentParser objects
----------------------
@@ -249,39 +121,21 @@ The following sections describe how each of these are used.
prog
^^^^
-By default, :class:`ArgumentParser` objects use the base name
-(see :func:`os.path.basename`) of ``sys.argv[0]`` to determine
-how to display the name of the program in help messages. This default is almost
-always desirable because it will make the help messages match the name that was
-used to invoke the program on the command line. For example, consider a file
-named ``myprogram.py`` with the following code::
-
- import argparse
- parser = argparse.ArgumentParser()
- parser.add_argument('--foo', help='foo help')
- args = parser.parse_args()
-
-The help for this program will display ``myprogram.py`` as the program name
-(regardless of where the program was invoked from):
-.. code-block:: shell-session
+By default, :class:`ArgumentParser` calculates the name of the program
+to display in help messages depending on the way the Python interpreter was run:
- $ python myprogram.py --help
- usage: myprogram.py [-h] [--foo FOO]
+* The :func:`base name ` of ``sys.argv[0]`` if a file was
+ passed as argument.
+* The Python interpreter name followed by ``sys.argv[0]`` if a directory or
+ a zipfile was passed as argument.
+* The Python interpreter name followed by ``-m`` followed by the
+ module or package name if the :option:`-m` option was used.
- options:
- -h, --help show this help message and exit
- --foo FOO foo help
- $ cd ..
- $ python subdir/myprogram.py --help
- usage: myprogram.py [-h] [--foo FOO]
-
- options:
- -h, --help show this help message and exit
- --foo FOO foo help
-
-To change this default behavior, another value can be supplied using the
-``prog=`` argument to :class:`ArgumentParser`::
+This default is almost always desirable because it will make the help messages
+match the string that was used to invoke the program on the command line.
+However, to change this default behavior, another value can be supplied using
+the ``prog=`` argument to :class:`ArgumentParser`::
>>> parser = argparse.ArgumentParser(prog='myprogram')
>>> parser.print_help()
@@ -310,22 +164,8 @@ usage
^^^^^
By default, :class:`ArgumentParser` calculates the usage message from the
-arguments it contains::
-
- >>> parser = argparse.ArgumentParser(prog='PROG')
- >>> parser.add_argument('--foo', nargs='?', help='foo help')
- >>> parser.add_argument('bar', nargs='+', help='bar help')
- >>> parser.print_help()
- usage: PROG [-h] [--foo [FOO]] bar [bar ...]
-
- positional arguments:
- bar bar help
-
- options:
- -h, --help show this help message and exit
- --foo [FOO] foo help
-
-The default message can be overridden with the ``usage=`` keyword argument::
+arguments it contains. The default message can be overridden with the
+``usage=`` keyword argument::
>>> parser = argparse.ArgumentParser(prog='PROG', usage='%(prog)s [options]')
>>> parser.add_argument('--foo', nargs='?', help='foo help')
@@ -353,16 +193,7 @@ Most calls to the :class:`ArgumentParser` constructor will use the
``description=`` keyword argument. This argument gives a brief description of
what the program does and how it works. In help messages, the description is
displayed between the command-line usage string and the help messages for the
-various arguments::
-
- >>> parser = argparse.ArgumentParser(description='A foo that bars')
- >>> parser.print_help()
- usage: argparse.py [-h]
-
- A foo that bars
-
- options:
- -h, --help show this help message and exit
+various arguments.
By default, the description will be line-wrapped so that it fits within the
given space. To change this behavior, see the formatter_class_ argument.
@@ -492,7 +323,7 @@ should not be line-wrapped::
-h, --help show this help message and exit
:class:`RawTextHelpFormatter` maintains whitespace for all sorts of help text,
-including argument descriptions. However, multiple new lines are replaced with
+including argument descriptions. However, multiple newlines are replaced with
one. If you wish to preserve multiple blank lines, add spaces between the
newlines.
@@ -586,8 +417,8 @@ arguments will never be treated as file references.
.. versionchanged:: 3.12
:class:`ArgumentParser` changed encoding and errors to read arguments files
- from default (e.g. :func:`locale.getpreferredencoding(False) ` and
- ``"strict"``) to :term:`filesystem encoding and error handler`.
+ from default (e.g. :func:`locale.getpreferredencoding(False) `
+ and ``"strict"``) to the :term:`filesystem encoding and error handler`.
Arguments file should be encoded in UTF-8 instead of ANSI Codepage on Windows.
@@ -673,25 +504,8 @@ add_help
^^^^^^^^
By default, ArgumentParser objects add an option which simply displays
-the parser's help message. For example, consider a file named
-``myprogram.py`` containing the following code::
-
- import argparse
- parser = argparse.ArgumentParser()
- parser.add_argument('--foo', help='foo help')
- args = parser.parse_args()
-
-If ``-h`` or ``--help`` is supplied at the command line, the ArgumentParser
-help will be printed:
-
-.. code-block:: shell-session
-
- $ python myprogram.py --help
- usage: myprogram.py [-h] [--foo FOO]
-
- options:
- -h, --help show this help message and exit
- --foo FOO foo help
+the parser's help message. If ``-h`` or ``--help`` is supplied at the command
+line, the ArgumentParser help will be printed.
Occasionally, it may be useful to disable the addition of this help option.
This can be achieved by passing ``False`` as the ``add_help=`` argument to
@@ -828,12 +642,7 @@ them, though most actions simply add an attribute to the object returned by
how the command-line arguments should be handled. The supplied actions are:
* ``'store'`` - This just stores the argument's value. This is the default
- action. For example::
-
- >>> parser = argparse.ArgumentParser()
- >>> parser.add_argument('--foo')
- >>> parser.parse_args('--foo 1'.split())
- Namespace(foo='1')
+ action.
* ``'store_const'`` - This stores the value specified by the const_ keyword
argument; note that the const_ keyword argument defaults to ``None``. The
@@ -848,7 +657,7 @@ how the command-line arguments should be handled. The supplied actions are:
* ``'store_true'`` and ``'store_false'`` - These are special cases of
``'store_const'`` used for storing the values ``True`` and ``False``
respectively. In addition, they create default values of ``False`` and
- ``True`` respectively. For example::
+ ``True`` respectively::
>>> parser = argparse.ArgumentParser()
>>> parser.add_argument('--foo', action='store_true')
@@ -1093,7 +902,7 @@ was not present at the command line::
Namespace(foo=42)
If the target namespace already has an attribute set, the action *default*
-will not over write it::
+will not overwrite it::
>>> parser = argparse.ArgumentParser()
>>> parser.add_argument('--foo', default=42)
@@ -1167,7 +976,6 @@ Common built-in types and functions can be used as type converters:
parser.add_argument('distance', type=float)
parser.add_argument('street', type=ascii)
parser.add_argument('code_point', type=ord)
- parser.add_argument('source_file', type=open)
parser.add_argument('dest_file', type=argparse.FileType('w', encoding='latin-1'))
parser.add_argument('datapath', type=pathlib.Path)
@@ -1198,10 +1006,11 @@ better reporting than can be given by the ``type`` keyword. A
:exc:`FileNotFoundError` exception would not be handled at all.
Even :class:`~argparse.FileType` has its limitations for use with the ``type``
-keyword. If one argument uses *FileType* and then a subsequent argument fails,
-an error is reported but the file is not automatically closed. In this case, it
-would be better to wait until after the parser has run and then use the
-:keyword:`with`-statement to manage the files.
+keyword. If one argument uses :class:`~argparse.FileType` and then a
+subsequent argument fails, an error is reported but the file is not
+automatically closed. In this case, it would be better to wait until after
+the parser has run and then use the :keyword:`with`-statement to manage the
+files.
For type checkers that simply check against a fixed set of values, consider
using the choices_ keyword instead.
@@ -1229,15 +1038,7 @@ if the argument was not one of the acceptable values::
Note that inclusion in the *choices* sequence is checked after any type_
conversions have been performed, so the type of the objects in the *choices*
-sequence should match the type_ specified::
-
- >>> parser = argparse.ArgumentParser(prog='doors.py')
- >>> parser.add_argument('door', type=int, choices=range(1, 4))
- >>> print(parser.parse_args(['3']))
- Namespace(door=3)
- >>> parser.parse_args(['4'])
- usage: doors.py [-h] {1,2,3}
- doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3)
+sequence should match the type_ specified.
Any sequence can be passed as the *choices* value, so :class:`list` objects,
:class:`tuple` objects, and custom sequences are all supported.
@@ -1287,22 +1088,7 @@ help
The ``help`` value is a string containing a brief description of the argument.
When a user requests help (usually by using ``-h`` or ``--help`` at the
command line), these ``help`` descriptions will be displayed with each
-argument::
-
- >>> parser = argparse.ArgumentParser(prog='frobble')
- >>> parser.add_argument('--foo', action='store_true',
- ... help='foo the bars before frobbling')
- >>> parser.add_argument('bar', nargs='+',
- ... help='one of the bars to be frobbled')
- >>> parser.parse_args(['-h'])
- usage: frobble [-h] [--foo] bar [bar ...]
-
- positional arguments:
- bar one of the bars to be frobbled
-
- options:
- -h, --help show this help message and exit
- --foo foo the bars before frobbling
+argument.
The ``help`` strings can include various format specifiers to avoid repetition
of things like the program name or the argument default_. The available
@@ -1455,40 +1241,41 @@ this API may be passed as the ``action`` parameter to
type=None, choices=None, required=False, help=None, \
metavar=None)
-Action objects are used by an ArgumentParser to represent the information
-needed to parse a single argument from one or more strings from the
-command line. The Action class must accept the two positional arguments
-plus any keyword arguments passed to :meth:`ArgumentParser.add_argument`
-except for the ``action`` itself.
+ Action objects are used by an ArgumentParser to represent the information
+ needed to parse a single argument from one or more strings from the
+ command line. The Action class must accept the two positional arguments
+ plus any keyword arguments passed to :meth:`ArgumentParser.add_argument`
+ except for the ``action`` itself.
-Instances of Action (or return value of any callable to the ``action``
-parameter) should have attributes "dest", "option_strings", "default", "type",
-"required", "help", etc. defined. The easiest way to ensure these attributes
-are defined is to call ``Action.__init__``.
+ Instances of Action (or return value of any callable to the ``action``
+ parameter) should have attributes "dest", "option_strings", "default", "type",
+ "required", "help", etc. defined. The easiest way to ensure these attributes
+ are defined is to call ``Action.__init__``.
-Action instances should be callable, so subclasses must override the
-``__call__`` method, which should accept four parameters:
+ Action instances should be callable, so subclasses must override the
+ ``__call__`` method, which should accept four parameters:
-* ``parser`` - The ArgumentParser object which contains this action.
+ * *parser* - The ArgumentParser object which contains this action.
-* ``namespace`` - The :class:`Namespace` object that will be returned by
- :meth:`~ArgumentParser.parse_args`. Most actions add an attribute to this
- object using :func:`setattr`.
+ * *namespace* - The :class:`Namespace` object that will be returned by
+ :meth:`~ArgumentParser.parse_args`. Most actions add an attribute to this
+ object using :func:`setattr`.
-* ``values`` - The associated command-line arguments, with any type conversions
- applied. Type conversions are specified with the type_ keyword argument to
- :meth:`~ArgumentParser.add_argument`.
+ * *values* - The associated command-line arguments, with any type conversions
+ applied. Type conversions are specified with the type_ keyword argument to
+ :meth:`~ArgumentParser.add_argument`.
-* ``option_string`` - The option string that was used to invoke this action.
- The ``option_string`` argument is optional, and will be absent if the action
- is associated with a positional argument.
+ * *option_string* - The option string that was used to invoke this action.
+ The ``option_string`` argument is optional, and will be absent if the action
+ is associated with a positional argument.
-The ``__call__`` method may perform arbitrary actions, but will typically set
-attributes on the ``namespace`` based on ``dest`` and ``values``.
+ The ``__call__`` method may perform arbitrary actions, but will typically set
+ attributes on the ``namespace`` based on ``dest`` and ``values``.
+
+ Action subclasses can define a ``format_usage`` method that takes no argument
+ and return a string which will be used when printing the usage of the program.
+ If such method is not provided, a sensible default will be used.
-Action subclasses can define a ``format_usage`` method that takes no argument
-and return a string which will be used when printing the usage of the program.
-If such method is not provided, a sensible default will be used.
The parse_args() method
-----------------------
@@ -1683,29 +1470,29 @@ The Namespace object
Simple class used by default by :meth:`~ArgumentParser.parse_args` to create
an object holding attributes and return it.
-This class is deliberately simple, just an :class:`object` subclass with a
-readable string representation. If you prefer to have dict-like view of the
-attributes, you can use the standard Python idiom, :func:`vars`::
+ This class is deliberately simple, just an :class:`object` subclass with a
+ readable string representation. If you prefer to have dict-like view of the
+ attributes, you can use the standard Python idiom, :func:`vars`::
- >>> parser = argparse.ArgumentParser()
- >>> parser.add_argument('--foo')
- >>> args = parser.parse_args(['--foo', 'BAR'])
- >>> vars(args)
- {'foo': 'BAR'}
-
-It may also be useful to have an :class:`ArgumentParser` assign attributes to an
-already existing object, rather than a new :class:`Namespace` object. This can
-be achieved by specifying the ``namespace=`` keyword argument::
-
- >>> class C:
- ... pass
- ...
- >>> c = C()
- >>> parser = argparse.ArgumentParser()
- >>> parser.add_argument('--foo')
- >>> parser.parse_args(args=['--foo', 'BAR'], namespace=c)
- >>> c.foo
- 'BAR'
+ >>> parser = argparse.ArgumentParser()
+ >>> parser.add_argument('--foo')
+ >>> args = parser.parse_args(['--foo', 'BAR'])
+ >>> vars(args)
+ {'foo': 'BAR'}
+
+ It may also be useful to have an :class:`ArgumentParser` assign attributes to an
+ already existing object, rather than a new :class:`Namespace` object. This can
+ be achieved by specifying the ``namespace=`` keyword argument::
+
+ >>> class C:
+ ... pass
+ ...
+ >>> c = C()
+ >>> parser = argparse.ArgumentParser()
+ >>> parser.add_argument('--foo')
+ >>> parser.parse_args(args=['--foo', 'BAR'], namespace=c)
+ >>> c.foo
+ 'BAR'
Other utilities
@@ -1719,12 +1506,12 @@ Sub-commands
[option_strings], [dest], [required], \
[help], [metavar])
- Many programs split up their functionality into a number of sub-commands,
- for example, the ``svn`` program can invoke sub-commands like ``svn
+ Many programs split up their functionality into a number of subcommands,
+ for example, the ``svn`` program can invoke subcommands like ``svn
checkout``, ``svn update``, and ``svn commit``. Splitting up functionality
this way can be a particularly good idea when a program performs several
different functions which require different kinds of command-line arguments.
- :class:`ArgumentParser` supports the creation of such sub-commands with the
+ :class:`ArgumentParser` supports the creation of such subcommands with the
:meth:`add_subparsers` method. The :meth:`add_subparsers` method is normally
called with no arguments and returns a special action object. This object
has a single method, :meth:`~_SubParsersAction.add_parser`, which takes a
@@ -1733,18 +1520,18 @@ Sub-commands
Description of parameters:
- * title - title for the sub-parser group in help output; by default
+ * *title* - title for the sub-parser group in help output; by default
"subcommands" if description is provided, otherwise uses title for
positional arguments
- * description - description for the sub-parser group in help output, by
+ * *description* - description for the sub-parser group in help output, by
default ``None``
- * prog - usage information that will be displayed with sub-command help,
+ * *prog* - usage information that will be displayed with sub-command help,
by default the name of the program and any positional arguments before the
subparser argument
- * parser_class - class which will be used to create sub-parser instances, by
+ * *parser_class* - class which will be used to create sub-parser instances, by
default the class of the current parser (e.g. ArgumentParser)
* action_ - the basic type of action to be taken when this argument is
@@ -1758,15 +1545,15 @@ Sub-commands
* help_ - help for sub-parser group in help output, by default ``None``
- * metavar_ - string presenting available sub-commands in help; by default it
- is ``None`` and presents sub-commands in form {cmd1, cmd2, ..}
+ * metavar_ - string presenting available subcommands in help; by default it
+ is ``None`` and presents subcommands in form {cmd1, cmd2, ..}
Some example usage::
>>> # create the top-level parser
>>> parser = argparse.ArgumentParser(prog='PROG')
>>> parser.add_argument('--foo', action='store_true', help='foo help')
- >>> subparsers = parser.add_subparsers(help='sub-command help')
+ >>> subparsers = parser.add_subparsers(help='subcommand help')
>>>
>>> # create the parser for the "a" command
>>> parser_a = subparsers.add_parser('a', help='a help')
@@ -1801,7 +1588,7 @@ Sub-commands
usage: PROG [-h] [--foo] {a,b} ...
positional arguments:
- {a,b} sub-command help
+ {a,b} subcommand help
a a help
b b help
@@ -1862,7 +1649,7 @@ Sub-commands
that each subparser knows which Python function it should execute. For
example::
- >>> # sub-command functions
+ >>> # subcommand functions
>>> def foo(args):
... print(args.x * args.y)
...
@@ -2144,20 +1931,20 @@ Partial parsing
.. method:: ArgumentParser.parse_known_args(args=None, namespace=None)
-Sometimes a script may only parse a few of the command-line arguments, passing
-the remaining arguments on to another script or program. In these cases, the
-:meth:`~ArgumentParser.parse_known_args` method can be useful. It works much like
-:meth:`~ArgumentParser.parse_args` except that it does not produce an error when
-extra arguments are present. Instead, it returns a two item tuple containing
-the populated namespace and the list of remaining argument strings.
+ Sometimes a script may only parse a few of the command-line arguments, passing
+ the remaining arguments on to another script or program. In these cases, the
+ :meth:`~ArgumentParser.parse_known_args` method can be useful. It works much like
+ :meth:`~ArgumentParser.parse_args` except that it does not produce an error when
+ extra arguments are present. Instead, it returns a two item tuple containing
+ the populated namespace and the list of remaining argument strings.
-::
+ ::
- >>> parser = argparse.ArgumentParser()
- >>> parser.add_argument('--foo', action='store_true')
- >>> parser.add_argument('bar')
- >>> parser.parse_known_args(['--foo', '--badger', 'BAR', 'spam'])
- (Namespace(bar='BAR', foo=True), ['--badger', 'spam'])
+ >>> parser = argparse.ArgumentParser()
+ >>> parser.add_argument('--foo', action='store_true')
+ >>> parser.add_argument('bar')
+ >>> parser.parse_known_args(['--foo', '--badger', 'BAR', 'spam'])
+ (Namespace(bar='BAR', foo=True), ['--badger', 'spam'])
.. warning::
:ref:`Prefix matching ` rules apply to
@@ -2215,90 +2002,38 @@ Intermixed parsing
.. method:: ArgumentParser.parse_intermixed_args(args=None, namespace=None)
.. method:: ArgumentParser.parse_known_intermixed_args(args=None, namespace=None)
-A number of Unix commands allow the user to intermix optional arguments with
-positional arguments. The :meth:`~ArgumentParser.parse_intermixed_args`
-and :meth:`~ArgumentParser.parse_known_intermixed_args` methods
-support this parsing style.
-
-These parsers do not support all the argparse features, and will raise
-exceptions if unsupported features are used. In particular, subparsers,
-and mutually exclusive groups that include both
-optionals and positionals are not supported.
-
-The following example shows the difference between
-:meth:`~ArgumentParser.parse_known_args` and
-:meth:`~ArgumentParser.parse_intermixed_args`: the former returns ``['2',
-'3']`` as unparsed arguments, while the latter collects all the positionals
-into ``rest``. ::
-
- >>> parser = argparse.ArgumentParser()
- >>> parser.add_argument('--foo')
- >>> parser.add_argument('cmd')
- >>> parser.add_argument('rest', nargs='*', type=int)
- >>> parser.parse_known_args('doit 1 --foo bar 2 3'.split())
- (Namespace(cmd='doit', foo='bar', rest=[1]), ['2', '3'])
- >>> parser.parse_intermixed_args('doit 1 --foo bar 2 3'.split())
- Namespace(cmd='doit', foo='bar', rest=[1, 2, 3])
-
-:meth:`~ArgumentParser.parse_known_intermixed_args` returns a two item tuple
-containing the populated namespace and the list of remaining argument strings.
-:meth:`~ArgumentParser.parse_intermixed_args` raises an error if there are any
-remaining unparsed argument strings.
-
-.. versionadded:: 3.7
-
-.. _upgrading-optparse-code:
-
-Upgrading optparse code
------------------------
-
-Originally, the :mod:`argparse` module had attempted to maintain compatibility
-with :mod:`optparse`. However, :mod:`optparse` was difficult to extend
-transparently, particularly with the changes required to support the new
-``nargs=`` specifiers and better usage messages. When most everything in
-:mod:`optparse` had either been copy-pasted over or monkey-patched, it no
-longer seemed practical to try to maintain the backwards compatibility.
-
-The :mod:`argparse` module improves on the standard library :mod:`optparse`
-module in a number of ways including:
-
-* Handling positional arguments.
-* Supporting sub-commands.
-* Allowing alternative option prefixes like ``+`` and ``/``.
-* Handling zero-or-more and one-or-more style arguments.
-* Producing more informative usage messages.
-* Providing a much simpler interface for custom ``type`` and ``action``.
+ A number of Unix commands allow the user to intermix optional arguments with
+ positional arguments. The :meth:`~ArgumentParser.parse_intermixed_args`
+ and :meth:`~ArgumentParser.parse_known_intermixed_args` methods
+ support this parsing style.
-A partial upgrade path from :mod:`optparse` to :mod:`argparse`:
+ These parsers do not support all the argparse features, and will raise
+ exceptions if unsupported features are used. In particular, subparsers,
+ and mutually exclusive groups that include both
+ optionals and positionals are not supported.
-* Replace all :meth:`optparse.OptionParser.add_option` calls with
- :meth:`ArgumentParser.add_argument` calls.
+ The following example shows the difference between
+ :meth:`~ArgumentParser.parse_known_args` and
+ :meth:`~ArgumentParser.parse_intermixed_args`: the former returns ``['2',
+ '3']`` as unparsed arguments, while the latter collects all the positionals
+ into ``rest``. ::
-* Replace ``(options, args) = parser.parse_args()`` with ``args =
- parser.parse_args()`` and add additional :meth:`ArgumentParser.add_argument`
- calls for the positional arguments. Keep in mind that what was previously
- called ``options``, now in the :mod:`argparse` context is called ``args``.
-
-* Replace :meth:`optparse.OptionParser.disable_interspersed_args`
- by using :meth:`~ArgumentParser.parse_intermixed_args` instead of
- :meth:`~ArgumentParser.parse_args`.
-
-* Replace callback actions and the ``callback_*`` keyword arguments with
- ``type`` or ``action`` arguments.
-
-* Replace string names for ``type`` keyword arguments with the corresponding
- type objects (e.g. int, float, complex, etc).
+ >>> parser = argparse.ArgumentParser()
+ >>> parser.add_argument('--foo')
+ >>> parser.add_argument('cmd')
+ >>> parser.add_argument('rest', nargs='*', type=int)
+ >>> parser.parse_known_args('doit 1 --foo bar 2 3'.split())
+ (Namespace(cmd='doit', foo='bar', rest=[1]), ['2', '3'])
+ >>> parser.parse_intermixed_args('doit 1 --foo bar 2 3'.split())
+ Namespace(cmd='doit', foo='bar', rest=[1, 2, 3])
-* Replace :class:`optparse.Values` with :class:`Namespace` and
- :exc:`optparse.OptionError` and :exc:`optparse.OptionValueError` with
- :exc:`ArgumentError`.
+ :meth:`~ArgumentParser.parse_known_intermixed_args` returns a two item tuple
+ containing the populated namespace and the list of remaining argument strings.
+ :meth:`~ArgumentParser.parse_intermixed_args` raises an error if there are any
+ remaining unparsed argument strings.
-* Replace strings with implicit arguments such as ``%default`` or ``%prog`` with
- the standard Python syntax to use dictionaries to format strings, that is,
- ``%(default)s`` and ``%(prog)s``.
+ .. versionadded:: 3.7
-* Replace the OptionParser constructor ``version`` argument with a call to
- ``parser.add_argument('--version', action='version', version='')``.
Exceptions
----------
@@ -2313,3 +2048,12 @@ Exceptions
.. exception:: ArgumentTypeError
Raised when something goes wrong converting a command line string to a type.
+
+
+.. rubric:: Guides and Tutorials
+
+.. toctree::
+ :maxdepth: 1
+
+ ../howto/argparse.rst
+ ../howto/argparse-optparse.rst
From 7187d4f3b65a26aca1229f4c987d19c6d3ead0bb Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 9 Oct 2024 16:16:13 +0200
Subject: [PATCH 028/269] [3.12] gh-101100: Fix Sphinx warnings in
`library/unittest.mock.rst` (GH-124106) (#125191)
Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
---
Doc/library/unittest.mock.rst | 60 +++++++++++++++++------------------
1 file changed, 30 insertions(+), 30 deletions(-)
diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst
index bb3baa97275e7f..84f34a0696a26c 100644
--- a/Doc/library/unittest.mock.rst
+++ b/Doc/library/unittest.mock.rst
@@ -68,7 +68,7 @@ available, and then make assertions about how they have been used:
3
>>> thing.method.assert_called_with(3, 4, 5, key='value')
-:attr:`side_effect` allows you to perform side effects, including raising an
+:attr:`~Mock.side_effect` allows you to perform side effects, including raising an
exception when a mock is called:
>>> from unittest.mock import Mock
@@ -756,8 +756,8 @@ the *new_callable* argument to :func:`patch`.
.. attribute:: __class__
- Normally the :attr:`__class__` attribute of an object will return its type.
- For a mock object with a :attr:`spec`, ``__class__`` returns the spec class
+ Normally the :attr:`!__class__` attribute of an object will return its type.
+ For a mock object with a :attr:`!spec`, :attr:`!__class__` returns the spec class
instead. This allows mock objects to pass :func:`isinstance` tests for the
object they are replacing / masquerading as:
@@ -765,7 +765,7 @@ the *new_callable* argument to :func:`patch`.
>>> isinstance(mock, int)
True
- :attr:`__class__` is assignable to, this allows a mock to pass an
+ :attr:`!__class__` is assignable to, this allows a mock to pass an
:func:`isinstance` check without forcing you to use a spec:
>>> mock = Mock()
@@ -779,8 +779,8 @@ the *new_callable* argument to :func:`patch`.
meaning of :class:`Mock`, with the exception of *return_value* and *side_effect*
which have no meaning on a non-callable mock.
-Mock objects that use a class or an instance as a :attr:`spec` or
-:attr:`spec_set` are able to pass :func:`isinstance` tests:
+Mock objects that use a class or an instance as a :attr:`!spec` or
+:attr:`!spec_set` are able to pass :func:`isinstance` tests:
>>> mock = Mock(spec=SomeClass)
>>> isinstance(mock, SomeClass)
@@ -1149,7 +1149,7 @@ Calls made to the object will be recorded in the attributes
like :attr:`~Mock.call_args` and :attr:`~Mock.call_args_list`.
If :attr:`~Mock.side_effect` is set then it will be called after the call has
-been recorded, so if :attr:`side_effect` raises an exception the call is still
+been recorded, so if :attr:`!side_effect` raises an exception the call is still
recorded.
The simplest way to make a mock raise an exception when called is to make
@@ -1170,8 +1170,8 @@ The simplest way to make a mock raise an exception when called is to make
>>> m.mock_calls
[call(1, 2, 3), call('two', 'three', 'four')]
-If :attr:`side_effect` is a function then whatever that function returns is what
-calls to the mock return. The :attr:`side_effect` function is called with the
+If :attr:`~Mock.side_effect` is a function then whatever that function returns is what
+calls to the mock return. The :attr:`!side_effect` function is called with the
same arguments as the mock. This allows you to vary the return value of the
call dynamically, based on the input:
@@ -1188,7 +1188,7 @@ call dynamically, based on the input:
If you want the mock to still return the default return value (a new mock), or
any set return value, then there are two ways of doing this. Either return
-:attr:`mock.return_value` from inside :attr:`side_effect`, or return :data:`DEFAULT`:
+:attr:`~Mock.return_value` from inside :attr:`~Mock.side_effect`, or return :data:`DEFAULT`:
>>> m = MagicMock()
>>> def side_effect(*args, **kwargs):
@@ -1205,8 +1205,8 @@ any set return value, then there are two ways of doing this. Either return
>>> m()
3
-To remove a :attr:`side_effect`, and return to the default behaviour, set the
-:attr:`side_effect` to ``None``:
+To remove a :attr:`~Mock.side_effect`, and return to the default behaviour, set the
+:attr:`!side_effect` to ``None``:
>>> m = MagicMock(return_value=6)
>>> def side_effect(*args, **kwargs):
@@ -1219,7 +1219,7 @@ To remove a :attr:`side_effect`, and return to the default behaviour, set the
>>> m()
6
-The :attr:`side_effect` can also be any iterable object. Repeated calls to the mock
+The :attr:`~Mock.side_effect` can also be any iterable object. Repeated calls to the mock
will return values from the iterable (until the iterable is exhausted and
a :exc:`StopIteration` is raised):
@@ -1260,7 +1260,7 @@ objects of any type.
You may want a mock object to return ``False`` to a :func:`hasattr` call, or raise an
:exc:`AttributeError` when an attribute is fetched. You can do this by providing
-an object as a :attr:`spec` for a mock, but that isn't always convenient.
+an object as a :attr:`!spec` for a mock, but that isn't always convenient.
You "block" attributes by deleting them. Once deleted, accessing an attribute
will raise an :exc:`AttributeError`.
@@ -1429,7 +1429,7 @@ patch
If you are patching builtins in a module then you don't
need to pass ``create=True``, it will be added by default.
- Patch can be used as a :class:`TestCase` class decorator. It works by
+ Patch can be used as a :class:`~unittest.TestCase` class decorator. It works by
decorating each test method in the class. This reduces the boilerplate
code when your test methods share a common patchings set. :func:`patch` finds
tests by looking for method names that start with ``patch.TEST_PREFIX``.
@@ -1467,7 +1467,7 @@ If the class is instantiated multiple times you could use
can set the *return_value* to be anything you want.
To configure return values on methods of *instances* on the patched class
-you must do this on the :attr:`return_value`. For example::
+you must do this on the :attr:`~Mock.return_value`. For example::
>>> class Class:
... def method(self):
@@ -1788,13 +1788,13 @@ context manager is a dictionary where created mocks are keyed by name::
patch methods: start and stop
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-All the patchers have :meth:`start` and :meth:`stop` methods. These make it simpler to do
+All the patchers have :meth:`!start` and :meth:`!stop` methods. These make it simpler to do
patching in ``setUp`` methods or where you want to do multiple patches without
nesting decorators or with statements.
To use them call :func:`patch`, :func:`patch.object` or :func:`patch.dict` as
normal and keep a reference to the returned ``patcher`` object. You can then
-call :meth:`start` to put the patch in place and :meth:`stop` to undo it.
+call :meth:`!start` to put the patch in place and :meth:`!stop` to undo it.
If you are using :func:`patch` to create a mock for you then it will be returned by
the call to ``patcher.start``. ::
@@ -1811,7 +1811,7 @@ the call to ``patcher.start``. ::
A typical use case for this might be for doing multiple patches in the ``setUp``
-method of a :class:`TestCase`::
+method of a :class:`~unittest.TestCase`::
>>> class MyTest(unittest.TestCase):
... def setUp(self):
@@ -2484,7 +2484,7 @@ behaviour you can switch it off by setting the module level switch
Alternatively you can just use ``vars(my_mock)`` (instance members) and
``dir(type(my_mock))`` (type members) to bypass the filtering irrespective of
-:const:`mock.FILTER_DIR`.
+:const:`FILTER_DIR`.
mock_open
@@ -2499,7 +2499,7 @@ mock_open
default) then a :class:`MagicMock` will be created for you, with the API limited
to methods or attributes available on standard file handles.
- *read_data* is a string for the :meth:`~io.IOBase.read`,
+ *read_data* is a string for the :meth:`~io.RawIOBase.read`,
:meth:`~io.IOBase.readline`, and :meth:`~io.IOBase.readlines` methods
of the file handle to return. Calls to those methods will take data from
*read_data* until it is depleted. The mock of these methods is pretty
@@ -2511,7 +2511,7 @@ mock_open
.. versionchanged:: 3.4
Added :meth:`~io.IOBase.readline` and :meth:`~io.IOBase.readlines` support.
- The mock of :meth:`~io.IOBase.read` changed to consume *read_data* rather
+ The mock of :meth:`~io.RawIOBase.read` changed to consume *read_data* rather
than returning it on each call.
.. versionchanged:: 3.5
@@ -2563,7 +2563,7 @@ And for reading files::
Autospeccing
~~~~~~~~~~~~
-Autospeccing is based on the existing :attr:`spec` feature of mock. It limits the
+Autospeccing is based on the existing :attr:`!spec` feature of mock. It limits the
api of mocks to the api of an original object (the spec), but it is recursive
(implemented lazily) so that attributes of mocks only have the same api as
the attributes of the spec. In addition mocked functions / methods have the
@@ -2588,8 +2588,8 @@ unit tests. Testing everything in isolation is all fine and dandy, but if you
don't test how your units are "wired together" there is still lots of room
for bugs that tests might have caught.
-:mod:`mock` already provides a feature to help with this, called speccing. If you
-use a class or instance as the :attr:`spec` for a mock then you can only access
+:mod:`unittest.mock` already provides a feature to help with this, called speccing. If you
+use a class or instance as the :attr:`!spec` for a mock then you can only access
attributes on the mock that exist on the real class:
>>> from urllib import request
@@ -2627,7 +2627,7 @@ Here's an example of it in use::
>>> mock_request.Request
-You can see that :class:`request.Request` has a spec. :class:`request.Request` takes two
+You can see that :class:`!request.Request` has a spec. :class:`!request.Request` takes two
arguments in the constructor (one of which is *self*). Here's what happens if
we try to call it incorrectly::
@@ -2643,8 +2643,8 @@ specced mocks)::
>>> req
-:class:`Request` objects are not callable, so the return value of instantiating our
-mocked out :class:`request.Request` is a non-callable mock. With the spec in place
+:class:`!Request` objects are not callable, so the return value of instantiating our
+mocked out :class:`!request.Request` is a non-callable mock. With the spec in place
any typos in our asserts will raise the correct error::
>>> req.add_header('spam', 'eggs')
@@ -2796,8 +2796,8 @@ Sealing mocks
.. versionadded:: 3.7
-Order of precedence of :attr:`side_effect`, :attr:`return_value` and *wraps*
-----------------------------------------------------------------------------
+Order of precedence of :attr:`!side_effect`, :attr:`!return_value` and *wraps*
+------------------------------------------------------------------------------
The order of their precedence is:
From 046687c620f5bff8fc9dead066ab30e9802ba5b0 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 9 Oct 2024 18:54:38 +0200
Subject: [PATCH 029/269] [3.12] docs: in venv table use executable name
(GH-124315) (#125171)
(cherry picked from commit 7f93dbf6fec152888727a0f25a3aa030d1fe27ca)
---
Doc/library/venv.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst
index 546ce3716d28b9..9eb0e9d191c7cd 100644
--- a/Doc/library/venv.rst
+++ b/Doc/library/venv.rst
@@ -205,7 +205,7 @@ containing the virtual environment):
| +------------+--------------------------------------------------+
| | csh/tcsh | :samp:`$ source {}/bin/activate.csh` |
| +------------+--------------------------------------------------+
-| | PowerShell | :samp:`$ {}/bin/Activate.ps1` |
+| | pwsh | :samp:`$ {}/bin/Activate.ps1` |
+-------------+------------+--------------------------------------------------+
| Windows | cmd.exe | :samp:`C:\\> {}\\Scripts\\activate.bat` |
| +------------+--------------------------------------------------+
From cba37762acfb82ca7fdf4b2c9ea3ea54595b7571 Mon Sep 17 00:00:00 2001
From: Alex Waygood
Date: Wed, 9 Oct 2024 20:18:38 +0100
Subject: [PATCH 030/269] [3.12] gh-101100: Consolidate documentation on
`ModuleType` attributes (#124709) (#125211)
Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
Co-authored-by: Barry Warsaw
Co-authored-by: Jelle Zijlstra
---
Doc/c-api/import.rst | 26 +--
Doc/c-api/module.rst | 22 +-
Doc/deprecations/pending-removal-in-3.14.rst | 7 +
Doc/glossary.rst | 11 +-
Doc/library/ast.rst | 2 +-
Doc/library/importlib.rst | 87 +++----
Doc/library/pkgutil.rst | 3 +-
Doc/library/sys.rst | 3 +-
Doc/library/types.rst | 65 +-----
Doc/reference/datamodel.rst | 232 ++++++++++++++++---
Doc/reference/import.rst | 161 +++----------
Doc/tutorial/modules.rst | 5 +-
Doc/whatsnew/2.6.rst | 8 +-
Doc/whatsnew/3.0.rst | 4 +-
Doc/whatsnew/3.12.rst | 13 +-
Doc/whatsnew/3.2.rst | 4 +-
Doc/whatsnew/3.4.rst | 3 +-
Doc/whatsnew/3.5.rst | 4 +-
Misc/NEWS.d/3.10.0a2.rst | 4 +-
Misc/NEWS.d/3.11.0a5.rst | 4 +-
Misc/NEWS.d/3.12.0a1.rst | 4 +-
21 files changed, 348 insertions(+), 324 deletions(-)
diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst
index b8687c61c26d0d..d6370bc9645ff2 100644
--- a/Doc/c-api/import.rst
+++ b/Doc/c-api/import.rst
@@ -120,14 +120,14 @@ Importing Modules
such modules have no way to know that the module object is an unknown (and
probably damaged with respect to the module author's intents) state.
- The module's :attr:`__spec__` and :attr:`__loader__` will be set, if
- not set already, with the appropriate values. The spec's loader will
- be set to the module's ``__loader__`` (if set) and to an instance of
- :class:`~importlib.machinery.SourceFileLoader` otherwise.
+ The module's :attr:`~module.__spec__` and :attr:`~module.__loader__` will be
+ set, if not set already, with the appropriate values. The spec's loader
+ will be set to the module's :attr:`!__loader__` (if set) and to an instance
+ of :class:`~importlib.machinery.SourceFileLoader` otherwise.
- The module's :attr:`__file__` attribute will be set to the code object's
- :attr:`~codeobject.co_filename`. If applicable, :attr:`__cached__` will also
- be set.
+ The module's :attr:`~module.__file__` attribute will be set to the code
+ object's :attr:`~codeobject.co_filename`. If applicable,
+ :attr:`~module.__cached__` will also be set.
This function will reload the module if it was already imported. See
:c:func:`PyImport_ReloadModule` for the intended way to reload a module.
@@ -139,29 +139,29 @@ Importing Modules
:c:func:`PyImport_ExecCodeModuleWithPathnames`.
.. versionchanged:: 3.12
- The setting of :attr:`__cached__` and :attr:`__loader__` is
- deprecated. See :class:`~importlib.machinery.ModuleSpec` for
+ The setting of :attr:`~module.__cached__` and :attr:`~module.__loader__`
+ is deprecated. See :class:`~importlib.machinery.ModuleSpec` for
alternatives.
.. c:function:: PyObject* PyImport_ExecCodeModuleEx(const char *name, PyObject *co, const char *pathname)
- Like :c:func:`PyImport_ExecCodeModule`, but the :attr:`__file__` attribute of
- the module object is set to *pathname* if it is non-``NULL``.
+ Like :c:func:`PyImport_ExecCodeModule`, but the :attr:`~module.__file__`
+ attribute of the module object is set to *pathname* if it is non-``NULL``.
See also :c:func:`PyImport_ExecCodeModuleWithPathnames`.
.. c:function:: PyObject* PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, PyObject *cpathname)
- Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`__cached__`
+ Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`~module.__cached__`
attribute of the module object is set to *cpathname* if it is
non-``NULL``. Of the three functions, this is the preferred one to use.
.. versionadded:: 3.3
.. versionchanged:: 3.12
- Setting :attr:`__cached__` is deprecated. See
+ Setting :attr:`~module.__cached__` is deprecated. See
:class:`~importlib.machinery.ModuleSpec` for alternatives.
diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst
index 910137f6302006..cfa6a1a985245e 100644
--- a/Doc/c-api/module.rst
+++ b/Doc/c-api/module.rst
@@ -37,18 +37,19 @@ Module Objects
single: __package__ (module attribute)
single: __loader__ (module attribute)
- Return a new module object with the :attr:`__name__` attribute set to *name*.
- The module's :attr:`__name__`, :attr:`__doc__`, :attr:`__package__`, and
- :attr:`__loader__` attributes are filled in (all but :attr:`__name__` are set
- to ``None``); the caller is responsible for providing a :attr:`__file__`
- attribute.
+ Return a new module object with :attr:`module.__name__` set to *name*.
+ The module's :attr:`!__name__`, :attr:`~module.__doc__`,
+ :attr:`~module.__package__` and :attr:`~module.__loader__` attributes are
+ filled in (all but :attr:`!__name__` are set to ``None``). The caller is
+ responsible for setting a :attr:`~module.__file__` attribute.
Return ``NULL`` with an exception set on error.
.. versionadded:: 3.3
.. versionchanged:: 3.4
- :attr:`__package__` and :attr:`__loader__` are set to ``None``.
+ :attr:`~module.__package__` and :attr:`~module.__loader__` are now set to
+ ``None``.
.. c:function:: PyObject* PyModule_New(const char *name)
@@ -77,8 +78,9 @@ Module Objects
single: __name__ (module attribute)
single: SystemError (built-in exception)
- Return *module*'s :attr:`__name__` value. If the module does not provide one,
- or if it is not a string, :exc:`SystemError` is raised and ``NULL`` is returned.
+ Return *module*'s :attr:`~module.__name__` value. If the module does not
+ provide one, or if it is not a string, :exc:`SystemError` is raised and
+ ``NULL`` is returned.
.. versionadded:: 3.3
@@ -108,8 +110,8 @@ Module Objects
single: SystemError (built-in exception)
Return the name of the file from which *module* was loaded using *module*'s
- :attr:`__file__` attribute. If this is not defined, or if it is not a
- unicode string, raise :exc:`SystemError` and return ``NULL``; otherwise return
+ :attr:`~module.__file__` attribute. If this is not defined, or if it is not a
+ string, raise :exc:`SystemError` and return ``NULL``; otherwise return
a reference to a Unicode object.
.. versionadded:: 3.2
diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst
index d1ad3300a9f6d2..5f01e093ce1b8b 100644
--- a/Doc/deprecations/pending-removal-in-3.14.rst
+++ b/Doc/deprecations/pending-removal-in-3.14.rst
@@ -1,6 +1,13 @@
Pending Removal in Python 3.14
------------------------------
+* The import system:
+
+ * Setting :attr:`~module.__loader__` on a module while
+ failing to set :attr:`__spec__.loader `
+ is deprecated. In Python 3.14, :attr:`!__loader__` will cease to be set or
+ taken into consideration by the import system or the standard library.
+
* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters
of :class:`!argparse.BooleanOptionalAction` are deprecated
and will be removed in 3.14.
diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index b233967adc9e6a..9b29699e866433 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -437,7 +437,7 @@ Glossary
` for use with :data:`sys.meta_path`, and :term:`path
entry finders ` for use with :data:`sys.path_hooks`.
- See :ref:`importsystem` and :mod:`importlib` for much more detail.
+ See :ref:`finders-and-loaders` and :mod:`importlib` for much more detail.
floor division
Mathematical division that rounds down to nearest integer. The floor
@@ -750,8 +750,11 @@ Glossary
loader
An object that loads a module. It must define a method named
:meth:`load_module`. A loader is typically returned by a
- :term:`finder`. See :pep:`302` for details and
- :class:`importlib.abc.Loader` for an :term:`abstract base class`.
+ :term:`finder`. See also:
+
+ * :ref:`finders-and-loaders`
+ * :class:`importlib.abc.Loader`
+ * :pep:`302`
locale encoding
On Unix, it is the encoding of the LC_CTYPE locale. It can be set with
@@ -821,6 +824,8 @@ Glossary
A namespace containing the import-related information used to load a
module. An instance of :class:`importlib.machinery.ModuleSpec`.
+ See also :ref:`module-specs`.
+
MRO
See :term:`method resolution order`.
diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst
index e6010f3c64e352..6a98d9b3c1167e 100644
--- a/Doc/library/ast.rst
+++ b/Doc/library/ast.rst
@@ -889,7 +889,7 @@ Statements
(indicating a "simple" target). A "simple" target consists solely of a
:class:`Name` node that does not appear between parentheses; all other
targets are considered complex. Only simple targets appear in
- the :attr:`__annotations__` dictionary of modules and classes.
+ the :attr:`~object.__annotations__` dictionary of modules and classes.
.. doctest::
diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst
index 7ecfc6ad3c4e1c..1dacbe64b748db 100644
--- a/Doc/library/importlib.rst
+++ b/Doc/library/importlib.rst
@@ -249,7 +249,7 @@ ABC hierarchy::
An abstract method for finding a :term:`spec ` for
the specified module. If this is a top-level import, *path* will
be ``None``. Otherwise, this is a search for a subpackage or
- module and *path* will be the value of :attr:`__path__` from the
+ module and *path* will be the value of :attr:`~module.__path__` from the
parent package. If a spec cannot be found, ``None`` is returned.
When passed in, ``target`` is a module object that the finder may
use to make a more educated guess about what spec to return.
@@ -355,34 +355,12 @@ ABC hierarchy::
(note that some of these attributes can change when a module is
reloaded):
- - :attr:`__name__`
- The module's fully qualified name.
- It is ``'__main__'`` for an executed module.
-
- - :attr:`__file__`
- The location the :term:`loader` used to load the module.
- For example, for modules loaded from a .py file this is the filename.
- It is not set on all modules (e.g. built-in modules).
-
- - :attr:`__cached__`
- The filename of a compiled version of the module's code.
- It is not set on all modules (e.g. built-in modules).
-
- - :attr:`__path__`
- The list of locations where the package's submodules will be found.
- Most of the time this is a single directory.
- The import system passes this attribute to ``__import__()`` and to finders
- in the same way as :data:`sys.path` but just for the package.
- It is not set on non-package modules so it can be used
- as an indicator that the module is a package.
-
- - :attr:`__package__`
- The fully qualified name of the package the module is in (or the
- empty string for a top-level module).
- If the module is a package then this is the same as :attr:`__name__`.
-
- - :attr:`__loader__`
- The :term:`loader` used to load the module.
+ - :attr:`module.__name__`
+ - :attr:`module.__file__`
+ - :attr:`module.__cached__`
+ - :attr:`module.__path__`
+ - :attr:`module.__package__`
+ - :attr:`module.__loader__` *(deprecated)*
When :meth:`exec_module` is available then backwards-compatible
functionality is provided.
@@ -418,7 +396,8 @@ ABC hierarchy::
can implement this abstract method to give direct access
to the data stored. :exc:`OSError` is to be raised if the *path* cannot
be found. The *path* is expected to be constructed using a module's
- :attr:`__file__` attribute or an item from a package's :attr:`__path__`.
+ :attr:`~module.__file__` attribute or an item from a package's
+ :attr:`~module.__path__`.
.. versionchanged:: 3.4
Raises :exc:`OSError` instead of :exc:`NotImplementedError`.
@@ -505,9 +484,9 @@ ABC hierarchy::
.. abstractmethod:: get_filename(fullname)
- An abstract method that is to return the value of :attr:`__file__` for
- the specified module. If no path is available, :exc:`ImportError` is
- raised.
+ An abstract method that is to return the value of
+ :attr:`~module.__file__` for the specified module. If no path is
+ available, :exc:`ImportError` is raised.
If source code is available, then the method should return the path to
the source file, regardless of whether a bytecode was used to load the
@@ -1166,43 +1145,45 @@ find and load modules.
.. class:: ModuleSpec(name, loader, *, origin=None, loader_state=None, is_package=None)
A specification for a module's import-system-related state. This is
- typically exposed as the module's :attr:`__spec__` attribute. Many
+ typically exposed as the module's :attr:`~module.__spec__` attribute. Many
of these attributes are also available directly on a module: for example,
``module.__spec__.origin == module.__file__``. Note, however, that
while the *values* are usually equivalent, they can differ since there is
- no synchronization between the two objects. For example, it is possible to update
- the module's :attr:`__file__` at runtime and this will not be automatically
- reflected in the module's :attr:`__spec__.origin`, and vice versa.
+ no synchronization between the two objects. For example, it is possible to
+ update the module's :attr:`~module.__file__` at runtime and this will not be
+ automatically reflected in the module's
+ :attr:`__spec__.origin `, and vice versa.
.. versionadded:: 3.4
.. attribute:: name
- The module's fully qualified name
- (see :attr:`__name__` attributes on modules).
+ The module's fully qualified name (see :attr:`module.__name__`).
The :term:`finder` should always set this attribute to a non-empty string.
.. attribute:: loader
- The :term:`loader` used to load the module
- (see :attr:`__loader__` attributes on modules).
+ The :term:`loader` used to load the module (see :attr:`module.__loader__`).
The :term:`finder` should always set this attribute.
.. attribute:: origin
The location the :term:`loader` should use to load the module
- (see :attr:`__file__` attributes on modules).
- For example, for modules loaded from a .py file this is the filename.
+ (see :attr:`module.__file__`).
+ For example, for modules loaded from a ``.py`` file this is the filename.
The :term:`finder` should always set this attribute to a meaningful value
for the :term:`loader` to use. In the uncommon case that there is not one
(like for namespace packages), it should be set to ``None``.
.. attribute:: submodule_search_locations
- The list of locations where the package's submodules will be found
- (see :attr:`__path__` attributes on modules).
- Most of the time this is a single directory.
- The :term:`finder` should set this attribute to a list, even an empty one, to indicate
+ A (possibly empty) :term:`sequence` of strings enumerating the locations
+ in which a package's submodules will be found
+ (see :attr:`module.__path__`). Most of the time there will only be a
+ single directory in this list.
+
+ The :term:`finder` should set this attribute to a sequence, even an empty
+ one, to indicate
to the import system that the module is a package. It should be set to ``None`` for
non-package modules. It is set automatically later to a special object for
namespace packages.
@@ -1216,7 +1197,7 @@ find and load modules.
.. attribute:: cached
The filename of a compiled version of the module's code
- (see :attr:`__cached__` attributes on modules).
+ (see :attr:`module.__cached__`).
The :term:`finder` should always set this attribute but it may be ``None``
for modules that do not need compiled code stored.
@@ -1224,14 +1205,14 @@ find and load modules.
(Read-only) The fully qualified name of the package the module is in (or the
empty string for a top-level module).
- See :attr:`__package__` attributes on modules.
+ See :attr:`module.__package__`.
If the module is a package then this is the same as :attr:`name`.
.. attribute:: has_location
``True`` if the spec's :attr:`origin` refers to a loadable location,
- ``False`` otherwise. This value impacts how :attr:`origin` is interpreted
- and how the module's :attr:`__file__` is populated.
+ ``False`` otherwise. This value impacts how :attr:`!origin` is interpreted
+ and how the module's :attr:`~module.__file__` is populated.
:mod:`importlib.util` -- Utility code for importers
@@ -1353,8 +1334,8 @@ an :term:`importer`.
.. versionchanged:: 3.7
Raises :exc:`ModuleNotFoundError` instead of :exc:`AttributeError` if
- **package** is in fact not a package (i.e. lacks a :attr:`__path__`
- attribute).
+ **package** is in fact not a package (i.e. lacks a
+ :attr:`~module.__path__` attribute).
.. function:: module_from_spec(spec)
diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst
index f095cc84173737..4a39d53a5f1440 100644
--- a/Doc/library/pkgutil.rst
+++ b/Doc/library/pkgutil.rst
@@ -26,7 +26,8 @@ support.
__path__ = extend_path(__path__, __name__)
For each directory on :data:`sys.path` that has a subdirectory that matches the
- package name, add the subdirectory to the package's :attr:`__path__`. This is useful
+ package name, add the subdirectory to the package's
+ :attr:`~module.__path__`. This is useful
if one wants to distribute different parts of a single logical package as multiple
directories.
diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index 40d0ef80a3fe43..bf2ea94f23f84d 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -1239,7 +1239,8 @@ always available.
that implement Python's default import semantics. The
:meth:`~importlib.abc.MetaPathFinder.find_spec` method is called with at
least the absolute name of the module being imported. If the module to be
- imported is contained in a package, then the parent package's :attr:`__path__`
+ imported is contained in a package, then the parent package's
+ :attr:`~module.__path__`
attribute is passed in as a second argument. The method returns a
:term:`module spec`, or ``None`` if the module cannot be found.
diff --git a/Doc/library/types.rst b/Doc/library/types.rst
index 00a126ef6de306..ce31a24888686c 100644
--- a/Doc/library/types.rst
+++ b/Doc/library/types.rst
@@ -260,63 +260,18 @@ Standard names are defined for the following types:
The type of :term:`modules `. The constructor takes the name of the
module to be created and optionally its :term:`docstring`.
- .. note::
- Use :func:`importlib.util.module_from_spec` to create a new module if you
- wish to set the various import-controlled attributes.
-
- .. attribute:: __doc__
-
- The :term:`docstring` of the module. Defaults to ``None``.
-
- .. attribute:: __loader__
-
- The :term:`loader` which loaded the module. Defaults to ``None``.
-
- This attribute is to match :attr:`importlib.machinery.ModuleSpec.loader`
- as stored in the :attr:`__spec__` object.
-
- .. note::
- A future version of Python may stop setting this attribute by default.
- To guard against this potential change, preferably read from the
- :attr:`__spec__` attribute instead or use
- ``getattr(module, "__loader__", None)`` if you explicitly need to use
- this attribute.
-
- .. versionchanged:: 3.4
- Defaults to ``None``. Previously the attribute was optional.
-
- .. attribute:: __name__
-
- The name of the module. Expected to match
- :attr:`importlib.machinery.ModuleSpec.name`.
-
- .. attribute:: __package__
-
- Which :term:`package` a module belongs to. If the module is top-level
- (i.e. not a part of any specific package) then the attribute should be set
- to ``''``, else it should be set to the name of the package (which can be
- :attr:`__name__` if the module is a package itself). Defaults to ``None``.
-
- This attribute is to match :attr:`importlib.machinery.ModuleSpec.parent`
- as stored in the :attr:`__spec__` object.
-
- .. note::
- A future version of Python may stop setting this attribute by default.
- To guard against this potential change, preferably read from the
- :attr:`__spec__` attribute instead or use
- ``getattr(module, "__package__", None)`` if you explicitly need to use
- this attribute.
-
- .. versionchanged:: 3.4
- Defaults to ``None``. Previously the attribute was optional.
-
- .. attribute:: __spec__
-
- A record of the module's import-system-related state. Expected to be an
- instance of :class:`importlib.machinery.ModuleSpec`.
+ .. seealso::
- .. versionadded:: 3.4
+ :ref:`Documentation on module objects `
+ Provides details on the special attributes that can be found on
+ instances of :class:`!ModuleType`.
+ :func:`importlib.util.module_from_spec`
+ Modules created using the :class:`!ModuleType` constructor are
+ created with many of their special attributes unset or set to default
+ values. :func:`!module_from_spec` provides a more robust way of
+ creating :class:`!ModuleType` instances which ensures the various
+ attributes are set appropriately.
.. data:: EllipsisType
diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index d1993fedb0ca44..7c0719bd93d5da 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -849,6 +849,8 @@ Instances of arbitrary classes can be made callable by defining a
:meth:`~object.__call__` method in their class.
+.. _module-objects:
+
Modules
-------
@@ -874,46 +876,222 @@ Attribute assignment updates the module's namespace dictionary, e.g.,
.. index::
single: __name__ (module attribute)
- single: __doc__ (module attribute)
+ single: __spec__ (module attribute)
+ single: __package__ (module attribute)
+ single: __loader__ (module attribute)
+ single: __path__ (module attribute)
single: __file__ (module attribute)
+ single: __cached__ (module attribute)
+ single: __doc__ (module attribute)
single: __annotations__ (module attribute)
pair: module; namespace
-Predefined (writable) attributes:
+.. _import-mod-attrs:
+
+Import-related attributes on module objects
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Module objects have the following attributes that relate to the
+:ref:`import system `. When a module is created using the machinery associated
+with the import system, these attributes are filled in based on the module's
+:term:`spec `, before the :term:`loader` executes and loads the
+module.
+
+To create a module dynamically rather than using the import system,
+it's recommended to use :func:`importlib.util.module_from_spec`,
+which will set the various import-controlled attributes to appropriate values.
+It's also possible to use the :class:`types.ModuleType` constructor to create
+modules directly, but this technique is more error-prone, as most attributes
+must be manually set on the module object after it has been created when using
+this approach.
+
+.. caution::
+
+ With the exception of :attr:`~module.__name__`, it is **strongly**
+ recommended that you rely on :attr:`~module.__spec__` and its attributes
+ instead of any of the other individual attributes listed in this subsection.
+ Note that updating an attribute on :attr:`!__spec__` will not update the
+ corresponding attribute on the module itself:
+
+ .. doctest::
+
+ >>> import typing
+ >>> typing.__name__, typing.__spec__.name
+ ('typing', 'typing')
+ >>> typing.__spec__.name = 'spelling'
+ >>> typing.__name__, typing.__spec__.name
+ ('typing', 'spelling')
+ >>> typing.__name__ = 'keyboard_smashing'
+ >>> typing.__name__, typing.__spec__.name
+ ('keyboard_smashing', 'spelling')
- :attr:`__name__`
- The module's name.
+.. attribute:: module.__name__
- :attr:`__doc__`
- The module's documentation string, or ``None`` if
- unavailable.
+ The name used to uniquely identify the module in the import system.
+ For a directly executed module, this will be set to ``"__main__"``.
+
+ This attribute must be set to the fully qualified name of the module.
+ It is expected to match the value of
+ :attr:`module.__spec__.name `.
+
+.. attribute:: module.__spec__
+
+ A record of the module's import-system-related state.
+
+ Set to the :class:`module spec ` that was
+ used when importing the module. See :ref:`module-specs` for more details.
+
+ .. versionadded:: 3.4
+
+.. attribute:: module.__package__
+
+ The :term:`package` a module belongs to.
+
+ If the module is top-level (that is, not a part of any specific package)
+ then the attribute should be set to ``''`` (the empty string). Otherwise,
+ it should be set to the name of the module's package (which can be equal to
+ :attr:`module.__name__` if the module itself is a package). See :pep:`366`
+ for further details.
+
+ This attribute is used instead of :attr:`~module.__name__` to calculate
+ explicit relative imports for main modules. It defaults to ``None`` for
+ modules created dynamically using the :class:`types.ModuleType` constructor;
+ use :func:`importlib.util.module_from_spec` instead to ensure the attribute
+ is set to a :class:`str`.
+
+ It is **strongly** recommended that you use
+ :attr:`module.__spec__.parent `
+ instead of :attr:`!module.__package__`. :attr:`__package__` is now only used
+ as a fallback if :attr:`!__spec__.parent` is not set, and this fallback
+ path is deprecated.
+
+ .. versionchanged:: 3.4
+ This attribute now defaults to ``None`` for modules created dynamically
+ using the :class:`types.ModuleType` constructor.
+ Previously the attribute was optional.
- :attr:`__file__`
- The pathname of the file from which the
- module was loaded, if it was loaded from a file.
- The :attr:`__file__`
- attribute may be missing for certain types of modules, such as C modules
- that are statically linked into the interpreter. For extension modules
- loaded dynamically from a shared library, it's the pathname of the shared
- library file.
+ .. versionchanged:: 3.6
+ The value of :attr:`!__package__` is expected to be the same as
+ :attr:`__spec__.parent `.
+ :attr:`__package__` is now only used as a fallback during import
+ resolution if :attr:`!__spec__.parent` is not defined.
- :attr:`__annotations__`
- A dictionary containing
- :term:`variable annotations ` collected during
- module body execution. For best practices on working
- with :attr:`__annotations__`, please see :ref:`annotations-howto`.
+ .. versionchanged:: 3.10
+ :exc:`ImportWarning` is raised if an import resolution falls back to
+ :attr:`!__package__` instead of
+ :attr:`__spec__.parent `.
+
+ .. versionchanged:: 3.12
+ Raise :exc:`DeprecationWarning` instead of :exc:`ImportWarning` when
+ falling back to :attr:`!__package__` during import resolution.
+
+.. attribute:: module.__loader__
+
+ The :term:`loader` object that the import machinery used to load the module.
+
+ This attribute is mostly useful for introspection, but can be used for
+ additional loader-specific functionality, for example getting data
+ associated with a loader.
+
+ :attr:`!__loader__` defaults to ``None`` for modules created dynamically
+ using the :class:`types.ModuleType` constructor;
+ use :func:`importlib.util.module_from_spec` instead to ensure the attribute
+ is set to a :term:`loader` object.
+
+ It is **strongly** recommended that you use
+ :attr:`module.__spec__.loader `
+ instead of :attr:`!module.__loader__`.
+
+ .. versionchanged:: 3.4
+ This attribute now defaults to ``None`` for modules created dynamically
+ using the :class:`types.ModuleType` constructor.
+ Previously the attribute was optional.
+
+ .. deprecated-removed:: 3.12 3.14
+ Setting :attr:`!__loader__` on a module while failing to set
+ :attr:`!__spec__.loader` is deprecated. In Python 3.14,
+ :attr:`!__loader__` will cease to be set or taken into consideration by
+ the import system or the standard library.
+
+.. attribute:: module.__path__
+
+ A (possibly empty) :term:`sequence` of strings enumerating the locations
+ where the package's submodules will be found. Non-package modules should
+ not have a :attr:`!__path__` attribute. See :ref:`package-path-rules` for
+ more details.
+
+ It is **strongly** recommended that you use
+ :attr:`module.__spec__.submodule_search_locations `
+ instead of :attr:`!module.__path__`.
+
+.. attribute:: module.__file__
+.. attribute:: module.__cached__
+
+ :attr:`!__file__` and :attr:`!__cached__` are both optional attributes that
+ may or may not be set. Both attributes should be a :class:`str` when they
+ are available.
+
+ :attr:`!__file__` indicates the pathname of the file from which the module
+ was loaded (if loaded from a file), or the pathname of the shared library
+ file for extension modules loaded dynamically from a shared library.
+ It might be missing for certain types of modules, such as C modules that are
+ statically linked into the interpreter, and the
+ :ref:`import system ` may opt to leave it unset if it
+ has no semantic meaning (for example, a module loaded from a database).
+
+ If :attr:`!__file__` is set then the :attr:`!__cached__` attribute might
+ also be set, which is the path to any compiled version of
+ the code (for example, a byte-compiled file). The file does not need to exist
+ to set this attribute; the path can simply point to where the
+ compiled file *would* exist (see :pep:`3147`).
+
+ Note that :attr:`!__cached__` may be set even if :attr:`!__file__` is not
+ set. However, that scenario is quite atypical. Ultimately, the
+ :term:`loader` is what makes use of the module spec provided by the
+ :term:`finder` (from which :attr:`!__file__` and :attr:`!__cached__` are
+ derived). So if a loader can load from a cached module but otherwise does
+ not load from a file, that atypical scenario may be appropriate.
+
+ It is **strongly** recommended that you use
+ :attr:`module.__spec__.cached `
+ instead of :attr:`!module.__cached__`.
+
+Other writable attributes on module objects
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+As well as the import-related attributes listed above, module objects also have
+the following writable attributes:
+
+.. attribute:: module.__doc__
+
+ The module's documentation string, or ``None`` if unavailable.
+ See also: :attr:`__doc__ attributes `.
+
+.. attribute:: module.__annotations__
+
+ A dictionary containing
+ :term:`variable annotations ` collected during module
+ body execution. For best practices on working with :attr:`__annotations__`,
+ please see :ref:`annotations-howto`.
+
+Module dictionaries
+^^^^^^^^^^^^^^^^^^^
+
+Module objects also have the following special read-only attribute:
.. index:: single: __dict__ (module attribute)
+.. attribute:: module.__dict__
-Special read-only attribute: :attr:`~object.__dict__` is the module's
-namespace as a dictionary object.
+ The module's namespace as a dictionary object. Uniquely among the attributes
+ listed here, :attr:`!__dict__` cannot be accessed as a global variable from
+ within a module; it can only be accessed as an attribute on module objects.
-.. impl-detail::
+ .. impl-detail::
- Because of the way CPython clears module dictionaries, the module
- dictionary will be cleared when the module falls out of scope even if the
- dictionary still has live references. To avoid this, copy the dictionary
- or keep the module around while using its dictionary directly.
+ Because of the way CPython clears module dictionaries, the module
+ dictionary will be cleared when the module falls out of scope even if the
+ dictionary still has live references. To avoid this, copy the dictionary
+ or keep the module around while using its dictionary directly.
.. _class-attrs-and-methods:
diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst
index 7de995b12702ec..ac363e8cfa00dc 100644
--- a/Doc/reference/import.rst
+++ b/Doc/reference/import.rst
@@ -513,8 +513,10 @@ holding is that if you have ``sys.modules['spam']`` and
``sys.modules['spam.foo']`` (as you would after the above import), the latter
must appear as the ``foo`` attribute of the former.
-Module spec
------------
+.. _module-specs:
+
+Module specs
+------------
The import machinery uses a variety of information about each module
during import, especially before loading. Most of the information is
@@ -527,155 +529,44 @@ and the loader that executes it. Most importantly, it allows the
import machinery to perform the boilerplate operations of loading,
whereas without a module spec the loader had that responsibility.
-The module's spec is exposed as the ``__spec__`` attribute on a module object.
+The module's spec is exposed as :attr:`module.__spec__`. Setting
+:attr:`!__spec__` appropriately applies equally to
+:ref:`modules initialized during interpreter startup `.
+The one exception is ``__main__``, where :attr:`!__spec__` is
+:ref:`set to None in some cases `.
+
See :class:`~importlib.machinery.ModuleSpec` for details on the contents of
the module spec.
.. versionadded:: 3.4
-.. _import-mod-attrs:
-
-Import-related module attributes
---------------------------------
-
-The import machinery fills in these attributes on each module object
-during loading, based on the module's spec, before the loader executes
-the module.
-
-It is **strongly** recommended that you rely on :attr:`__spec__` and
-its attributes instead of any of the other individual attributes
-listed below.
-
-.. attribute:: __name__
-
- The ``__name__`` attribute must be set to the fully qualified name of
- the module. This name is used to uniquely identify the module in
- the import system.
-
-.. attribute:: __loader__
-
- The ``__loader__`` attribute must be set to the loader object that
- the import machinery used when loading the module. This is mostly
- for introspection, but can be used for additional loader-specific
- functionality, for example getting data associated with a loader.
-
- It is **strongly** recommended that you rely on :attr:`__spec__`
- instead of this attribute.
-
- .. versionchanged:: 3.12
- The value of ``__loader__`` is expected to be the same as
- ``__spec__.loader``. The use of ``__loader__`` is deprecated and slated
- for removal in Python 3.14.
-
-.. attribute:: __package__
-
- The module's ``__package__`` attribute may be set. Its value must
- be a string, but it can be the same value as its ``__name__``. When
- the module is a package, its ``__package__`` value should be set to
- its ``__name__``. When the module is not a package, ``__package__``
- should be set to the empty string for top-level modules, or for
- submodules, to the parent package's name. See :pep:`366` for further
- details.
-
- This attribute is used instead of ``__name__`` to calculate explicit
- relative imports for main modules, as defined in :pep:`366`.
-
- It is **strongly** recommended that you rely on :attr:`__spec__`
- instead of this attribute.
-
- .. versionchanged:: 3.6
- The value of ``__package__`` is expected to be the same as
- ``__spec__.parent``.
-
- .. versionchanged:: 3.10
- :exc:`ImportWarning` is raised if import falls back to
- ``__package__`` instead of
- :attr:`~importlib.machinery.ModuleSpec.parent`.
-
- .. versionchanged:: 3.12
- Raise :exc:`DeprecationWarning` instead of :exc:`ImportWarning`
- when falling back to ``__package__``.
-
-
-.. attribute:: __spec__
-
- The ``__spec__`` attribute must be set to the module spec that was
- used when importing the module. Setting ``__spec__``
- appropriately applies equally to :ref:`modules initialized during
- interpreter startup `. The one exception is ``__main__``,
- where ``__spec__`` is :ref:`set to None in some cases `.
-
- When ``__spec__.parent`` is not set, ``__package__`` is used as
- a fallback.
-
- .. versionadded:: 3.4
-
- .. versionchanged:: 3.6
- ``__spec__.parent`` is used as a fallback when ``__package__`` is
- not defined.
-
-.. attribute:: __path__
-
- If the module is a package (either regular or namespace), the module
- object's ``__path__`` attribute must be set. The value must be
- iterable, but may be empty if ``__path__`` has no further significance.
- If ``__path__`` is not empty, it must produce strings when iterated
- over. More details on the semantics of ``__path__`` are given
- :ref:`below `.
-
- Non-package modules should not have a ``__path__`` attribute.
-
-.. attribute:: __file__
-.. attribute:: __cached__
-
- ``__file__`` is optional (if set, value must be a string). It indicates
- the pathname of the file from which the module was loaded (if
- loaded from a file), or the pathname of the shared library file
- for extension modules loaded dynamically from a shared library.
- It might be missing for certain types of modules, such as C
- modules that are statically linked into the interpreter, and the
- import system may opt to leave it unset if it has no semantic
- meaning (e.g. a module loaded from a database).
-
- If ``__file__`` is set then the ``__cached__`` attribute might also
- be set, which is the path to any compiled version of
- the code (e.g. byte-compiled file). The file does not need to exist
- to set this attribute; the path can simply point to where the
- compiled file would exist (see :pep:`3147`).
-
- Note that ``__cached__`` may be set even if ``__file__`` is not
- set. However, that scenario is quite atypical. Ultimately, the
- loader is what makes use of the module spec provided by the finder
- (from which ``__file__`` and ``__cached__`` are derived). So
- if a loader can load from a cached module but otherwise does not load
- from a file, that atypical scenario may be appropriate.
-
- It is **strongly** recommended that you rely on :attr:`__spec__`
- instead of ``__cached__``.
-
.. _package-path-rules:
-module.__path__
----------------
+__path__ attributes on modules
+------------------------------
-By definition, if a module has a ``__path__`` attribute, it is a package.
+The :attr:`~module.__path__` attribute should be a (possibly empty)
+:term:`sequence` of strings enumerating the locations where the package's
+submodules will be found. By definition, if a module has a :attr:`!__path__`
+attribute, it is a :term:`package`.
-A package's ``__path__`` attribute is used during imports of its subpackages.
+A package's :attr:`~module.__path__` attribute is used during imports of its
+subpackages.
Within the import machinery, it functions much the same as :data:`sys.path`,
i.e. providing a list of locations to search for modules during import.
-However, ``__path__`` is typically much more constrained than
-:data:`sys.path`.
+However, :attr:`!__path__` is typically much more constrained than
+:data:`!sys.path`.
-``__path__`` must be an iterable of strings, but it may be empty.
The same rules used for :data:`sys.path` also apply to a package's
-``__path__``, and :data:`sys.path_hooks` (described below) are
-consulted when traversing a package's ``__path__``.
+:attr:`!__path__`. :data:`sys.path_hooks` (described below) are
+consulted when traversing a package's :attr:`!__path__`.
-A package's ``__init__.py`` file may set or alter the package's ``__path__``
+A package's ``__init__.py`` file may set or alter the package's
+:attr:`~module.__path__`
attribute, and this was typically the way namespace packages were implemented
prior to :pep:`420`. With the adoption of :pep:`420`, namespace packages no
-longer need to supply ``__init__.py`` files containing only ``__path__``
-manipulation code; the import machinery automatically sets ``__path__``
+longer need to supply ``__init__.py`` files containing only :attr:`!__path__`
+manipulation code; the import machinery automatically sets :attr:`!__path__`
correctly for the namespace package.
Module reprs
diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst
index 0316239e776a95..de7aa0e2342946 100644
--- a/Doc/tutorial/modules.rst
+++ b/Doc/tutorial/modules.rst
@@ -585,8 +585,9 @@ as the main module of a Python application must always use absolute imports.
Packages in Multiple Directories
--------------------------------
-Packages support one more special attribute, :attr:`__path__`. This is
-initialized to be a list containing the name of the directory holding the
+Packages support one more special attribute, :attr:`~module.__path__`. This is
+initialized to be a :term:`sequence` of strings containing the name of the
+directory holding the
package's :file:`__init__.py` before the code in that file is executed. This
variable can be modified; doing so affects future searches for modules and
subpackages contained in the package.
diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst
index 7e575e82523898..e04684b9f901e5 100644
--- a/Doc/whatsnew/2.6.rst
+++ b/Doc/whatsnew/2.6.rst
@@ -502,12 +502,12 @@ Python's :option:`-m` switch allows running a module as a script.
When you ran a module that was located inside a package, relative
imports didn't work correctly.
-The fix for Python 2.6 adds a :attr:`__package__` attribute to
-modules. When this attribute is present, relative imports will be
+The fix for Python 2.6 adds a :attr:`module.__package__` attribute.
+When this attribute is present, relative imports will be
relative to the value of this attribute instead of the
-:attr:`__name__` attribute.
+:attr:`~module.__name__` attribute.
-PEP 302-style importers can then set :attr:`__package__` as necessary.
+PEP 302-style importers can then set :attr:`~module.__package__` as necessary.
The :mod:`runpy` module that implements the :option:`-m` switch now
does this, so relative imports will now work correctly in scripts
running from inside a package.
diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst
index 22e44671caf79b..766db5ecc67b73 100644
--- a/Doc/whatsnew/3.0.rst
+++ b/Doc/whatsnew/3.0.rst
@@ -357,8 +357,8 @@ New Syntax
provides a standardized way of annotating a function's parameters
and return value. There are no semantics attached to such
annotations except that they can be introspected at runtime using
- the :attr:`__annotations__` attribute. The intent is to encourage
- experimentation through metaclasses, decorators or frameworks.
+ the :attr:`~object.__annotations__` attribute. The intent is to
+ encourage experimentation through metaclasses, decorators or frameworks.
* :pep:`3102`: Keyword-only arguments. Named parameters occurring
after ``*args`` in the parameter list *must* be specified using
diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst
index 5c36e2d4bcfb49..7ec1935ce956d3 100644
--- a/Doc/whatsnew/3.12.rst
+++ b/Doc/whatsnew/3.12.rst
@@ -1320,14 +1320,15 @@ Deprecated
may be removed in a future version of Python. Use the single-arg versions
of these functions instead. (Contributed by Ofey Chan in :gh:`89874`.)
-* :exc:`DeprecationWarning` is now raised when ``__package__`` on a
- module differs from ``__spec__.parent`` (previously it was
- :exc:`ImportWarning`).
+* :exc:`DeprecationWarning` is now raised when :attr:`~module.__package__` on a
+ module differs from
+ :attr:`__spec__.parent ` (previously
+ it was :exc:`ImportWarning`).
(Contributed by Brett Cannon in :gh:`65961`.)
-* Setting ``__package__`` or ``__cached__`` on a module is deprecated,
- and will cease to be set or taken into consideration by the import system in Python 3.14.
- (Contributed by Brett Cannon in :gh:`65961`.)
+* Setting :attr:`~module.__package__` or :attr:`~module.__cached__` on a
+ module is deprecated, and will cease to be set or taken into consideration by
+ the import system in Python 3.14. (Contributed by Brett Cannon in :gh:`65961`.)
* The bitwise inversion operator (``~``) on bool is deprecated. It will throw an
error in Python 3.16. Use ``not`` for logical negation of bools instead.
diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst
index 89c717875e7d7e..135ecda4cd75be 100644
--- a/Doc/whatsnew/3.2.rst
+++ b/Doc/whatsnew/3.2.rst
@@ -312,8 +312,8 @@ cluttering source directories, the *pyc* files are now collected in a
Aside from the filenames and target directories, the new scheme has a few
aspects that are visible to the programmer:
-* Imported modules now have a :attr:`__cached__` attribute which stores the name
- of the actual file that was imported:
+* Imported modules now have a :attr:`~module.__cached__` attribute which stores
+ the name of the actual file that was imported:
>>> import collections
>>> collections.__cached__ # doctest: +SKIP
diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst
index 79dbfc5788d9b6..33534ff2c93f9e 100644
--- a/Doc/whatsnew/3.4.rst
+++ b/Doc/whatsnew/3.4.rst
@@ -2271,7 +2271,8 @@ Changes in the Python API
:func:`super` and falling through all the way to the ABCs. For compatibility,
catch both :exc:`NotImplementedError` or the appropriate exception as needed.
-* The module type now initializes the :attr:`__package__` and :attr:`__loader__`
+* The module type now initializes the :attr:`~module.__package__` and
+ :attr:`~module.__loader__`
attributes to ``None`` by default. To determine if these attributes were set
in a backwards-compatible fashion, use e.g.
``getattr(module, '__loader__', None) is not None``. (:issue:`17115`.)
diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst
index fe829fc19fe482..b18fbb00bc62c7 100644
--- a/Doc/whatsnew/3.5.rst
+++ b/Doc/whatsnew/3.5.rst
@@ -423,8 +423,8 @@ are declared in the annotations::
return 'Hello ' + name
While these annotations are available at runtime through the usual
-:attr:`__annotations__` attribute, *no automatic type checking happens at
-runtime*. Instead, it is assumed that a separate off-line type checker
+:attr:`~object.__annotations__` attribute, *no automatic type checking happens
+at runtime*. Instead, it is assumed that a separate off-line type checker
(e.g. `mypy `_) will be used for on-demand
source code analysis.
diff --git a/Misc/NEWS.d/3.10.0a2.rst b/Misc/NEWS.d/3.10.0a2.rst
index bd002b6ad3db9b..3e82de9ef266d6 100644
--- a/Misc/NEWS.d/3.10.0a2.rst
+++ b/Misc/NEWS.d/3.10.0a2.rst
@@ -226,8 +226,8 @@ thread at the time the function is called.
.. section: Core and Builtins
Enable ``from __future__ import annotations`` (:pep:`563`) by default. The
-values found in :attr:`__annotations__` dicts are now strings, e.g. ``{"x":
-"int"}`` instead of ``{"x": int}``.
+values found in :attr:`~object.__annotations__` dicts are now strings, for
+example ``{"x": "int"}`` instead of ``{"x": int}``.
..
diff --git a/Misc/NEWS.d/3.11.0a5.rst b/Misc/NEWS.d/3.11.0a5.rst
index 954f5c18b48000..5418d5d59dd583 100644
--- a/Misc/NEWS.d/3.11.0a5.rst
+++ b/Misc/NEWS.d/3.11.0a5.rst
@@ -486,8 +486,8 @@ Use ``dis.Positions`` in ``dis.Instruction`` instead of a regular ``tuple``.
.. nonce: geS-aP
.. section: Library
-:mod:`pdb` now gracefully handles ``help`` when :attr:`__doc__` is missing,
-for example when run with pregenerated optimized ``.pyc`` files.
+:mod:`pdb` now gracefully handles ``help`` when :attr:`~module.__doc__` is
+missing, for example when run with pregenerated optimized ``.pyc`` files.
..
diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst
index 305607c297f1f4..f95a1231d5c73b 100644
--- a/Misc/NEWS.d/3.12.0a1.rst
+++ b/Misc/NEWS.d/3.12.0a1.rst
@@ -4237,8 +4237,8 @@ by :mod:`asyncio` to AIX platform only.
.. nonce: 4dzB80
.. section: Library
-Set :attr:`doctest.DocTest.lineno` to ``None`` when object does not have
-:attr:`__doc__`.
+Set :attr:`doctest.DocTest.lineno` to ``None`` when an object does not have
+:attr:`~definition.__doc__`.
..
From caa4924917f2c5e70d9e199a71893a6fc8bc43da Mon Sep 17 00:00:00 2001
From: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
Date: Thu, 10 Oct 2024 10:16:03 +0100
Subject: [PATCH 031/269] [3.12] GH-121970: Extract ``availability`` into a new
extension (GH-125082) (#125238)
(cherry picked from commit cbfd39247983309a9ef0ae6da6c61cc71665b967)
---
Doc/conf.py | 1 +
Doc/tools/extensions/availability.py | 123 +++++++++++++++++++++++++++
Doc/tools/extensions/pyspecific.py | 76 -----------------
3 files changed, 124 insertions(+), 76 deletions(-)
create mode 100644 Doc/tools/extensions/availability.py
diff --git a/Doc/conf.py b/Doc/conf.py
index 95e2e1774479b3..4859063edd24ea 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -21,6 +21,7 @@
extensions = [
'audit_events',
+ 'availability',
'c_annotations',
'glossary_search',
'lexers',
diff --git a/Doc/tools/extensions/availability.py b/Doc/tools/extensions/availability.py
new file mode 100644
index 00000000000000..897af70a9f4b40
--- /dev/null
+++ b/Doc/tools/extensions/availability.py
@@ -0,0 +1,123 @@
+"""Support for documenting platform availability"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from docutils import nodes
+from sphinx import addnodes
+from sphinx.util import logging
+from sphinx.util.docutils import SphinxDirective
+
+if TYPE_CHECKING:
+ from sphinx.application import Sphinx
+ from sphinx.util.typing import ExtensionMetadata
+
+logger = logging.getLogger("availability")
+
+# known platform, libc, and threading implementations
+_PLATFORMS = frozenset({
+ "AIX",
+ "Android",
+ "BSD",
+ "DragonFlyBSD",
+ "Emscripten",
+ "FreeBSD",
+ "Linux",
+ "macOS",
+ "NetBSD",
+ "OpenBSD",
+ "POSIX",
+ "Solaris",
+ "Unix",
+ "VxWorks",
+ "WASI",
+ "Windows",
+})
+_LIBC = frozenset({
+ "BSD libc",
+ "glibc",
+ "musl",
+})
+_THREADING = frozenset({
+ # POSIX platforms with pthreads
+ "pthreads",
+})
+KNOWN_PLATFORMS = _PLATFORMS | _LIBC | _THREADING
+
+
+class Availability(SphinxDirective):
+ has_content = True
+ required_arguments = 1
+ optional_arguments = 0
+ final_argument_whitespace = True
+
+ def run(self) -> list[nodes.container]:
+ title = "Availability"
+ refnode = addnodes.pending_xref(
+ title,
+ nodes.inline(title, title, classes=["xref", "std", "std-ref"]),
+ refdoc=self.env.docname,
+ refdomain="std",
+ refexplicit=True,
+ reftarget="availability",
+ reftype="ref",
+ refwarn=True,
+ )
+ sep = nodes.Text(": ")
+ parsed, msgs = self.state.inline_text(self.arguments[0], self.lineno)
+ pnode = nodes.paragraph(title, "", refnode, sep, *parsed, *msgs)
+ self.set_source_info(pnode)
+ cnode = nodes.container("", pnode, classes=["availability"])
+ self.set_source_info(cnode)
+ if self.content:
+ self.state.nested_parse(self.content, self.content_offset, cnode)
+ self.parse_platforms()
+
+ return [cnode]
+
+ def parse_platforms(self) -> dict[str, str | bool]:
+ """Parse platform information from arguments
+
+ Arguments is a comma-separated string of platforms. A platform may
+ be prefixed with "not " to indicate that a feature is not available.
+
+ Example::
+
+ .. availability:: Windows, Linux >= 4.2, not WASI
+
+ Arguments like "Linux >= 3.17 with glibc >= 2.27" are currently not
+ parsed into separate tokens.
+ """
+ platforms = {}
+ for arg in self.arguments[0].rstrip(".").split(","):
+ arg = arg.strip()
+ platform, _, version = arg.partition(" >= ")
+ if platform.startswith("not "):
+ version = False
+ platform = platform.removeprefix("not ")
+ elif not version:
+ version = True
+ platforms[platform] = version
+
+ if unknown := set(platforms).difference(KNOWN_PLATFORMS):
+ logger.warning(
+ "Unknown platform%s or syntax '%s' in '.. availability:: %s', "
+ "see %s:KNOWN_PLATFORMS for a set of known platforms.",
+ "s" if len(platforms) != 1 else "",
+ " ".join(sorted(unknown)),
+ self.arguments[0],
+ __file__,
+ )
+
+ return platforms
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_directive("availability", Availability)
+
+ return {
+ "version": "1.0",
+ "parallel_read_safe": True,
+ "parallel_write_safe": True,
+ }
diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py
index 96a4f24fad33b7..d3a4835edd58a4 100644
--- a/Doc/tools/extensions/pyspecific.py
+++ b/Doc/tools/extensions/pyspecific.py
@@ -24,7 +24,6 @@
from sphinx.domains.changeset import VersionChange, versionlabels, versionlabel_classes
from sphinx.domains.python import PyFunction, PyMethod, PyModule
from sphinx.locale import _ as sphinx_gettext
-from sphinx.util import logging
from sphinx.util.docutils import SphinxDirective
from sphinx.writers.text import TextWriter, TextTranslator
from sphinx.util.display import status_iterator
@@ -108,80 +107,6 @@ def run(self):
return [pnode]
-# Support for documenting platform availability
-
-class Availability(SphinxDirective):
-
- has_content = True
- required_arguments = 1
- optional_arguments = 0
- final_argument_whitespace = True
-
- # known platform, libc, and threading implementations
- known_platforms = frozenset({
- "AIX", "Android", "BSD", "DragonFlyBSD", "Emscripten", "FreeBSD",
- "Linux", "NetBSD", "OpenBSD", "POSIX", "Solaris", "Unix", "VxWorks",
- "WASI", "Windows", "macOS",
- # libc
- "BSD libc", "glibc", "musl",
- # POSIX platforms with pthreads
- "pthreads",
- })
-
- def run(self):
- availability_ref = ':ref:`Availability `: '
- avail_nodes, avail_msgs = self.state.inline_text(
- availability_ref + self.arguments[0],
- self.lineno)
- pnode = nodes.paragraph(availability_ref + self.arguments[0],
- '', *avail_nodes, *avail_msgs)
- self.set_source_info(pnode)
- cnode = nodes.container("", pnode, classes=["availability"])
- self.set_source_info(cnode)
- if self.content:
- self.state.nested_parse(self.content, self.content_offset, cnode)
- self.parse_platforms()
-
- return [cnode]
-
- def parse_platforms(self):
- """Parse platform information from arguments
-
- Arguments is a comma-separated string of platforms. A platform may
- be prefixed with "not " to indicate that a feature is not available.
-
- Example::
-
- .. availability:: Windows, Linux >= 4.2, not Emscripten, not WASI
-
- Arguments like "Linux >= 3.17 with glibc >= 2.27" are currently not
- parsed into separate tokens.
- """
- platforms = {}
- for arg in self.arguments[0].rstrip(".").split(","):
- arg = arg.strip()
- platform, _, version = arg.partition(" >= ")
- if platform.startswith("not "):
- version = False
- platform = platform[4:]
- elif not version:
- version = True
- platforms[platform] = version
-
- unknown = set(platforms).difference(self.known_platforms)
- if unknown:
- cls = type(self)
- logger = logging.getLogger(cls.__qualname__)
- logger.warning(
- f"Unknown platform(s) or syntax '{' '.join(sorted(unknown))}' "
- f"in '.. availability:: {self.arguments[0]}', see "
- f"{__file__}:{cls.__qualname__}.known_platforms for a set "
- "known platforms."
- )
-
- return platforms
-
-
# Support for documenting decorators
class PyDecoratorMixin(object):
@@ -473,7 +398,6 @@ def setup(app):
app.add_role('issue', issue_role)
app.add_role('gh', gh_issue_role)
app.add_directive('impl-detail', ImplementationDetail)
- app.add_directive('availability', Availability)
app.add_directive('deprecated-removed', DeprecatedRemoved)
app.add_builder(PydocTopicsBuilder)
app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature)
From 01d9a899978626615d172aa5d832aaf0e4500bc7 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 10 Oct 2024 11:52:54 +0200
Subject: [PATCH 032/269] [3.12] Pin the doctest workflow to Ubuntu 22.04
(GH-125236) (#125241)
Pin the doctest workflow to Ubuntu 22.04 (GH-125236)
(cherry picked from commit 7a10cdec359750b5154490fa9e24475c90d05aab)
Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
---
.github/workflows/reusable-docs.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml
index 4b384f4b3fa602..55c42ff396e05a 100644
--- a/.github/workflows/reusable-docs.yml
+++ b/.github/workflows/reusable-docs.yml
@@ -92,7 +92,7 @@ jobs:
# Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release
doctest:
name: 'Doctest'
- runs-on: ubuntu-latest
+ runs-on: ubuntu-22.04
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
From 01daccff3c2c7b389a2c2ef5a7f80f1b733f5881 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 10 Oct 2024 12:37:14 +0200
Subject: [PATCH 033/269] [3.12] gh-71784: [doc] add usage examples for
traceback.TracebackException (GH-125189) (#125248)
Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com>
Co-authored-by: Alex Waygood
---
Doc/library/traceback.rst | 133 +++++++++++++++++++++++++++++++++-----
1 file changed, 118 insertions(+), 15 deletions(-)
diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst
index d3f47b9e4fb596..9d57c354523370 100644
--- a/Doc/library/traceback.rst
+++ b/Doc/library/traceback.rst
@@ -8,11 +8,15 @@
--------------
-This module provides a standard interface to extract, format and print stack
-traces of Python programs. It exactly mimics the behavior of the Python
-interpreter when it prints a stack trace. This is useful when you want to print
-stack traces under program control, such as in a "wrapper" around the
-interpreter.
+This module provides a standard interface to extract, format and print
+stack traces of Python programs. It is more flexible than the
+interpreter's default traceback display, and therefore makes it
+possible to configure certain aspects of the output. Finally,
+it contains a utility for capturing enough information about an
+exception to print it later, without the need to save a reference
+to the actual exception. Since exceptions can be the roots of large
+objects graph, this utility can significantly improve
+memory management.
.. index:: pair: object; traceback
@@ -29,7 +33,20 @@ which are assigned to the :attr:`~BaseException.__traceback__` field of
Module :mod:`pdb`
Interactive source code debugger for Python programs.
-The module defines the following functions:
+The module's API can be divided into two parts:
+
+* Module-level functions offering basic functionality, which are useful for interactive
+ inspection of exceptions and tracebacks.
+
+* :class:`TracebackException` class and its helper classes
+ :class:`StackSummary` and :class:`FrameSummary`. These offer both more
+ flexibility in the output generated and the ability to store the information
+ necessary for later formatting without holding references to actual exception
+ and traceback objects.
+
+
+Module-Level Functions
+----------------------
.. function:: print_tb(tb, limit=None, file=None)
@@ -230,7 +247,6 @@ The module defines the following functions:
.. versionadded:: 3.5
-The module also defines the following classes:
:class:`!TracebackException` Objects
------------------------------------
@@ -238,12 +254,17 @@ The module also defines the following classes:
.. versionadded:: 3.5
:class:`!TracebackException` objects are created from actual exceptions to
-capture data for later printing in a lightweight fashion.
+capture data for later printing. They offer a more lightweight method of
+storing this information by avoiding holding references to
+:ref:`traceback` and :ref:`frame` objects
+In addition, they expose more options to configure the output compared to
+the module-level functions described above.
.. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, max_group_width=15, max_group_depth=10)
- Capture an exception for later rendering. *limit*, *lookup_lines* and
- *capture_locals* are as for the :class:`StackSummary` class.
+ Capture an exception for later rendering. The meaning of *limit*,
+ *lookup_lines* and *capture_locals* are as for the :class:`StackSummary`
+ class.
If *compact* is true, only data that is required by
:class:`!TracebackException`'s :meth:`format` method
@@ -488,8 +509,8 @@ in a :ref:`traceback `.
.. _traceback-example:
-Traceback Examples
-------------------
+Examples of Using the Module-Level Functions
+--------------------------------------------
This simple example implements a basic read-eval-print loop, similar to (but
less useful than) the standard Python interactive interpreter loop. For a more
@@ -528,8 +549,7 @@ exception and traceback:
try:
lumberjack()
- except IndexError:
- exc = sys.exception()
+ except IndexError as exc:
print("*** print_tb:")
traceback.print_tb(exc.__traceback__, limit=1, file=sys.stdout)
print("*** print_exception:")
@@ -627,5 +647,88 @@ This last example demonstrates the final few formatting functions:
[' File "spam.py", line 3, in \n spam.eggs()\n',
' File "eggs.py", line 42, in eggs\n return "bacon"\n']
>>> an_error = IndexError('tuple index out of range')
- >>> traceback.format_exception_only(type(an_error), an_error)
+ >>> traceback.format_exception_only(an_error)
['IndexError: tuple index out of range\n']
+
+
+Examples of Using :class:`TracebackException`
+---------------------------------------------
+
+With the helper class, we have more options::
+
+ >>> import sys
+ >>> from traceback import TracebackException
+ >>>
+ >>> def lumberjack():
+ ... bright_side_of_life()
+ ...
+ >>> def bright_side_of_life():
+ ... t = "bright", "side", "of", "life"
+ ... return t[5]
+ ...
+ >>> try:
+ ... lumberjack()
+ ... except IndexError as e:
+ ... exc = e
+ ...
+ >>> try:
+ ... try:
+ ... lumberjack()
+ ... except:
+ ... 1/0
+ ... except Exception as e:
+ ... chained_exc = e
+ ...
+ >>> # limit works as with the module-level functions
+ >>> TracebackException.from_exception(exc, limit=-2).print()
+ Traceback (most recent call last):
+ File "", line 6, in lumberjack
+ bright_side_of_life()
+ ~~~~~~~~~~~~~~~~~~~^^
+ File "", line 10, in bright_side_of_life
+ return t[5]
+ ~^^^
+ IndexError: tuple index out of range
+
+ >>> # capture_locals adds local variables in frames
+ >>> TracebackException.from_exception(exc, limit=-2, capture_locals=True).print()
+ Traceback (most recent call last):
+ File "", line 6, in lumberjack
+ bright_side_of_life()
+ ~~~~~~~~~~~~~~~~~~~^^
+ File "", line 10, in bright_side_of_life
+ return t[5]
+ ~^^^
+ t = ("bright", "side", "of", "life")
+ IndexError: tuple index out of range
+
+ >>> # The *chain* kwarg to print() controls whether chained
+ >>> # exceptions are displayed
+ >>> TracebackException.from_exception(chained_exc).print()
+ Traceback (most recent call last):
+ File "", line 4, in
+ lumberjack()
+ ~~~~~~~~~~^^
+ File "", line 7, in lumberjack
+ bright_side_of_life()
+ ~~~~~~~~~~~~~~~~~~~^^
+ File "", line 11, in bright_side_of_life
+ return t[5]
+ ~^^^
+ IndexError: tuple index out of range
+
+ During handling of the above exception, another exception occurred:
+
+ Traceback (most recent call last):
+ File "", line 6, in
+ 1/0
+ ~^~
+ ZeroDivisionError: division by zero
+
+ >>> TracebackException.from_exception(chained_exc).print(chain=False)
+ Traceback (most recent call last):
+ File "", line 6, in
+ 1/0
+ ~^~
+ ZeroDivisionError: division by zero
+
From 74df3a79d308d036e79b600b1e7850e7828e6b39 Mon Sep 17 00:00:00 2001
From: Donghee Na
Date: Thu, 10 Oct 2024 22:01:41 +0900
Subject: [PATCH 034/269] [3.12] gh-124471: Set name for unnamed reusable
workflow (GH-124475) (gh-125257)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
(cherry picked from commit e4cab488d4445e8444932f3bed1c329c0d9e5038)
Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко)
---
.github/workflows/reusable-change-detection.yml | 4 +---
.github/workflows/reusable-docs.yml | 2 +-
.github/workflows/reusable-macos.yml | 2 ++
.github/workflows/reusable-tsan.yml | 2 ++
.github/workflows/reusable-ubuntu.yml | 2 ++
.github/workflows/reusable-windows-msi.yml | 2 +-
.github/workflows/reusable-windows.yml | 2 ++
7 files changed, 11 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml
index 6f599f75547ceb..5cd6fb39f1e12f 100644
--- a/.github/workflows/reusable-change-detection.yml
+++ b/.github/workflows/reusable-change-detection.yml
@@ -1,6 +1,4 @@
----
-
-name: Change detection
+name: Reusable change detection
on: # yamllint disable-line rule:truthy
workflow_call:
diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml
index 55c42ff396e05a..4d1dc04e8b638a 100644
--- a/.github/workflows/reusable-docs.yml
+++ b/.github/workflows/reusable-docs.yml
@@ -1,4 +1,4 @@
-name: Docs
+name: Reusable Docs
on:
workflow_call:
diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml
index 47f5ee2fb63888..6df272c8839f16 100644
--- a/.github/workflows/reusable-macos.yml
+++ b/.github/workflows/reusable-macos.yml
@@ -1,3 +1,5 @@
+name: Reusable macOS
+
on:
workflow_call:
inputs:
diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml
index 96a9c1b0cda3c3..b20ba062d62d63 100644
--- a/.github/workflows/reusable-tsan.yml
+++ b/.github/workflows/reusable-tsan.yml
@@ -1,3 +1,5 @@
+name: Reusable Thread Sanitizer
+
on:
workflow_call:
inputs:
diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml
index b2ac6f6127d883..a5abec11555083 100644
--- a/.github/workflows/reusable-ubuntu.yml
+++ b/.github/workflows/reusable-ubuntu.yml
@@ -1,3 +1,5 @@
+name: Reusable Ubuntu
+
on:
workflow_call:
inputs:
diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml
index fc34ab7c3eb1f2..abdb1a1982fef8 100644
--- a/.github/workflows/reusable-windows-msi.yml
+++ b/.github/workflows/reusable-windows-msi.yml
@@ -1,4 +1,4 @@
-name: TestsMSI
+name: Reusable Windows MSI
on:
workflow_call:
diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml
index 8b886056fe56b8..9393328fa1458b 100644
--- a/.github/workflows/reusable-windows.yml
+++ b/.github/workflows/reusable-windows.yml
@@ -1,3 +1,5 @@
+name: Reusable Windows
+
on:
workflow_call:
inputs:
From 67f8302b9c94079f73dba23eaa47443fcb6008ad Mon Sep 17 00:00:00 2001
From: Sergey B Kirpichev
Date: Thu, 10 Oct 2024 17:56:49 +0300
Subject: [PATCH 035/269] [3.12] gh-125118: don't copy arbitrary values to
_Bool in the struct module (GH-125169) (#125265)
memcopy'ing arbitrary values to _Bool variable triggers undefined
behaviour. Avoid this.
We assume that `false` is represented by all zero bytes.
Credits to Alex Gaynor.
(cherry picked from commit 87d7315ac57250046372b0d9ae4619ba619c8c87)
Co-authored-by: Sam Gross
Co-authored-by: Victor Stinner
Co-authored-by: Petr Viktorin
---
Lib/test/test_struct.py | 3 +++
.../Library/2024-10-09-07-09-00.gh-issue-125118.J9rQ1S.rst | 1 +
Modules/_struct.c | 5 ++---
3 files changed, 6 insertions(+), 3 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-09-07-09-00.gh-issue-125118.J9rQ1S.rst
diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py
index 39ca78cece1fb0..c683e5dd6c47c6 100644
--- a/Lib/test/test_struct.py
+++ b/Lib/test/test_struct.py
@@ -529,6 +529,9 @@ def __bool__(self):
for c in [b'\x01', b'\x7f', b'\xff', b'\x0f', b'\xf0']:
self.assertTrue(struct.unpack('>?', c)[0])
+ self.assertTrue(struct.unpack('', c)[0])
+ self.assertTrue(struct.unpack('=?', c)[0])
+ self.assertTrue(struct.unpack('@?', c)[0])
def test_count_overflow(self):
hugecount = '{}b'.format(sys.maxsize+1)
diff --git a/Misc/NEWS.d/next/Library/2024-10-09-07-09-00.gh-issue-125118.J9rQ1S.rst b/Misc/NEWS.d/next/Library/2024-10-09-07-09-00.gh-issue-125118.J9rQ1S.rst
new file mode 100644
index 00000000000000..5d57cdbbbc2fe9
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-09-07-09-00.gh-issue-125118.J9rQ1S.rst
@@ -0,0 +1 @@
+Don't copy arbitrary values to :c:expr:`_Bool` in the :mod:`struct` module.
diff --git a/Modules/_struct.c b/Modules/_struct.c
index 84474043b01b28..6532174a0b172a 100644
--- a/Modules/_struct.c
+++ b/Modules/_struct.c
@@ -483,9 +483,8 @@ nu_ulonglong(_structmodulestate *state, const char *p, const formatdef *f)
static PyObject *
nu_bool(_structmodulestate *state, const char *p, const formatdef *f)
{
- _Bool x;
- memcpy((char *)&x, p, sizeof x);
- return PyBool_FromLong(x != 0);
+ const _Bool bool_false = 0;
+ return PyBool_FromLong(memcmp(p, &bool_false, sizeof(_Bool)));
}
From ba292cc3019135fdd3670d72d6e3252a5a99a570 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 10 Oct 2024 19:31:27 +0200
Subject: [PATCH 036/269] [3.12] gh-121607: Edited source file import recipe to
make it more clear (GH-121519) (GH-124081)
gh-121607: Edited source file import recipe to make it more clear (GH-121519)
(cherry picked from commit 38809171b8768517824fb62d48abe2cb0aff8429)
Co-authored-by: Chris Barker
Co-authored-by: Brett Cannon
Co-authored-by: Peter Bierma
---
Doc/library/importlib.rst | 37 +++++++++++++++++++++++++------------
1 file changed, 25 insertions(+), 12 deletions(-)
diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst
index 1dacbe64b748db..e4e09b096f7c04 100644
--- a/Doc/library/importlib.rst
+++ b/Doc/library/importlib.rst
@@ -1495,20 +1495,34 @@ Note that if ``name`` is a submodule (contains a dot),
Importing a source file directly
''''''''''''''''''''''''''''''''
-To import a Python source file directly, use the following recipe::
+This recipe should be used with caution: it is an approximation of an import
+statement where the file path is specified directly, rather than
+:data:`sys.path` being searched. Alternatives should first be considered first,
+such as modifying :data:`sys.path` when a proper module is required, or using
+:func:`runpy.run_path` when the global namespace resulting from running a Python
+file is appropriate.
- import importlib.util
- import sys
+To import a Python source file directly from a path, use the following recipe::
+
+ import importlib.util
+ import sys
- # For illustrative purposes.
- import tokenize
- file_path = tokenize.__file__
- module_name = tokenize.__name__
- spec = importlib.util.spec_from_file_location(module_name, file_path)
- module = importlib.util.module_from_spec(spec)
- sys.modules[module_name] = module
- spec.loader.exec_module(module)
+ def import_from_path(module_name, file_path):
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[module_name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+ # For illustrative purposes only (use of `json` is arbitrary).
+ import json
+ file_path = json.__file__
+ module_name = json.__name__
+
+ # Similar outcome as `import json`.
+ json = import_from_path(module_name, file_path)
Implementing lazy imports
@@ -1534,7 +1548,6 @@ The example below shows how to implement lazy imports::
False
-
Setting up an importer
''''''''''''''''''''''
From eb320f5ac846faa9ad81fc0c07553cf5b181652d Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 10 Oct 2024 20:18:01 +0200
Subject: [PATCH 037/269] [3.12] Note argparse exit code in documentation
(GH-119568) (GH-125275)
(cherry picked from commit 3b87fb74c907510402678bf1b7c4a94df0e5e65a)
Co-authored-by: Justin Kunimune
Co-authored-by: Savannah Ostrowski
---
Doc/library/argparse.rst | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index abe20d7d611695..6d8b207cbc03ab 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -537,7 +537,8 @@ exit_on_error
^^^^^^^^^^^^^
Normally, when you pass an invalid argument list to the :meth:`~ArgumentParser.parse_args`
-method of an :class:`ArgumentParser`, it will exit with error info.
+method of an :class:`ArgumentParser`, it will print a *message* to :data:`sys.stderr` and exit with a status
+code of 2.
If the user would like to catch errors manually, the feature can be enabled by setting
``exit_on_error`` to ``False``::
From 91c7a2cb890856dde865cbc98b70c7421550f49e Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 10 Oct 2024 22:32:40 +0200
Subject: [PATCH 038/269] [3.12] Doc: Upgrade Sphinx to 8.1 (GH-125276)
(#125279)
Doc: Upgrade Sphinx to 8.1 (GH-125276)
(cherry picked from commit dd0ee201da34d1d4a631d77b420728f9233f53f9)
Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
---
Doc/conf.py | 31 ++++++++++++++++++++++++-------
Doc/requirements.txt | 2 +-
2 files changed, 25 insertions(+), 8 deletions(-)
diff --git a/Doc/conf.py b/Doc/conf.py
index 4859063edd24ea..36f6243969c8b2 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -11,6 +11,8 @@
import sys
import time
+import sphinx
+
sys.path.append(os.path.abspath('tools/extensions'))
sys.path.append(os.path.abspath('includes'))
@@ -56,7 +58,10 @@
# General substitutions.
project = 'Python'
-copyright = f"2001-{time.strftime('%Y')}, Python Software Foundation"
+if sphinx.version_info[:2] >= (8, 1):
+ copyright = "2001-%Y, Python Software Foundation"
+else:
+ copyright = f"2001-{time.strftime('%Y')}, Python Software Foundation"
# We look for the Include/patchlevel.h file in the current Python source tree
# and replace the values accordingly.
@@ -339,10 +344,14 @@
}
# This 'Last updated on:' timestamp is inserted at the bottom of every page.
-html_time = int(os.environ.get('SOURCE_DATE_EPOCH', time.time()))
-html_last_updated_fmt = time.strftime(
- '%b %d, %Y (%H:%M UTC)', time.gmtime(html_time)
-)
+html_last_updated_fmt = '%b %d, %Y (%H:%M UTC)'
+if sphinx.version_info[:2] >= (8, 1):
+ html_last_updated_use_utc = True
+else:
+ html_time = int(os.environ.get('SOURCE_DATE_EPOCH', time.time()))
+ html_last_updated_fmt = time.strftime(
+ html_last_updated_fmt, time.gmtime(html_time)
+ )
# Path to find HTML templates.
templates_path = ['tools/templates']
@@ -575,13 +584,21 @@
# mapping unique short aliases to a base URL and a prefix.
# https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html
extlinks = {
- "cve": ("https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-%s", "CVE-%s"),
- "cwe": ("https://cwe.mitre.org/data/definitions/%s.html", "CWE-%s"),
"pypi": ("https://pypi.org/project/%s/", "%s"),
"source": (SOURCE_URI, "%s"),
}
extlinks_detect_hardcoded_links = True
+if sphinx.version_info[:2] < (8, 1):
+ # Sphinx 8.1 has in-built CVE and CWE roles.
+ extlinks |= {
+ "cve": (
+ "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-%s",
+ "CVE-%s",
+ ),
+ "cwe": ("https://cwe.mitre.org/data/definitions/%s.html", "CWE-%s"),
+ }
+
# Options for c_annotations
# -------------------------
diff --git a/Doc/requirements.txt b/Doc/requirements.txt
index bf1028020b7af7..5105786ccf283c 100644
--- a/Doc/requirements.txt
+++ b/Doc/requirements.txt
@@ -6,7 +6,7 @@
# Sphinx version is pinned so that new versions that introduce new warnings
# won't suddenly cause build failures. Updating the version is fine as long
# as no warnings are raised by doing so.
-sphinx~=8.0.0
+sphinx~=8.1.0
blurb
From 171ebcd4df268c77f1ca03339a137ca6a54242e5 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Fri, 11 Oct 2024 08:37:55 +0200
Subject: [PATCH 039/269] [3.12] [3.13] gh-124969: Fix
locale.nl_langinfo(locale.ALT_DIGITS) (GH-124974) (GH-125232) (GH-125284)
(cherry picked from commit 26a93189e4c3674a9e0acbd7923b1f27ff01419e)
Co-authored-by: Serhiy Storchaka
Returns a tuple of up to 100 strings for ALT_DIGITS lookup (an empty tuple on most locales).
Previously it returned the first item of that tuple or an empty string.
(cherry picked from commit 21c04e1a972bd1b6285e0ea41fa107d635bbe43a)
Co-authored-by: Serhiy Storchaka
---
Doc/library/locale.rst | 6 +--
Lib/test/test__locale.py | 42 ++++++++++++++++++-
...-10-08-12-09-09.gh-issue-124969._VBQLq.rst | 3 ++
Modules/_localemodule.c | 29 ++++++++++++-
4 files changed, 75 insertions(+), 5 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-08-12-09-09.gh-issue-124969._VBQLq.rst
diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst
index 60975bf9177dad..f0553d51fedf14 100644
--- a/Doc/library/locale.rst
+++ b/Doc/library/locale.rst
@@ -158,7 +158,8 @@ The :mod:`locale` module defines the following exception and functions:
.. function:: nl_langinfo(option)
- Return some locale-specific information as a string. This function is not
+ Return some locale-specific information as a string (or a tuple for
+ ``ALT_DIGITS``). This function is not
available on all systems, and the set of possible options might also vary
across platforms. The possible argument values are numbers, for which
symbolic constants are available in the locale module.
@@ -311,8 +312,7 @@ The :mod:`locale` module defines the following exception and functions:
.. data:: ALT_DIGITS
- Get a representation of up to 100 values used to represent the values
- 0 to 99.
+ Get a tuple of up to 100 strings used to represent the values 0 to 99.
.. function:: getdefaultlocale([envvars])
diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py
index 0947464bb8c04e..5041def7216197 100644
--- a/Lib/test/test__locale.py
+++ b/Lib/test/test__locale.py
@@ -1,4 +1,4 @@
-from _locale import (setlocale, LC_ALL, LC_CTYPE, LC_NUMERIC, localeconv, Error)
+from _locale import (setlocale, LC_ALL, LC_CTYPE, LC_NUMERIC, LC_TIME, localeconv, Error)
try:
from _locale import (RADIXCHAR, THOUSEP, nl_langinfo)
except ImportError:
@@ -74,6 +74,17 @@ def accept(loc):
'ps_AF': ('\u066b', '\u066c'),
}
+known_alt_digits = {
+ 'C': (0, {}),
+ 'en_US': (0, {}),
+ 'fa_IR': (100, {0: '\u06f0\u06f0', 10: '\u06f1\u06f0', 99: '\u06f9\u06f9'}),
+ 'ja_JP': (100, {0: '\u3007', 10: '\u5341', 99: '\u4e5d\u5341\u4e5d'}),
+ 'lzh_TW': (32, {0: '\u3007', 10: '\u5341', 31: '\u5345\u4e00'}),
+ 'my_MM': (100, {0: '\u1040\u1040', 10: '\u1041\u1040', 99: '\u1049\u1049'}),
+ 'or_IN': (100, {0: '\u0b66', 10: '\u0b67\u0b66', 99: '\u0b6f\u0b6f'}),
+ 'shn_MM': (100, {0: '\u1090\u1090', 10: '\u1091\u1090', 99: '\u1099\u1099'}),
+}
+
if sys.platform == 'win32':
# ps_AF doesn't work on Windows: see bpo-38324 (msg361830)
del known_numerics['ps_AF']
@@ -176,6 +187,35 @@ def test_lc_numeric_basic(self):
if not tested:
self.skipTest('no suitable locales')
+ @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available")
+ @unittest.skipUnless(hasattr(locale, 'ALT_DIGITS'), "requires locale.ALT_DIGITS")
+ @unittest.skipIf(
+ support.is_emscripten or support.is_wasi,
+ "musl libc issue on Emscripten, bpo-46390"
+ )
+ def test_alt_digits_nl_langinfo(self):
+ # Test nl_langinfo(ALT_DIGITS)
+ tested = False
+ for loc, (count, samples) in known_alt_digits.items():
+ with self.subTest(locale=loc):
+ try:
+ setlocale(LC_TIME, loc)
+ setlocale(LC_CTYPE, loc)
+ except Error:
+ self.skipTest(f'no locale {loc!r}')
+ continue
+ with self.subTest(locale=loc):
+ alt_digits = nl_langinfo(locale.ALT_DIGITS)
+ self.assertIsInstance(alt_digits, tuple)
+ if count and not alt_digits and support.is_apple:
+ self.skipTest(f'ALT_DIGITS is not set for locale {loc!r} on Apple platforms')
+ self.assertEqual(len(alt_digits), count)
+ for i in samples:
+ self.assertEqual(alt_digits[i], samples[i])
+ tested = True
+ if not tested:
+ self.skipTest('no suitable locales')
+
def test_float_parsing(self):
# Bug #1391872: Test whether float parsing is okay on European
# locales.
diff --git a/Misc/NEWS.d/next/Library/2024-10-08-12-09-09.gh-issue-124969._VBQLq.rst b/Misc/NEWS.d/next/Library/2024-10-08-12-09-09.gh-issue-124969._VBQLq.rst
new file mode 100644
index 00000000000000..b5082b90721d42
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-08-12-09-09.gh-issue-124969._VBQLq.rst
@@ -0,0 +1,3 @@
+Fix ``locale.nl_langinfo(locale.ALT_DIGITS)``. Now it returns a tuple of up
+to 100 strings (an empty tuple on most locales). Previously it returned the
+first item of that tuple or an empty string.
diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c
index f080b97034c0c0..3dea764fdaad27 100644
--- a/Modules/_localemodule.c
+++ b/Modules/_localemodule.c
@@ -618,7 +618,34 @@ _locale_nl_langinfo_impl(PyObject *module, int item)
instead of an empty string for nl_langinfo(ERA). */
const char *result = nl_langinfo(item);
result = result != NULL ? result : "";
- return PyUnicode_DecodeLocale(result, NULL);
+ PyObject *pyresult;
+#ifdef ALT_DIGITS
+ if (item == ALT_DIGITS) {
+ /* The result is a sequence of up to 100 NUL-separated strings. */
+ const char *s = result;
+ int count = 0;
+ for (; count < 100 && *s; count++) {
+ s += strlen(s) + 1;
+ }
+ pyresult = PyTuple_New(count);
+ if (pyresult != NULL) {
+ for (int i = 0; i < count; i++) {
+ PyObject *unicode = PyUnicode_DecodeLocale(result, NULL);
+ if (unicode == NULL) {
+ Py_CLEAR(pyresult);
+ break;
+ }
+ PyTuple_SET_ITEM(pyresult, i, unicode);
+ result += strlen(result) + 1;
+ }
+ }
+ }
+ else
+#endif
+ {
+ pyresult = PyUnicode_DecodeLocale(result, NULL);
+ }
+ return pyresult;
}
PyErr_SetString(PyExc_ValueError, "unsupported langinfo constant");
return NULL;
From ee1257cb9ca913745fd96730b739f538bce60f0a Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Fri, 11 Oct 2024 08:39:54 +0200
Subject: [PATCH 040/269] [3.12] gh-125296: Fix strange fragment identifier for
`name or flags` in argparse docs (GH-125297) (#125300)
gh-125296: Fix strange fragment identifier for `name or flags` in argparse docs (GH-125297)
(cherry picked from commit c1913effeed4e4da4d5310a40ab518945001ffba)
Co-authored-by: Savannah Ostrowski
---
Doc/library/argparse.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index 6d8b207cbc03ab..bcbdb0516bba93 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -596,7 +596,7 @@ The add_argument() method
The following sections describe how each of these are used.
-.. _name_or_flags:
+.. _`name or flags`:
name or flags
^^^^^^^^^^^^^
From 4c40381023df1319661b27f4ab65075569532c73 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Fri, 11 Oct 2024 10:22:34 +0200
Subject: [PATCH 041/269] [3.12] gh-125058: update `_thread` docs regarding
interruptibility of `lock.acquire()` (GH-125141) (#125307)
gh-125058: update `_thread` docs regarding interruptibility of `lock.acquire()` (GH-125141)
(cherry picked from commit 0135848059162ad81478a7776fec622d68a36524)
Co-authored-by: Jan Kaliszewski
---
Doc/library/_thread.rst | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst
index d82f63834dd2d1..e5cbff0b1ef4bc 100644
--- a/Doc/library/_thread.rst
+++ b/Doc/library/_thread.rst
@@ -216,9 +216,11 @@ In addition to these methods, lock objects can also be used via the
* Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is
equivalent to calling :func:`_thread.exit`.
-* It is not possible to interrupt the :meth:`~threading.Lock.acquire` method on
- a lock --- the :exc:`KeyboardInterrupt` exception will happen after the lock
- has been acquired.
+* It is platform-dependent whether the :meth:`~threading.Lock.acquire` method
+ on a lock can be interrupted (so that the :exc:`KeyboardInterrupt` exception
+ will happen immediately, rather than only after the lock has been acquired or
+ the operation has timed out). It can be interrupted on POSIX, but not on
+ Windows.
* When the main thread exits, it is system defined whether the other threads
survive. On most systems, they are killed without executing
From 23cefd9f4c9e49fbdef50b7deaf76499760e4c4b Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Fri, 11 Oct 2024 11:07:03 +0200
Subject: [PATCH 042/269] [3.12] gh-61011: Fix inheritance of nested mutually
exclusive groups in argparse (GH-125210) (GH-125309)
Previously, all nested mutually exclusive groups lost their connection
to the group containing them and were displayed as belonging directly
to the parser.
(cherry picked from commit 18c74497681e0107d7cde53e63ea42feb38f2176)
Co-authored-by: Serhiy Storchaka
Co-authored-by: Danica J. Sutherland
---
Lib/argparse.py | 6 +++-
Lib/test/test_argparse.py | 29 +++++++++++++++++++
Misc/ACKS | 1 +
...4-10-09-21-42-43.gh-issue-61011.pQXZb1.rst | 4 +++
4 files changed, 39 insertions(+), 1 deletion(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-09-21-42-43.gh-issue-61011.pQXZb1.rst
diff --git a/Lib/argparse.py b/Lib/argparse.py
index bd5c757197fcf8..2a3253453dff84 100644
--- a/Lib/argparse.py
+++ b/Lib/argparse.py
@@ -1564,7 +1564,11 @@ def _add_container_actions(self, container):
# NOTE: if add_mutually_exclusive_group ever gains title= and
# description= then this code will need to be expanded as above
for group in container._mutually_exclusive_groups:
- mutex_group = self.add_mutually_exclusive_group(
+ if group._container is container:
+ cont = self
+ else:
+ cont = title_group_map[group._container.title]
+ mutex_group = cont.add_mutually_exclusive_group(
required=group.required)
# map the actions to their new mutex group
diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py
index 13d15fbf075861..84f8b09fb1d2a4 100644
--- a/Lib/test/test_argparse.py
+++ b/Lib/test/test_argparse.py
@@ -2880,6 +2880,35 @@ def test_groups_parents(self):
-x X
'''.format(progname, ' ' if progname else '' )))
+ def test_mutex_groups_parents(self):
+ parent = ErrorRaisingArgumentParser(add_help=False)
+ g = parent.add_argument_group(title='g', description='gd')
+ g.add_argument('-w')
+ g.add_argument('-x')
+ m = g.add_mutually_exclusive_group()
+ m.add_argument('-y')
+ m.add_argument('-z')
+ parser = ErrorRaisingArgumentParser(prog='PROG', parents=[parent])
+
+ self.assertRaises(ArgumentParserError, parser.parse_args,
+ ['-y', 'Y', '-z', 'Z'])
+
+ parser_help = parser.format_help()
+ self.assertEqual(parser_help, textwrap.dedent('''\
+ usage: PROG [-h] [-w W] [-x X] [-y Y | -z Z]
+
+ options:
+ -h, --help show this help message and exit
+
+ g:
+ gd
+
+ -w W
+ -x X
+ -y Y
+ -z Z
+ '''))
+
# ==============================
# Mutually exclusive group tests
# ==============================
diff --git a/Misc/ACKS b/Misc/ACKS
index b5dc32ee507689..837ffbda18aea1 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1792,6 +1792,7 @@ Reuben Sumner
Eryk Sun
Sanjay Sundaresan
Marek Šuppa
+Danica J. Sutherland
Hisao Suzuki
Kalle Svensson
Andrew Svetlov
diff --git a/Misc/NEWS.d/next/Library/2024-10-09-21-42-43.gh-issue-61011.pQXZb1.rst b/Misc/NEWS.d/next/Library/2024-10-09-21-42-43.gh-issue-61011.pQXZb1.rst
new file mode 100644
index 00000000000000..20f9c0b9c78b12
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-09-21-42-43.gh-issue-61011.pQXZb1.rst
@@ -0,0 +1,4 @@
+Fix inheritance of nested mutually exclusive groups from parent parser in
+:class:`argparse.ArgumentParser`. Previously, all nested mutually exclusive
+groups lost their connection to the group containing them and were displayed
+as belonging directly to the parser.
From 7c48c630213ada618c7c7b20aa6af88273d75650 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Fri, 11 Oct 2024 11:52:15 +0200
Subject: [PATCH 043/269] [3.12] Add some doctest cleanups for `configparser`
(GH-125288) (#125291)
Co-authored-by: Alex Waygood
Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
---
Doc/library/configparser.rst | 1 +
1 file changed, 1 insertion(+)
diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst
index 5f04cbc42bf374..4f3549d9a8c0a3 100644
--- a/Doc/library/configparser.rst
+++ b/Doc/library/configparser.rst
@@ -54,6 +54,7 @@ can be customized by end users easily.
import os
os.remove("example.ini")
+ os.remove("override.ini")
Quick Start
From 59036318bfeae049fe13e16b7169caa0e2c00fcd Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka
Date: Fri, 11 Oct 2024 14:22:27 +0300
Subject: [PATCH 044/269] [3.12] gh-125301: Backport some test support helpers
(is_apple_mobile, is_apple) (GH-125311)
(cherry picked from commit 391659b3da570bfa28fed5fbdb6f2d9c26ab3dd0)
---
Lib/test/support/__init__.py | 26 +++++++++++++++++++++-----
1 file changed, 21 insertions(+), 5 deletions(-)
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 9c3dcbc1d2bc29..5432b1ec5c9d84 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -43,7 +43,7 @@
"requires_limited_api", "requires_specialization",
# sys
"MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi",
- "check_impl_detail", "unix_shell", "setswitchinterval",
+ "is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval",
# os
"get_pagesize",
# network
@@ -531,7 +531,7 @@ def requires_legacy_unicode_capi():
is_android = hasattr(sys, 'getandroidapilevel')
-if sys.platform not in ('win32', 'vxworks'):
+if sys.platform not in {"win32", "vxworks", "ios", "tvos", "watchos"}:
unix_shell = '/system/bin/sh' if is_android else '/bin/sh'
else:
unix_shell = None
@@ -541,19 +541,35 @@ def requires_legacy_unicode_capi():
is_emscripten = sys.platform == "emscripten"
is_wasi = sys.platform == "wasi"
-has_fork_support = hasattr(os, "fork") and not is_emscripten and not is_wasi
+# Apple mobile platforms (iOS/tvOS/watchOS) are POSIX-like but do not
+# have subprocess or fork support.
+is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"}
+is_apple = is_apple_mobile or sys.platform == "darwin"
+
+has_fork_support = hasattr(os, "fork") and not (
+ is_emscripten
+ or is_wasi
+ or is_apple_mobile
+)
def requires_fork():
return unittest.skipUnless(has_fork_support, "requires working os.fork()")
-has_subprocess_support = not is_emscripten and not is_wasi
+has_subprocess_support = not (
+ is_emscripten
+ or is_wasi
+ or is_apple_mobile
+)
def requires_subprocess():
"""Used for subprocess, os.spawn calls, fd inheritance"""
return unittest.skipUnless(has_subprocess_support, "requires subprocess support")
# Emscripten's socket emulation and WASI sockets have limitations.
-has_socket_support = not is_emscripten and not is_wasi
+has_socket_support = not (
+ is_emscripten
+ or is_wasi
+)
def requires_working_socket(*, module=False):
"""Skip tests or modules that require working sockets
From 4ab19f912d9220d9762a1f2a0bda2add8af6ae94 Mon Sep 17 00:00:00 2001
From: Jelle Zijlstra
Date: Fri, 11 Oct 2024 15:18:46 -0700
Subject: [PATCH 045/269] [3.12] gh-124917: Allow keyword args to
os.path.exists/lexists on Windows (GH-124918) (#125334)
(cherry picked from commit cc2938a18967c9d462ebb18bc09f73e4364aa7d2)
---
Lib/test/test_genericpath.py | 5 +++
...-10-02-21-11-18.gh-issue-124917.Lnwh5b.rst | 2 +
Modules/clinic/posixmodule.c.h | 40 ++++++++++++++++---
Modules/posixmodule.c | 3 +-
4 files changed, 43 insertions(+), 7 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-02-21-11-18.gh-issue-124917.Lnwh5b.rst
diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py
index bdfc5bfe260799..3eefb722b81e55 100644
--- a/Lib/test/test_genericpath.py
+++ b/Lib/test/test_genericpath.py
@@ -158,6 +158,11 @@ def test_exists(self):
self.assertIs(self.pathmodule.lexists(filename + '\x00'), False)
self.assertIs(self.pathmodule.lexists(bfilename + b'\x00'), False)
+ # Keyword arguments are accepted
+ self.assertIs(self.pathmodule.exists(path=filename), True)
+ if self.pathmodule is not genericpath:
+ self.assertIs(self.pathmodule.lexists(path=filename), True)
+
@unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
@unittest.skipIf(is_emscripten, "Emscripten pipe fds have no stat")
def test_exists_fd(self):
diff --git a/Misc/NEWS.d/next/Library/2024-10-02-21-11-18.gh-issue-124917.Lnwh5b.rst b/Misc/NEWS.d/next/Library/2024-10-02-21-11-18.gh-issue-124917.Lnwh5b.rst
new file mode 100644
index 00000000000000..6218528d2079c3
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-02-21-11-18.gh-issue-124917.Lnwh5b.rst
@@ -0,0 +1,2 @@
+Allow calling :func:`os.path.exists` and :func:`os.path.lexists` with
+keyword arguments on Windows. Fixes a regression in 3.12.4.
diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h
index f46aa65148da7d..a33461dc5600dd 100644
--- a/Modules/clinic/posixmodule.c.h
+++ b/Modules/clinic/posixmodule.c.h
@@ -1975,25 +1975,55 @@ os__path_splitroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py
#if defined(MS_WINDOWS)
PyDoc_STRVAR(os__path_exists__doc__,
-"_path_exists($module, path, /)\n"
+"_path_exists($module, /, path)\n"
"--\n"
"\n"
"Test whether a path exists. Returns False for broken symbolic links.");
#define OS__PATH_EXISTS_METHODDEF \
- {"_path_exists", (PyCFunction)os__path_exists, METH_O, os__path_exists__doc__},
+ {"_path_exists", _PyCFunction_CAST(os__path_exists), METH_FASTCALL|METH_KEYWORDS, os__path_exists__doc__},
static int
os__path_exists_impl(PyObject *module, path_t *path);
static PyObject *
-os__path_exists(PyObject *module, PyObject *arg)
+os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
{
PyObject *return_value = NULL;
+ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
+
+ #define NUM_KEYWORDS 1
+ static struct {
+ PyGC_Head _this_is_not_used;
+ PyObject_VAR_HEAD
+ PyObject *ob_item[NUM_KEYWORDS];
+ } _kwtuple = {
+ .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
+ .ob_item = { &_Py_ID(path), },
+ };
+ #undef NUM_KEYWORDS
+ #define KWTUPLE (&_kwtuple.ob_base.ob_base)
+
+ #else // !Py_BUILD_CORE
+ # define KWTUPLE NULL
+ #endif // !Py_BUILD_CORE
+
+ static const char * const _keywords[] = {"path", NULL};
+ static _PyArg_Parser _parser = {
+ .keywords = _keywords,
+ .fname = "_path_exists",
+ .kwtuple = KWTUPLE,
+ };
+ #undef KWTUPLE
+ PyObject *argsbuf[1];
path_t path = PATH_T_INITIALIZE_P("_path_exists", "path", 0, 0, 1, 1);
int _return_value;
- if (!path_converter(arg, &path)) {
+ args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf);
+ if (!args) {
+ goto exit;
+ }
+ if (!path_converter(args[0], &path)) {
goto exit;
}
_return_value = os__path_exists_impl(module, &path);
@@ -12002,4 +12032,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na
#ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF
#define OS_WAITSTATUS_TO_EXITCODE_METHODDEF
#endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */
-/*[clinic end generated code: output=67c2e3d4537287c1 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=6d34c4564aca7725 input=a9049054013a1b77]*/
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 2277caee58f505..b8558cc2265a58 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -5209,7 +5209,6 @@ _testFileType(path_t *path, int testedType)
os._path_exists -> bool
path: path_t(allow_fd=True, suppress_value_error=True)
- /
Test whether a path exists. Returns False for broken symbolic links.
@@ -5217,7 +5216,7 @@ Test whether a path exists. Returns False for broken symbolic links.
static int
os__path_exists_impl(PyObject *module, path_t *path)
-/*[clinic end generated code: output=8da13acf666e16ba input=29198507a6082a57]*/
+/*[clinic end generated code: output=8da13acf666e16ba input=142beabfc66783eb]*/
{
return _testFileExists(path, TRUE);
}
From 3f38ea11c029b68472d6c23aec3cf711e3e62d3c Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sat, 12 Oct 2024 01:23:54 +0200
Subject: [PATCH 046/269] [3.12] gh-116938: Clarify documentation of `dict` and
`dict.update` regarding the positional argument they accept (GH-125213)
(#125337)
Co-authored-by: Victorien <65306057+Viicos@users.noreply.github.com>
Co-authored-by: Alex Waygood
---
Doc/library/stdtypes.rst | 25 +++++++++++++------------
Lib/_collections_abc.py | 2 +-
2 files changed, 14 insertions(+), 13 deletions(-)
diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index e72711dc9cb5b7..0c1f29d8b69d97 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -4460,14 +4460,14 @@ can be used interchangeably to index the same dictionary entry.
``dict([('foo', 100), ('bar', 200)])``, ``dict(foo=100, bar=200)``
If no positional argument is given, an empty dictionary is created.
- If a positional argument is given and it is a mapping object, a dictionary
- is created with the same key-value pairs as the mapping object. Otherwise,
- the positional argument must be an :term:`iterable` object. Each item in
- the iterable must itself be an iterable with exactly two objects. The
- first object of each item becomes a key in the new dictionary, and the
- second object the corresponding value. If a key occurs more than once, the
- last value for that key becomes the corresponding value in the new
- dictionary.
+ If a positional argument is given and it defines a ``keys()`` method, a
+ dictionary is created by calling :meth:`~object.__getitem__` on the argument with
+ each returned key from the method. Otherwise, the positional argument must be an
+ :term:`iterable` object. Each item in the iterable must itself be an iterable
+ with exactly two elements. The first element of each item becomes a key in the
+ new dictionary, and the second element the corresponding value. If a key occurs
+ more than once, the last value for that key becomes the corresponding value in
+ the new dictionary.
If keyword arguments are given, the keyword arguments and their values are
added to the dictionary created from the positional argument. If a key
@@ -4624,10 +4624,11 @@ can be used interchangeably to index the same dictionary entry.
Update the dictionary with the key/value pairs from *other*, overwriting
existing keys. Return ``None``.
- :meth:`update` accepts either another dictionary object or an iterable of
- key/value pairs (as tuples or other iterables of length two). If keyword
- arguments are specified, the dictionary is then updated with those
- key/value pairs: ``d.update(red=1, blue=2)``.
+ :meth:`update` accepts either another object with a ``keys()`` method (in
+ which case :meth:`~object.__getitem__` is called with every key returned from
+ the method). or an iterable of key/value pairs (as tuples or other iterables
+ of length two). If keyword arguments are specified, the dictionary is then
+ updated with those key/value pairs: ``d.update(red=1, blue=2)``.
.. method:: values()
diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py
index 601107d2d86771..09745658de1925 100644
--- a/Lib/_collections_abc.py
+++ b/Lib/_collections_abc.py
@@ -973,7 +973,7 @@ def clear(self):
def update(self, other=(), /, **kwds):
''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F.
- If E present and has a .keys() method, does: for k in E: D[k] = E[k]
+ If E present and has a .keys() method, does: for k in E.keys(): D[k] = E[k]
If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v
In either case, this is followed by: for k, v in F.items(): D[k] = v
'''
From 2264c097e09810a563aab379cc17e6b5d9d2bf74 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sat, 12 Oct 2024 02:47:23 +0200
Subject: [PATCH 047/269] [3.12] Doc: Fix a typo in "Function Examples" in the
control-flow tutorial (GH-125338) (#125342)
Doc: Fix a typo in "Function Examples" in the control-flow tutorial (GH-125338)
(cherry picked from commit 5a074aab845f82f4a150c27b905dae05c337d381)
Co-authored-by: Rafael Fontenelle
---
Doc/tutorial/controlflow.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst
index fd765e58ff2485..9b73ac475c78d5 100644
--- a/Doc/tutorial/controlflow.rst
+++ b/Doc/tutorial/controlflow.rst
@@ -832,7 +832,7 @@ parameters as there is a ``/`` in the function definition::
File "", line 1, in
TypeError: pos_only_arg() got some positional-only arguments passed as keyword arguments: 'arg'
-The third function ``kwd_only_args`` only allows keyword arguments as indicated
+The third function ``kwd_only_arg`` only allows keyword arguments as indicated
by a ``*`` in the function definition::
>>> kwd_only_arg(3)
From affffc7ddac68d01ab8d63872770a44e96c46c44 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sat, 12 Oct 2024 05:12:11 +0200
Subject: [PATCH 048/269] [3.12] gh-124309: fix staggered race on eager tasks
(GH-124847) (#125340)
gh-124309: fix staggered race on eager tasks (GH-124847)
This patch is entirely by Thomas and Peter
(cherry picked from commit 979c0df7c0adfb744159a5fc184043dc733d8534)
Co-authored-by: Thomas Grainger
Co-authored-by: Peter Bierma
---
Lib/asyncio/staggered.py | 17 +++++--
.../test_asyncio/test_eager_task_factory.py | 46 +++++++++++++++++++
Lib/test/test_asyncio/test_staggered.py | 27 +++++++++++
...-10-01-13-46-58.gh-issue-124390.dK1Zcm.rst | 1 +
4 files changed, 88 insertions(+), 3 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-01-13-46-58.gh-issue-124390.dK1Zcm.rst
diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py
index c3a7441a7b091d..7aafcea4d885eb 100644
--- a/Lib/asyncio/staggered.py
+++ b/Lib/asyncio/staggered.py
@@ -69,7 +69,11 @@ async def staggered_race(coro_fns, delay, *, loop=None):
exceptions = []
running_tasks = []
- async def run_one_coro(previous_failed) -> None:
+ async def run_one_coro(ok_to_start, previous_failed) -> None:
+ # in eager tasks this waits for the calling task to append this task
+ # to running_tasks, in regular tasks this wait is a no-op that does
+ # not yield a future. See gh-124309.
+ await ok_to_start.wait()
# Wait for the previous task to finish, or for delay seconds
if previous_failed is not None:
with contextlib.suppress(exceptions_mod.TimeoutError):
@@ -85,8 +89,12 @@ async def run_one_coro(previous_failed) -> None:
return
# Start task that will run the next coroutine
this_failed = locks.Event()
- next_task = loop.create_task(run_one_coro(this_failed))
+ next_ok_to_start = locks.Event()
+ next_task = loop.create_task(run_one_coro(next_ok_to_start, this_failed))
running_tasks.append(next_task)
+ # next_task has been appended to running_tasks so next_task is ok to
+ # start.
+ next_ok_to_start.set()
assert len(running_tasks) == this_index + 2
# Prepare place to put this coroutine's exceptions if not won
exceptions.append(None)
@@ -116,8 +124,11 @@ async def run_one_coro(previous_failed) -> None:
if i != this_index:
t.cancel()
- first_task = loop.create_task(run_one_coro(None))
+ ok_to_start = locks.Event()
+ first_task = loop.create_task(run_one_coro(ok_to_start, None))
running_tasks.append(first_task)
+ # first_task has been appended to running_tasks so first_task is ok to start.
+ ok_to_start.set()
try:
# Wait for a growing list of tasks to all finish: poor man's version of
# curio's TaskGroup or trio's nursery
diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py
index 58c06287bc3c5d..b06832e02f00d6 100644
--- a/Lib/test/test_asyncio/test_eager_task_factory.py
+++ b/Lib/test/test_asyncio/test_eager_task_factory.py
@@ -218,6 +218,52 @@ async def run():
self.run_coro(run())
+ def test_staggered_race_with_eager_tasks(self):
+ # See https://github.com/python/cpython/issues/124309
+
+ async def fail():
+ await asyncio.sleep(0)
+ raise ValueError("no good")
+
+ async def run():
+ winner, index, excs = await asyncio.staggered.staggered_race(
+ [
+ lambda: asyncio.sleep(2, result="sleep2"),
+ lambda: asyncio.sleep(1, result="sleep1"),
+ lambda: fail()
+ ],
+ delay=0.25
+ )
+ self.assertEqual(winner, 'sleep1')
+ self.assertEqual(index, 1)
+ self.assertIsNone(excs[index])
+ self.assertIsInstance(excs[0], asyncio.CancelledError)
+ self.assertIsInstance(excs[2], ValueError)
+
+ self.run_coro(run())
+
+ def test_staggered_race_with_eager_tasks_no_delay(self):
+ # See https://github.com/python/cpython/issues/124309
+ async def fail():
+ raise ValueError("no good")
+
+ async def run():
+ winner, index, excs = await asyncio.staggered.staggered_race(
+ [
+ lambda: fail(),
+ lambda: asyncio.sleep(1, result="sleep1"),
+ lambda: asyncio.sleep(0, result="sleep0"),
+ ],
+ delay=None
+ )
+ self.assertEqual(winner, 'sleep1')
+ self.assertEqual(index, 1)
+ self.assertIsNone(excs[index])
+ self.assertIsInstance(excs[0], ValueError)
+ self.assertEqual(len(excs), 2)
+
+ self.run_coro(run())
+
class PyEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase):
Task = tasks._PyTask
diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py
index e6e32f7dbbbcba..74941f704c4890 100644
--- a/Lib/test/test_asyncio/test_staggered.py
+++ b/Lib/test/test_asyncio/test_staggered.py
@@ -95,3 +95,30 @@ async def coro(index):
self.assertEqual(len(excs), 2)
self.assertIsInstance(excs[0], ValueError)
self.assertIsInstance(excs[1], ValueError)
+
+
+ async def test_multiple_winners(self):
+ event = asyncio.Event()
+
+ async def coro(index):
+ await event.wait()
+ return index
+
+ async def do_set():
+ event.set()
+ await asyncio.Event().wait()
+
+ winner, index, excs = await staggered_race(
+ [
+ lambda: coro(0),
+ lambda: coro(1),
+ do_set,
+ ],
+ delay=0.1,
+ )
+ self.assertIs(winner, 0)
+ self.assertIs(index, 0)
+ self.assertEqual(len(excs), 3)
+ self.assertIsNone(excs[0], None)
+ self.assertIsInstance(excs[1], asyncio.CancelledError)
+ self.assertIsInstance(excs[2], asyncio.CancelledError)
diff --git a/Misc/NEWS.d/next/Library/2024-10-01-13-46-58.gh-issue-124390.dK1Zcm.rst b/Misc/NEWS.d/next/Library/2024-10-01-13-46-58.gh-issue-124390.dK1Zcm.rst
new file mode 100644
index 00000000000000..89610fa44bf743
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-01-13-46-58.gh-issue-124390.dK1Zcm.rst
@@ -0,0 +1 @@
+Fixed :exc:`AssertionError` when using :func:`!asyncio.staggered.staggered_race` with :attr:`asyncio.eager_task_factory`.
From aa0cdeb93cedef1e28bc3e4dc6cd4c5c3b1425d6 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sat, 12 Oct 2024 15:00:24 +0200
Subject: [PATCH 049/269] [3.12] gh-125254: Fix error report about ambiguous
option in argparse (GH-125273) (GH-125360)
This was a regression introduced in gh-58573. It was only tested for the
case when the ambiguous option is the last argument in the command line.
(cherry picked from commit 63cf4e914f879ee28a75c02e867baa7c6047ea2b)
Co-authored-by: Serhiy Storchaka
---
Lib/argparse.py | 2 +-
Lib/test/test_argparse.py | 14 ++++++++++++--
.../2024-10-10-19-57-35.gh-issue-125254.RtZxXS.rst | 1 +
3 files changed, 14 insertions(+), 3 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-10-19-57-35.gh-issue-125254.RtZxXS.rst
diff --git a/Lib/argparse.py b/Lib/argparse.py
index 2a3253453dff84..0e13ea5860da97 100644
--- a/Lib/argparse.py
+++ b/Lib/argparse.py
@@ -2025,7 +2025,7 @@ def consume_optional(start_index):
if len(option_tuples) > 1:
options = ', '.join([option_string
for action, option_string, sep, explicit_arg in option_tuples])
- args = {'option': arg_string, 'matches': options}
+ args = {'option': arg_strings[start_index], 'matches': options}
msg = _('ambiguous option: %(option)s could match %(matches)s')
raise ArgumentError(None, msg % args)
diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py
index 84f8b09fb1d2a4..956c1cd505a96e 100644
--- a/Lib/test/test_argparse.py
+++ b/Lib/test/test_argparse.py
@@ -6310,9 +6310,19 @@ def test_conflicting_mutually_exclusive_args_zero_or_more_with_metavar2(self):
def test_ambiguous_option(self):
self.parser.add_argument('--foobaz')
self.parser.add_argument('--fooble', action='store_true')
+ self.parser.add_argument('--foogle')
self.assertRaisesRegex(argparse.ArgumentError,
- "ambiguous option: --foob could match --foobaz, --fooble",
- self.parser.parse_args, ['--foob'])
+ "ambiguous option: --foob could match --foobaz, --fooble",
+ self.parser.parse_args, ['--foob'])
+ self.assertRaisesRegex(argparse.ArgumentError,
+ "ambiguous option: --foob=1 could match --foobaz, --fooble$",
+ self.parser.parse_args, ['--foob=1'])
+ self.assertRaisesRegex(argparse.ArgumentError,
+ "ambiguous option: --foob could match --foobaz, --fooble$",
+ self.parser.parse_args, ['--foob', '1', '--foogle', '2'])
+ self.assertRaisesRegex(argparse.ArgumentError,
+ "ambiguous option: --foob=1 could match --foobaz, --fooble$",
+ self.parser.parse_args, ['--foob=1', '--foogle', '2'])
def test_os_error(self):
self.parser.add_argument('file')
diff --git a/Misc/NEWS.d/next/Library/2024-10-10-19-57-35.gh-issue-125254.RtZxXS.rst b/Misc/NEWS.d/next/Library/2024-10-10-19-57-35.gh-issue-125254.RtZxXS.rst
new file mode 100644
index 00000000000000..abe37fefedc3be
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-10-19-57-35.gh-issue-125254.RtZxXS.rst
@@ -0,0 +1 @@
+Fix a bug where ArgumentError includes the incorrect ambiguous option in :mod:`argparse`.
From e01a1784dbfe366cd2d5e22bba8b964107651bf0 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sat, 12 Oct 2024 15:02:14 +0200
Subject: [PATCH 050/269] [3.12] gh-85935: Explicitly document the case nargs=0
in argparse (GH-125302) (GH-125358)
(cherry picked from commit 07c2d15977738165e9dc4248e7edda7c75ecc14b)
Co-authored-by: Serhiy Storchaka
---
Doc/library/argparse.rst | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index bcbdb0516bba93..a1f08ea272934f 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -726,6 +726,9 @@ how the command-line arguments should be handled. The supplied actions are:
.. versionadded:: 3.8
+Only actions that consume command-line arguments (e.g. ``'store'``,
+``'append'`` or ``'extend'``) can be used with positional arguments.
+
You may also specify an arbitrary action by passing an Action subclass or
other object that implements the same interface. The ``BooleanOptionalAction``
is available in ``argparse`` and adds support for boolean actions such as
@@ -853,6 +856,8 @@ See also :ref:`specifying-ambiguous-arguments`. The supported values are:
If the ``nargs`` keyword argument is not provided, the number of arguments consumed
is determined by the action_. Generally this means a single command-line argument
will be consumed and a single item (not a list) will be produced.
+Actions that do not consume command-line arguments (e.g.
+``'store_const'``) set ``nargs=0``.
.. _const:
From 331fc017ce6afdf778464f6e9540670bb6a0fa4f Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sat, 12 Oct 2024 20:02:52 +0200
Subject: [PATCH 051/269] [3.12] gh-53203: Fix strptime() for %c and %x formats
on many locales (GH-124946) (GH-125370)
In some locales (like French or Hebrew) the full or abbreviated names of
the default month and weekday used in __calc_date_time can be part of
other name or constant part of the %c format. The month name can also
match %m with constant suffix (like in Japanese). So the code failed to
correctly distinguish formats %a, %A, %b, %B and %m.
Cycle all month and all days of the week to find the variable part
and distinguish %a from %A and %b from %B or %m.
Fixed locales for the following languges:
Arabic, Bislama, Breton, Bodo, Kashubian, Chuvash, Estonian, French, Irish,
Ge'ez, Gurajati, Manx Gaelic, Hebrew, Hindi, Chhattisgarhi, Haitian Kreyol,
Japanese, Kannada, Korean, Marathi, Malay, Norwegian, Nynorsk, Punjabi,
Rajasthani, Tok Pisin, Yoruba, Yue Chinese, Yau/Nungon and Chinese.
(cherry picked from commit c05f9dde8a12dfd63d3ade93da616042df2dc925)
Co-authored-by: Serhiy Storchaka
Co-authored-by: Eli Bendersky
---
Lib/_strptime.py | 127 +++++++++++++++---
Lib/test/test_strptime.py | 36 +++--
...4-10-03-20-45-57.gh-issue-53203.3Sk4Ia.rst | 5 +
3 files changed, 134 insertions(+), 34 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-03-20-45-57.gh-issue-53203.3Sk4Ia.rst
diff --git a/Lib/_strptime.py b/Lib/_strptime.py
index 798cf9f9d3fffe..d740c15519a75d 100644
--- a/Lib/_strptime.py
+++ b/Lib/_strptime.py
@@ -27,6 +27,18 @@ def _getlang():
# Figure out what the current language is set to.
return locale.getlocale(locale.LC_TIME)
+def _findall(haystack, needle):
+ # Find all positions of needle in haystack.
+ if not needle:
+ return
+ i = 0
+ while True:
+ i = haystack.find(needle, i)
+ if i < 0:
+ break
+ yield i
+ i += len(needle)
+
class LocaleTime(object):
"""Stores and handles locale-specific information related to time.
@@ -101,7 +113,8 @@ def __calc_am_pm(self):
am_pm = []
for hour in (1, 22):
time_tuple = time.struct_time((1999,3,17,hour,44,55,2,76,0))
- am_pm.append(time.strftime("%p", time_tuple).lower())
+ # br_FR has AM/PM info (' ',' ').
+ am_pm.append(time.strftime("%p", time_tuple).lower().strip())
self.am_pm = am_pm
def __calc_date_time(self):
@@ -113,42 +126,114 @@ def __calc_date_time(self):
# values within the format string is very important; it eliminates
# possible ambiguity for what something represents.
time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0))
- date_time = [None, None, None]
- date_time[0] = time.strftime("%c", time_tuple).lower()
- date_time[1] = time.strftime("%x", time_tuple).lower()
- date_time[2] = time.strftime("%X", time_tuple).lower()
- replacement_pairs = [('%', '%%'), (self.f_weekday[2], '%A'),
- (self.f_month[3], '%B'), (self.a_weekday[2], '%a'),
- (self.a_month[3], '%b'), (self.am_pm[1], '%p'),
+ time_tuple2 = time.struct_time((1999,1,3,1,1,1,6,3,0))
+ replacement_pairs = [
('1999', '%Y'), ('99', '%y'), ('22', '%H'),
('44', '%M'), ('55', '%S'), ('76', '%j'),
('17', '%d'), ('03', '%m'), ('3', '%m'),
# '3' needed for when no leading zero.
('2', '%w'), ('10', '%I')]
- replacement_pairs.extend([(tz, "%Z") for tz_values in self.timezone
- for tz in tz_values])
- for offset,directive in ((0,'%c'), (1,'%x'), (2,'%X')):
- current_format = date_time[offset]
- for old, new in replacement_pairs:
+ date_time = []
+ for directive in ('%c', '%x', '%X'):
+ current_format = time.strftime(directive, time_tuple).lower()
+ current_format = current_format.replace('%', '%%')
+ # The month and the day of the week formats are treated specially
+ # because of a possible ambiguity in some locales where the full
+ # and abbreviated names are equal or names of different types
+ # are equal. See doc of __find_month_format for more details.
+ lst, fmt = self.__find_weekday_format(directive)
+ if lst:
+ current_format = current_format.replace(lst[2], fmt, 1)
+ lst, fmt = self.__find_month_format(directive)
+ if lst:
+ current_format = current_format.replace(lst[3], fmt, 1)
+ if self.am_pm[1]:
# Must deal with possible lack of locale info
# manifesting itself as the empty string (e.g., Swedish's
# lack of AM/PM info) or a platform returning a tuple of empty
# strings (e.g., MacOS 9 having timezone as ('','')).
- if old:
- current_format = current_format.replace(old, new)
+ current_format = current_format.replace(self.am_pm[1], '%p')
+ for tz_values in self.timezone:
+ for tz in tz_values:
+ if tz:
+ current_format = current_format.replace(tz, "%Z")
+ for old, new in replacement_pairs:
+ current_format = current_format.replace(old, new)
# If %W is used, then Sunday, 2005-01-03 will fall on week 0 since
# 2005-01-03 occurs before the first Monday of the year. Otherwise
# %U is used.
- time_tuple = time.struct_time((1999,1,3,1,1,1,6,3,0))
- if '00' in time.strftime(directive, time_tuple):
+ if '00' in time.strftime(directive, time_tuple2):
U_W = '%W'
else:
U_W = '%U'
- date_time[offset] = current_format.replace('11', U_W)
+ current_format = current_format.replace('11', U_W)
+ date_time.append(current_format)
self.LC_date_time = date_time[0]
self.LC_date = date_time[1]
self.LC_time = date_time[2]
+ def __find_month_format(self, directive):
+ """Find the month format appropriate for the current locale.
+
+ In some locales (for example French and Hebrew), the default month
+ used in __calc_date_time has the same name in full and abbreviated
+ form. Also, the month name can by accident match other part of the
+ representation: the day of the week name (for example in Morisyen)
+ or the month number (for example in Japanese). Thus, cycle months
+ of the year and find all positions that match the month name for
+ each month, If no common positions are found, the representation
+ does not use the month name.
+ """
+ full_indices = abbr_indices = None
+ for m in range(1, 13):
+ time_tuple = time.struct_time((1999, m, 17, 22, 44, 55, 2, 76, 0))
+ datetime = time.strftime(directive, time_tuple).lower()
+ indices = set(_findall(datetime, self.f_month[m]))
+ if full_indices is None:
+ full_indices = indices
+ else:
+ full_indices &= indices
+ indices = set(_findall(datetime, self.a_month[m]))
+ if abbr_indices is None:
+ abbr_indices = indices
+ else:
+ abbr_indices &= indices
+ if not full_indices and not abbr_indices:
+ return None, None
+ if full_indices:
+ return self.f_month, '%B'
+ if abbr_indices:
+ return self.a_month, '%b'
+ return None, None
+
+ def __find_weekday_format(self, directive):
+ """Find the day of the week format appropriate for the current locale.
+
+ Similar to __find_month_format().
+ """
+ full_indices = abbr_indices = None
+ for wd in range(7):
+ time_tuple = time.struct_time((1999, 3, 17, 22, 44, 55, wd, 76, 0))
+ datetime = time.strftime(directive, time_tuple).lower()
+ indices = set(_findall(datetime, self.f_weekday[wd]))
+ if full_indices is None:
+ full_indices = indices
+ else:
+ full_indices &= indices
+ if self.f_weekday[wd] != self.a_weekday[wd]:
+ indices = set(_findall(datetime, self.a_weekday[wd]))
+ if abbr_indices is None:
+ abbr_indices = indices
+ else:
+ abbr_indices &= indices
+ if not full_indices and not abbr_indices:
+ return None, None
+ if full_indices:
+ return self.f_weekday, '%A'
+ if abbr_indices:
+ return self.a_weekday, '%a'
+ return None, None
+
def __calc_timezone(self):
# Set self.timezone by using time.tzname.
# Do not worry about possibility of time.tzname[0] == time.tzname[1]
@@ -186,7 +271,7 @@ def __init__(self, locale_time=None):
'd': r"(?P3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])",
'f': r"(?P[0-9]{1,6})",
'H': r"(?P2[0-3]|[0-1]\d|\d)",
- 'I': r"(?P1[0-2]|0[1-9]|[1-9])",
+ 'I': r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])",
'G': r"(?P\d\d\d\d)",
'j': r"(?P36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])",
'm': r"(?P1[0-2]|0[1-9]|[1-9])",
@@ -330,8 +415,8 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
_regex_cache[format] = format_regex
found = format_regex.match(data_string)
if not found:
- raise ValueError("time data %r does not match format %r" %
- (data_string, format))
+ raise ValueError("time data %r does not match format %r :: /%s/" %
+ (data_string, format, format_regex.pattern))
if len(data_string) != found.end():
raise ValueError("unconverted data remains: %s" %
data_string[found.end():])
diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py
index 66a9815e69b9c2..fa7915bead83fd 100644
--- a/Lib/test/test_strptime.py
+++ b/Lib/test/test_strptime.py
@@ -5,6 +5,7 @@
import locale
import re
import os
+import platform
import sys
from test import support
from test.support import skip_if_buggy_ucrt_strfptime, run_with_locales
@@ -12,6 +13,13 @@
import _strptime
+libc_ver = platform.libc_ver()
+if libc_ver[0] == 'glibc':
+ glibc_ver = tuple(map(int, libc_ver[1].split('.')))
+else:
+ glibc_ver = None
+
+
class getlang_Tests(unittest.TestCase):
"""Test _getlang"""
def test_basic(self):
@@ -476,16 +484,16 @@ def test_bad_timezone(self):
# * Year is not included: ha_NG.
# * Use non-Gregorian calendar: lo_LA, thai, th_TH.
#
- # BUG: Generates invalid regexp for br_FR, csb_PL, Arabic.
- # BUG: Generates regexp that does not match the current date and time
- # for fa_IR, gez_ER, gez_ET, lzh_TW, my_MM, or_IN, shn_MM, yo_NG.
# BUG: Generates regexp that does not match the current date and time
- # for fa_IR, gez_ER, gez_ET, lzh_TW, my_MM, or_IN, shn_MM, yo_NG,
- # fr_FR, ja_JP, he_IL, ko_KR, zh_CN, etc.
- @run_with_locales('LC_TIME', 'C', 'en_US', 'de_DE',
- 'eu_ES', 'mfe_MU')
+ # for az_IR, fa_IR, lzh_TW, my_MM, or_IN, shn_MM.
+ @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP',
+ 'he_IL', 'eu_ES', 'ar_AE', 'mfe_MU', 'yo_NG',
+ 'csb_PL', 'br_FR', 'gez_ET', 'brx_IN')
def test_date_time_locale(self):
# Test %c directive
+ loc = locale.getlocale(locale.LC_TIME)[0]
+ if glibc_ver and glibc_ver < (2, 31) and loc == 'br_FR':
+ self.skipTest('%c in locale br_FR does not include time')
now = time.time()
self.roundtrip('%c', slice(0, 6), time.localtime(now))
# 1 hour 20 minutes 30 seconds ago
@@ -503,7 +511,9 @@ def test_date_time_locale(self):
# NB: Dates before 1969 do not roundtrip on some locales:
# bo_CN, bo_IN, dz_BT, eu_ES, eu_FR.
- @run_with_locales('LC_TIME', 'C', 'en_US', 'de_DE', 'ja_JP')
+ @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP',
+ 'he_IL', 'ar_AE', 'mfe_MU', 'yo_NG',
+ 'csb_PL', 'br_FR', 'gez_ET', 'brx_IN')
def test_date_time_locale2(self):
# Test %c directive
self.roundtrip('%c', slice(0, 6), (1900, 1, 1, 0, 0, 0, 0, 1, 0))
@@ -511,10 +521,9 @@ def test_date_time_locale2(self):
# NB: Does not roundtrip because use non-Gregorian calendar:
# lo_LA, thai, th_TH.
# BUG: Generates regexp that does not match the current date
- # for az_IR, fa_IR, lzh_TW, my_MM, or_IN, shn_MM,
- # Arabic, ja_JP, ko_KR, zh_CN, etc.
- @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE',
- 'he_IL', 'eu_ES')
+ # for az_IR, fa_IR, lzh_TW, my_MM, or_IN, shn_MM.
+ @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP',
+ 'he_IL', 'eu_ES', 'ar_AE')
def test_date_locale(self):
# Test %x directive
now = time.time()
@@ -533,7 +542,8 @@ def test_date_locale(self):
support.is_emscripten or support.is_wasi,
"musl libc issue on Emscripten, bpo-46390"
)
- @run_with_locales('LC_TIME', 'en_US', 'fr_FR', 'de_DE', 'ja_JP')
+ @run_with_locales('LC_TIME', 'en_US', 'fr_FR', 'de_DE', 'ja_JP',
+ 'eu_ES', 'ar_AE')
def test_date_locale2(self):
# Test %x directive
self.roundtrip('%x', slice(0, 3), (1900, 1, 1, 0, 0, 0, 0, 1, 0))
diff --git a/Misc/NEWS.d/next/Library/2024-10-03-20-45-57.gh-issue-53203.3Sk4Ia.rst b/Misc/NEWS.d/next/Library/2024-10-03-20-45-57.gh-issue-53203.3Sk4Ia.rst
new file mode 100644
index 00000000000000..6895cffcf545fd
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-03-20-45-57.gh-issue-53203.3Sk4Ia.rst
@@ -0,0 +1,5 @@
+Fix :func:`time.strptime` for ``%c`` and ``%x`` formats in many locales:
+Arabic, Bislama, Breton, Bodo, Kashubian, Chuvash, Estonian, French, Irish,
+Ge'ez, Gurajati, Manx Gaelic, Hebrew, Hindi, Chhattisgarhi, Haitian Kreyol,
+Japanese, Kannada, Korean, Marathi, Malay, Norwegian, Nynorsk, Punjabi,
+Rajasthani, Tok Pisin, Yoruba, Yue Chinese, Yau/Nungon and Chinese.
From 1d45fae26dacd9c07d4a57831e224073ef56cbe7 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sat, 12 Oct 2024 22:44:41 +0200
Subject: [PATCH 052/269] [3.12] gh-125289: Update sample code in
asyncio-task.rst (GH-125292) (GH-125375)
gh-125289: Update sample code in asyncio-task.rst (GH-125292)
* Update sample code in asyncio-task.rst
This will change **coroutines** sample code in the **Awaitables** section and make the example clearer.
* Update Doc/library/asyncio-task.rst
Revert the added print
* Update Doc/library/asyncio-task.rst
---------
(cherry picked from commit fa52b82c91a8e1a0971bd5fef656473ec93f41e3)
Co-authored-by: Ghorban M. Tavakoly <58617996+galmyk@users.noreply.github.com>
Co-authored-by: Carol Willing
---
Doc/library/asyncio-task.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index e9b45cd2967310..0ba6e84dab5a41 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -158,7 +158,7 @@ other coroutines::
# Nothing happens if we just call "nested()".
# A coroutine object is created but not awaited,
# so it *won't run at all*.
- nested()
+ nested() # will raise a "RuntimeWarning".
# Let's do it differently now and await it:
print(await nested()) # will print "42".
From 77020dae341764ee440c99cf8ba13c7766730b46 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sun, 13 Oct 2024 09:55:37 +0200
Subject: [PATCH 053/269] [3.12] Trivial change: Update comments in activate
about what running hash -r does (GH-125385) (#125388)
(cherry picked from commit 82bcaf15890cf85b76b4f62d2dd1710bb49c3ed1)
---
Lib/venv/scripts/common/activate | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate
index d5914e0cbb40d5..44df44a7404b43 100644
--- a/Lib/venv/scripts/common/activate
+++ b/Lib/venv/scripts/common/activate
@@ -14,8 +14,9 @@ deactivate () {
unset _OLD_VIRTUAL_PYTHONHOME
fi
- # Call hash to forget past commands. Without forgetting
- # past commands the $PATH changes we made may not be respected
+ # Call hash to forget past locations. Without forgetting
+ # past locations the $PATH changes we made may not be respected.
+ # See "man bash" for more details. hash is usually a builtin of your shell
hash -r 2> /dev/null
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
From afddaeb7698d2c572364d61e02c4271439650f6d Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sun, 13 Oct 2024 09:56:04 +0200
Subject: [PATCH 054/269] [3.12] gh-86673: Loosen test_ttk.test_identify()
requirements (GH-125335) (#125391)
In aeca373b3 (PR gh-12011, issue gh-71500), test_identify() was changed to expect different results on Darwin. Ned's fix was later adjusted by e52f9bee8. This workaround is only needed for some variants of Tk/Tcl on macOS, so we now allow both the workaround and the generic results for these tests.
(cherry picked from commit 4197a796ecf3a751ad7245b8d4f980d6d444b614)
Co-authored-by: Erlend E. Aasland
---
Lib/test/test_ttk/test_widgets.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/Lib/test/test_ttk/test_widgets.py b/Lib/test/test_ttk/test_widgets.py
index 3e541fbc5a4ded..ceba95c92a1fc1 100644
--- a/Lib/test/test_ttk/test_widgets.py
+++ b/Lib/test/test_ttk/test_widgets.py
@@ -336,7 +336,8 @@ class EntryTest(AbstractWidgetTest, unittest.TestCase):
'show', 'state', 'style', 'takefocus', 'textvariable',
'validate', 'validatecommand', 'width', 'xscrollcommand',
)
- IDENTIFY_AS = 'Entry.field' if sys.platform == 'darwin' else 'textarea'
+ # bpo-27313: macOS Tk/Tcl may or may not report 'Entry.field'.
+ IDENTIFY_AS = {'Entry.field', 'textarea'}
def setUp(self):
super().setUp()
@@ -373,8 +374,7 @@ def test_identify(self):
self.entry.pack()
self.entry.update()
- # bpo-27313: macOS Cocoa widget differs from X, allow either
- self.assertEqual(self.entry.identify(5, 5), self.IDENTIFY_AS)
+ self.assertIn(self.entry.identify(5, 5), self.IDENTIFY_AS)
self.assertEqual(self.entry.identify(-1, -1), "")
self.assertRaises(tkinter.TclError, self.entry.identify, None, 5)
@@ -461,7 +461,7 @@ class ComboboxTest(EntryTest, unittest.TestCase):
'validate', 'validatecommand', 'values',
'width', 'xscrollcommand',
)
- IDENTIFY_AS = 'Combobox.button' if sys.platform == 'darwin' else 'textarea'
+ IDENTIFY_AS = {'Combobox.button', 'textarea'}
def setUp(self):
super().setUp()
@@ -1204,7 +1204,7 @@ class SpinboxTest(EntryTest, unittest.TestCase):
'takefocus', 'textvariable', 'to', 'validate', 'validatecommand',
'values', 'width', 'wrap', 'xscrollcommand',
)
- IDENTIFY_AS = 'Spinbox.field' if sys.platform == 'darwin' else 'textarea'
+ IDENTIFY_AS = {'Spinbox.field', 'textarea'}
def setUp(self):
super().setUp()
From a144037ac3501720b5587ee53de590684943639a Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sun, 13 Oct 2024 10:39:05 +0200
Subject: [PATCH 055/269] [3.12] gh-61698: Use launchctl to detect macOS window
manager in tests (GH-118390) (#125393)
(cherry picked from commit ce740d46246b28bb675ba9d62214b59be9b8411e)
Co-authored-by: Erlend E. Aasland
---
Lib/test/support/__init__.py | 24 +++++++++---------------
1 file changed, 9 insertions(+), 15 deletions(-)
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 5432b1ec5c9d84..78f410e1455852 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -250,22 +250,16 @@ class USEROBJECTFLAGS(ctypes.Structure):
# process not running under the same user id as the current console
# user. To avoid that, raise an exception if the window manager
# connection is not available.
- from ctypes import cdll, c_int, pointer, Structure
- from ctypes.util import find_library
-
- app_services = cdll.LoadLibrary(find_library("ApplicationServices"))
-
- if app_services.CGMainDisplayID() == 0:
- reason = "gui tests cannot run without OS X window manager"
+ import subprocess
+ try:
+ rc = subprocess.run(["launchctl", "managername"],
+ capture_output=True, check=True)
+ managername = rc.stdout.decode("utf-8").strip()
+ except subprocess.CalledProcessError:
+ reason = "unable to detect macOS launchd job manager"
else:
- class ProcessSerialNumber(Structure):
- _fields_ = [("highLongOfPSN", c_int),
- ("lowLongOfPSN", c_int)]
- psn = ProcessSerialNumber()
- psn_p = pointer(psn)
- if ( (app_services.GetCurrentProcess(psn_p) < 0) or
- (app_services.SetFrontProcess(psn_p) < 0) ):
- reason = "cannot run without OS X gui process"
+ if managername != "Aqua":
+ reason = f"{managername=} -- can only run in a macOS GUI session"
# check on every platform whether tkinter can actually do anything
if not reason:
From 00c596d8ad913ef393d5b75d561608fc9ac521fa Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sun, 13 Oct 2024 13:17:55 +0200
Subject: [PATCH 056/269] [3.12] gh-125383: Update `fib` function comment for
accuracy (GH-125386) (#125396)
gh-125383: Update `fib` function comment for accuracy (GH-125386)
`Doc/tutorial/controlflow.rst`: fix comment for `fib` function
(cherry picked from commit 283ea5f3b2b6a18605b8598a979afe263b0f21ce)
Co-authored-by: Wulian
---
Doc/tutorial/controlflow.rst | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst
index 9b73ac475c78d5..b830ce94ba4f47 100644
--- a/Doc/tutorial/controlflow.rst
+++ b/Doc/tutorial/controlflow.rst
@@ -461,8 +461,8 @@ Defining Functions
We can create a function that writes the Fibonacci series to an arbitrary
boundary::
- >>> def fib(n): # write Fibonacci series up to n
- ... """Print a Fibonacci series up to n."""
+ >>> def fib(n): # write Fibonacci series less than n
+ ... """Print a Fibonacci series less than n."""
... a, b = 0, 1
... while a < n:
... print(a, end=' ')
From 243a8a9d68a87d103340b894366497101e6b0226 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sun, 13 Oct 2024 19:45:59 +0200
Subject: [PATCH 057/269] [3.12] gh-101291: Add versionadded directives for
PyUnstable_Long_* (GH-125384) (#125408)
gh-101291: Add versionadded directives for PyUnstable_Long_* (GH-125384)
(cherry picked from commit c6d7b644c2425b397cfb641f336bea70eb8a329a)
Co-authored-by: Sergey B Kirpichev
---
Doc/c-api/long.rst | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst
index af86810c6b166b..972d69a5194511 100644
--- a/Doc/c-api/long.rst
+++ b/Doc/c-api/long.rst
@@ -350,6 +350,9 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
Exactly what values are considered compact is an implementation detail
and is subject to change.
+ .. versionadded:: 3.12
+
+
.. c:function:: Py_ssize_t PyUnstable_Long_CompactValue(const PyLongObject* op)
If *op* is compact, as determined by :c:func:`PyUnstable_Long_IsCompact`,
@@ -357,3 +360,5 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
Otherwise, the return value is undefined.
+ .. versionadded:: 3.12
+
From f49221af4609ea98e0e41bfae0d056f98bdccae4 Mon Sep 17 00:00:00 2001
From: Barney Gale
Date: Sun, 13 Oct 2024 19:18:41 +0100
Subject: [PATCH 058/269] [3.12] GH-125069: Fix inconsistent joining in
`WindowsPath(PosixPath(...))` (GH-125156) (#125410)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
`PurePath.__init__()` incorrectly uses the `_raw_paths` of a given
`PurePath` object with a different flavour, even though the procedure to
join path segments can differ between flavours.
This change makes the `_raw_paths`-enabled deferred joining apply _only_
when the path flavours match.
(cherry picked from commit cb8e5995d89d9b90e83cf43310ec50e177484e70)
Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
---
Lib/pathlib.py | 4 ++--
Lib/test/test_pathlib.py | 8 ++++++++
.../2024-10-08-21-17-16.gh-issue-125069.0RP0Mx.rst | 4 ++++
3 files changed, 14 insertions(+), 2 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-08-21-17-16.gh-issue-125069.0RP0Mx.rst
diff --git a/Lib/pathlib.py b/Lib/pathlib.py
index 65ff0ee1977f80..02eb5c25981e31 100644
--- a/Lib/pathlib.py
+++ b/Lib/pathlib.py
@@ -359,9 +359,9 @@ def __init__(self, *args):
paths = []
for arg in args:
if isinstance(arg, PurePath):
- if arg._flavour is ntpath and self._flavour is posixpath:
+ if arg._flavour is not self._flavour:
# GH-103631: Convert separators for backwards compatibility.
- paths.extend(path.replace('\\', '/') for path in arg._raw_paths)
+ paths.append(arg.as_posix())
else:
paths.extend(arg._raw_paths)
else:
diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py
index ca604df70a9a4e..4437f878004c70 100644
--- a/Lib/test/test_pathlib.py
+++ b/Lib/test/test_pathlib.py
@@ -831,6 +831,14 @@ class PureWindowsPathTest(_BasePurePathTest, unittest.TestCase):
],
})
+ def test_constructor_nested_foreign_flavour(self):
+ # See GH-125069.
+ p1 = pathlib.PurePosixPath('b/c:\\d')
+ p2 = pathlib.PurePosixPath('b/', 'c:\\d')
+ self.assertEqual(p1, p2)
+ self.assertEqual(self.cls(p1), self.cls('b/c:/d'))
+ self.assertEqual(self.cls(p2), self.cls('b/c:/d'))
+
def test_drive_root_parts(self):
check = self._check_drive_root_parts
# First part is anchored.
diff --git a/Misc/NEWS.d/next/Library/2024-10-08-21-17-16.gh-issue-125069.0RP0Mx.rst b/Misc/NEWS.d/next/Library/2024-10-08-21-17-16.gh-issue-125069.0RP0Mx.rst
new file mode 100644
index 00000000000000..73d5fa59303d4d
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-08-21-17-16.gh-issue-125069.0RP0Mx.rst
@@ -0,0 +1,4 @@
+Fix an issue where providing a :class:`pathlib.PurePath` object as an
+initializer argument to a second :class:`~pathlib.PurePath` object with a
+different flavour resulted in arguments to the former object's initializer
+ being joined by the latter object's flavour.
From 20323bf733810d7dc7e6bcd86dfbc16b7f3c4b30 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 14 Oct 2024 06:51:59 +0200
Subject: [PATCH 059/269] [3.12] gh-123133: clarify p=0 case for "f" and "e"
formatting types (GH-125426) (#125429)
gh-123133: clarify p=0 case for "f" and "e" formatting types (GH-125426)
(cherry picked from commit cfc27bc50fe165330f2295f9ac0ad56ca5b0f31c)
Co-authored-by: Sergey B Kirpichev
Co-authored-by: Serhiy Storchaka
---
Doc/library/string.rst | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/Doc/library/string.rst b/Doc/library/string.rst
index 57a1f920523035..49aeb28d57c8d1 100644
--- a/Doc/library/string.rst
+++ b/Doc/library/string.rst
@@ -509,9 +509,8 @@ The available presentation types for :class:`float` and
| | significant digits. With no precision given, uses a |
| | precision of ``6`` digits after the decimal point for |
| | :class:`float`, and shows all coefficient digits |
- | | for :class:`~decimal.Decimal`. If no digits follow the |
- | | decimal point, the decimal point is also removed unless |
- | | the ``#`` option is used. |
+ | | for :class:`~decimal.Decimal`. If ``p=0``, the decimal |
+ | | point is omitted unless the ``#`` option is used. |
+---------+----------------------------------------------------------+
| ``'E'`` | Scientific notation. Same as ``'e'`` except it uses |
| | an upper case 'E' as the separator character. |
@@ -522,9 +521,8 @@ The available presentation types for :class:`float` and
| | precision given, uses a precision of ``6`` digits after |
| | the decimal point for :class:`float`, and uses a |
| | precision large enough to show all coefficient digits |
- | | for :class:`~decimal.Decimal`. If no digits follow the |
- | | decimal point, the decimal point is also removed unless |
- | | the ``#`` option is used. |
+ | | for :class:`~decimal.Decimal`. If ``p=0``, the decimal |
+ | | point is omitted unless the ``#`` option is used. |
+---------+----------------------------------------------------------+
| ``'F'`` | Fixed-point notation. Same as ``'f'``, but converts |
| | ``nan`` to ``NAN`` and ``inf`` to ``INF``. |
From 21524eec48f5b1c807f185253e9350cfdd897ce0 Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka
Date: Mon, 14 Oct 2024 10:04:44 +0300
Subject: [PATCH 060/269] [3.12] gh-86357: argparse: use str() consistently and
explicitly to print choices (GH-117766) (GH-125432)
(cherry picked from commit 66b3922b97388c328c9bd8df050eef11c0261fae)
Signed-off-by: Jan Chren ~rindeal
Co-authored-by: rindeal
---
Lib/argparse.py | 12 +++----
Lib/test/test_argparse.py | 31 ++++++++++++++++++-
...-04-19-05-58-50.gh-issue-117766.J3xepp.rst | 1 +
3 files changed, 36 insertions(+), 8 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-04-19-05-58-50.gh-issue-117766.J3xepp.rst
diff --git a/Lib/argparse.py b/Lib/argparse.py
index 0e13ea5860da97..22c9b07db18a4f 100644
--- a/Lib/argparse.py
+++ b/Lib/argparse.py
@@ -588,8 +588,7 @@ def _metavar_formatter(self, action, default_metavar):
if action.metavar is not None:
result = action.metavar
elif action.choices is not None:
- choice_strs = [str(choice) for choice in action.choices]
- result = '{%s}' % ','.join(choice_strs)
+ result = '{%s}' % ','.join(map(str, action.choices))
else:
result = default_metavar
@@ -637,8 +636,7 @@ def _expand_help(self, action):
if hasattr(params[name], '__name__'):
params[name] = params[name].__name__
if params.get('choices') is not None:
- choices_str = ', '.join([str(c) for c in params['choices']])
- params['choices'] = choices_str
+ params['choices'] = ', '.join(map(str, params['choices']))
return self._get_help_string(action) % params
def _iter_indented_subactions(self, action):
@@ -763,7 +761,7 @@ def _get_action_name(argument):
elif argument.dest not in (None, SUPPRESS):
return argument.dest
elif argument.choices:
- return '{' + ','.join(argument.choices) + '}'
+ return '{%s}' % ','.join(map(str, argument.choices))
else:
return None
@@ -2600,8 +2598,8 @@ def _check_value(self, action, value):
if isinstance(choices, str):
choices = iter(choices)
if value not in choices:
- args = {'value': value,
- 'choices': ', '.join(map(repr, action.choices))}
+ args = {'value': str(value),
+ 'choices': ', '.join(map(str, action.choices))}
msg = _('invalid choice: %(value)r (choose from %(choices)s)')
raise ArgumentError(action, msg % args)
diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py
index 956c1cd505a96e..d8a4a00292d230 100644
--- a/Lib/test/test_argparse.py
+++ b/Lib/test/test_argparse.py
@@ -15,6 +15,7 @@
import argparse
import warnings
+from enum import StrEnum
from test.support import os_helper
from unittest import mock
@@ -1021,6 +1022,34 @@ class TestDisallowLongAbbreviationAllowsShortGroupingPrefix(ParserTestCase):
]
+class TestStrEnumChoices(TestCase):
+ class Color(StrEnum):
+ RED = "red"
+ GREEN = "green"
+ BLUE = "blue"
+
+ def test_parse_enum_value(self):
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--color', choices=self.Color)
+ args = parser.parse_args(['--color', 'red'])
+ self.assertEqual(args.color, self.Color.RED)
+
+ def test_help_message_contains_enum_choices(self):
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--color', choices=self.Color, help='Choose a color')
+ self.assertIn('[--color {red,green,blue}]', parser.format_usage())
+ self.assertIn(' --color {red,green,blue}', parser.format_help())
+
+ def test_invalid_enum_value_raises_error(self):
+ parser = argparse.ArgumentParser(exit_on_error=False)
+ parser.add_argument('--color', choices=self.Color)
+ self.assertRaisesRegex(
+ argparse.ArgumentError,
+ r"invalid choice: 'yellow' \(choose from red, green, blue\)",
+ parser.parse_args,
+ ['--color', 'yellow'],
+ )
+
# ================
# Positional tests
# ================
@@ -2422,7 +2451,7 @@ def test_wrong_argument_subparsers_no_destination_error(self):
parser.parse_args(('baz',))
self.assertRegex(
excinfo.exception.stderr,
- r"error: argument {foo,bar}: invalid choice: 'baz' \(choose from 'foo', 'bar'\)\n$"
+ r"error: argument {foo,bar}: invalid choice: 'baz' \(choose from foo, bar\)\n$"
)
def test_optional_subparsers(self):
diff --git a/Misc/NEWS.d/next/Library/2024-04-19-05-58-50.gh-issue-117766.J3xepp.rst b/Misc/NEWS.d/next/Library/2024-04-19-05-58-50.gh-issue-117766.J3xepp.rst
new file mode 100644
index 00000000000000..d090f931f0238d
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-04-19-05-58-50.gh-issue-117766.J3xepp.rst
@@ -0,0 +1 @@
+Always use :func:`str` to print ``choices`` in :mod:`argparse`.
From f1a6f687774ca09255c48d76deb060bd09c8e1ca Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 14 Oct 2024 17:49:56 +0200
Subject: [PATCH 061/269] [3.12] gh-125461: Remove Python 2 from identifiers in
doc (GH-125462) (#125465)
gh-125461: Remove Python 2 from identifiers in doc (GH-125462)
Remove Python 2 from identifiers in doc
(cherry picked from commit 5dac0dceda9097d46a0b5a6ad7c927e002c6c7a5)
Co-authored-by: Paul Hoffman
---
Doc/reference/lexical_analysis.rst | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst
index cfae01ba97a555..41d8fbaee97750 100644
--- a/Doc/reference/lexical_analysis.rst
+++ b/Doc/reference/lexical_analysis.rst
@@ -284,11 +284,10 @@ UAX-31, with elaboration and changes as defined below; see also :pep:`3131` for
further details.
Within the ASCII range (U+0001..U+007F), the valid characters for identifiers
-are the same as in Python 2.x: the uppercase and lowercase letters ``A`` through
+include the uppercase and lowercase letters ``A`` through
``Z``, the underscore ``_`` and, except for the first character, the digits
``0`` through ``9``.
-
-Python 3.0 introduces additional characters from outside the ASCII range (see
+Python 3.0 introduced additional characters from outside the ASCII range (see
:pep:`3131`). For these characters, the classification uses the version of the
Unicode Character Database as included in the :mod:`unicodedata` module.
From 92908681557308baa60f86d9ed24c132debfc35b Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka
Date: Mon, 14 Oct 2024 23:59:01 +0300
Subject: [PATCH 062/269] [3.12] gh-53203: Fix strptime() for %c, %x and %X
formats on many locales (GH-125406) (GH-125454) (GH-125483)
Fixed most locales that use non-ASCII digits, like Persian, Burmese,
Odia and Shan.
(cherry picked from commit 5f4e5b598cab86d5fd5727d423c9728221889ed0)
(cherry picked from commit cbcdf34a4b69a28a7e31715423fb080cb5cf8075)
Co-authored-by: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com>
---
Lib/_strptime.py | 64 ++++++++++++-------
Lib/test/test_strptime.py | 48 +++++++++-----
Lib/test/test_time.py | 2 +-
...4-10-13-20-21-35.gh-issue-53203.Rz1c8A.rst | 2 +
4 files changed, 74 insertions(+), 42 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-13-20-21-35.gh-issue-53203.Rz1c8A.rst
diff --git a/Lib/_strptime.py b/Lib/_strptime.py
index d740c15519a75d..dfd2bc5d8b4af5 100644
--- a/Lib/_strptime.py
+++ b/Lib/_strptime.py
@@ -14,6 +14,7 @@
import locale
import calendar
from re import compile as re_compile
+from re import sub as re_sub
from re import IGNORECASE
from re import escape as re_escape
from datetime import (date as datetime_date,
@@ -128,11 +129,23 @@ def __calc_date_time(self):
time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0))
time_tuple2 = time.struct_time((1999,1,3,1,1,1,6,3,0))
replacement_pairs = [
- ('1999', '%Y'), ('99', '%y'), ('22', '%H'),
- ('44', '%M'), ('55', '%S'), ('76', '%j'),
- ('17', '%d'), ('03', '%m'), ('3', '%m'),
- # '3' needed for when no leading zero.
- ('2', '%w'), ('10', '%I')]
+ ('1999', '%Y'), ('99', '%y'), ('22', '%H'),
+ ('44', '%M'), ('55', '%S'), ('76', '%j'),
+ ('17', '%d'), ('03', '%m'), ('3', '%m'),
+ # '3' needed for when no leading zero.
+ ('2', '%w'), ('10', '%I'),
+ # Non-ASCII digits
+ ('\u0661\u0669\u0669\u0669', '%Y'),
+ ('\u0669\u0669', '%Oy'),
+ ('\u0662\u0662', '%OH'),
+ ('\u0664\u0664', '%OM'),
+ ('\u0665\u0665', '%OS'),
+ ('\u0661\u0667', '%Od'),
+ ('\u0660\u0663', '%Om'),
+ ('\u0663', '%Om'),
+ ('\u0662', '%Ow'),
+ ('\u0661\u0660', '%OI'),
+ ]
date_time = []
for directive in ('%c', '%x', '%X'):
current_format = time.strftime(directive, time_tuple).lower()
@@ -157,6 +170,10 @@ def __calc_date_time(self):
for tz in tz_values:
if tz:
current_format = current_format.replace(tz, "%Z")
+ # Transform all non-ASCII digits to digits in range U+0660 to U+0669.
+ current_format = re_sub(r'\d(?3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])",
'f': r"(?P[0-9]{1,6})",
@@ -295,11 +312,15 @@ def __init__(self, locale_time=None):
'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone
for tz in tz_names),
'Z'),
- '%': '%'})
- base.__setitem__('W', base.__getitem__('U').replace('U', 'W'))
- base.__setitem__('c', self.pattern(self.locale_time.LC_date_time))
- base.__setitem__('x', self.pattern(self.locale_time.LC_date))
+ '%': '%'}
+ for d in 'dmyHIMS':
+ mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d
+ mapping['Ow'] = r'(?P\d)'
+ mapping['W'] = mapping['U'].replace('U', 'W')
+ base.__init__(mapping)
base.__setitem__('X', self.pattern(self.locale_time.LC_time))
+ base.__setitem__('x', self.pattern(self.locale_time.LC_date))
+ base.__setitem__('c', self.pattern(self.locale_time.LC_date_time))
def __seqToRE(self, to_convert, directive):
"""Convert a list to a regex string for matching a directive.
@@ -327,21 +348,16 @@ def pattern(self, format):
regex syntax are escaped.
"""
- processed_format = ''
# The sub() call escapes all characters that might be misconstrued
# as regex syntax. Cannot use re.escape since we have to deal with
# format directives (%m, etc.).
- regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])")
- format = regex_chars.sub(r"\\\1", format)
- whitespace_replacement = re_compile(r'\s+')
- format = whitespace_replacement.sub(r'\\s+', format)
- while '%' in format:
- directive_index = format.index('%')+1
- processed_format = "%s%s%s" % (processed_format,
- format[:directive_index-1],
- self[format[directive_index]])
- format = format[directive_index+1:]
- return "%s%s" % (processed_format, format)
+ format = re_sub(r"([\\.^$*+?\(\){}\[\]|])", r"\\\1", format)
+ format = re_sub(r'\s+', r'\\s+', format)
+ format = re_sub(r"'", "['\u02bc]", format) # needed for br_FR
+ def repl(m):
+ return self[m[1]]
+ format = re_sub(r'%(O?.)', repl, format)
+ return format
def compile(self, format):
"""Return a compiled re object for the format string."""
@@ -415,8 +431,8 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
_regex_cache[format] = format_regex
found = format_regex.match(data_string)
if not found:
- raise ValueError("time data %r does not match format %r :: /%s/" %
- (data_string, format, format_regex.pattern))
+ raise ValueError("time data %r does not match format %r" %
+ (data_string, format))
if len(data_string) != found.end():
raise ValueError("unconverted data remains: %s" %
data_string[found.end():])
diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py
index fa7915bead83fd..45b5c8ed2c3051 100644
--- a/Lib/test/test_strptime.py
+++ b/Lib/test/test_strptime.py
@@ -290,7 +290,7 @@ def test_strptime_exception_context(self):
# additional check for IndexError branch (issue #19545)
with self.assertRaises(ValueError) as e:
_strptime._strptime_time('19', '%Y %')
- self.assertIs(e.exception.__suppress_context__, True)
+ self.assertIsNone(e.exception.__context__)
def test_unconverteddata(self):
# Check ValueError is raised when there is unconverted data
@@ -483,12 +483,14 @@ def test_bad_timezone(self):
# id_ID, ms_MY.
# * Year is not included: ha_NG.
# * Use non-Gregorian calendar: lo_LA, thai, th_TH.
+ # On Windows: ar_IN, ar_SA, fa_IR, ps_AF.
#
# BUG: Generates regexp that does not match the current date and time
- # for az_IR, fa_IR, lzh_TW, my_MM, or_IN, shn_MM.
+ # for lzh_TW.
@run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP',
'he_IL', 'eu_ES', 'ar_AE', 'mfe_MU', 'yo_NG',
- 'csb_PL', 'br_FR', 'gez_ET', 'brx_IN')
+ 'csb_PL', 'br_FR', 'gez_ET', 'brx_IN',
+ 'my_MM', 'or_IN', 'shn_MM', 'az_IR')
def test_date_time_locale(self):
# Test %c directive
loc = locale.getlocale(locale.LC_TIME)[0]
@@ -510,20 +512,23 @@ def test_date_time_locale(self):
self.roundtrip('%c', slice(0, 6), time.localtime(now - 366*24*3600))
# NB: Dates before 1969 do not roundtrip on some locales:
- # bo_CN, bo_IN, dz_BT, eu_ES, eu_FR.
+ # az_IR, bo_CN, bo_IN, dz_BT, eu_ES, eu_FR, fa_IR, or_IN.
@run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP',
'he_IL', 'ar_AE', 'mfe_MU', 'yo_NG',
- 'csb_PL', 'br_FR', 'gez_ET', 'brx_IN')
+ 'csb_PL', 'br_FR', 'gez_ET', 'brx_IN',
+ 'my_MM', 'shn_MM')
def test_date_time_locale2(self):
# Test %c directive
self.roundtrip('%c', slice(0, 6), (1900, 1, 1, 0, 0, 0, 0, 1, 0))
+ self.roundtrip('%c', slice(0, 6), (1800, 1, 1, 0, 0, 0, 0, 1, 0))
# NB: Does not roundtrip because use non-Gregorian calendar:
- # lo_LA, thai, th_TH.
+ # lo_LA, thai, th_TH. On Windows: ar_IN, ar_SA, fa_IR, ps_AF.
# BUG: Generates regexp that does not match the current date
- # for az_IR, fa_IR, lzh_TW, my_MM, or_IN, shn_MM.
+ # for lzh_TW.
@run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP',
- 'he_IL', 'eu_ES', 'ar_AE')
+ 'he_IL', 'eu_ES', 'ar_AE',
+ 'az_IR', 'my_MM', 'or_IN', 'shn_MM')
def test_date_locale(self):
# Test %x directive
now = time.time()
@@ -543,10 +548,11 @@ def test_date_locale(self):
"musl libc issue on Emscripten, bpo-46390"
)
@run_with_locales('LC_TIME', 'en_US', 'fr_FR', 'de_DE', 'ja_JP',
- 'eu_ES', 'ar_AE')
+ 'eu_ES', 'ar_AE', 'my_MM', 'shn_MM')
def test_date_locale2(self):
# Test %x directive
self.roundtrip('%x', slice(0, 3), (1900, 1, 1, 0, 0, 0, 0, 1, 0))
+ self.roundtrip('%x', slice(0, 3), (1800, 1, 1, 0, 0, 0, 0, 1, 0))
# NB: Does not roundtrip in some locales due to the ambiguity of
# the time representation (bugs in locales?):
@@ -554,19 +560,27 @@ def test_date_locale2(self):
# norwegian, nynorsk.
# * Hours are in 12-hour notation without AM/PM indication: hy_AM,
# ms_MY, sm_WS.
- # BUG: Generates regexp that does not match the current time for
- # aa_DJ, aa_ER, aa_ET, am_ET, az_IR, byn_ER, fa_IR, gez_ER, gez_ET,
- # lzh_TW, my_MM, om_ET, om_KE, or_IN, shn_MM, sid_ET, so_DJ, so_ET,
- # so_SO, ti_ER, ti_ET, tig_ER, wal_ET.
- @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP')
+ # BUG: Generates regexp that does not match the current time for lzh_TW.
+ @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP',
+ 'aa_ET', 'am_ET', 'az_IR', 'byn_ER', 'fa_IR', 'gez_ET',
+ 'my_MM', 'om_ET', 'or_IN', 'shn_MM', 'sid_ET', 'so_SO',
+ 'ti_ET', 'tig_ER', 'wal_ET')
def test_time_locale(self):
# Test %X directive
+ loc = locale.getlocale(locale.LC_TIME)[0]
+ pos = slice(3, 6)
+ if glibc_ver and glibc_ver < (2, 29) and loc in {
+ 'aa_ET', 'am_ET', 'byn_ER', 'gez_ET', 'om_ET',
+ 'sid_ET', 'so_SO', 'ti_ET', 'tig_ER', 'wal_ET'}:
+ # Hours are in 12-hour notation without AM/PM indication.
+ # Ignore hours.
+ pos = slice(4, 6)
now = time.time()
- self.roundtrip('%X', slice(3, 6), time.localtime(now))
+ self.roundtrip('%X', pos, time.localtime(now))
# 1 hour 20 minutes 30 seconds ago
- self.roundtrip('%X', slice(3, 6), time.localtime(now - 4830))
+ self.roundtrip('%X', pos, time.localtime(now - 4830))
# 12 hours ago
- self.roundtrip('%X', slice(3, 6), time.localtime(now - 12*3600))
+ self.roundtrip('%X', pos, time.localtime(now - 12*3600))
def test_percent(self):
# Make sure % signs are handled properly
diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py
index 921e4eea649d6b..ca40415f0cdd76 100644
--- a/Lib/test/test_time.py
+++ b/Lib/test/test_time.py
@@ -292,7 +292,7 @@ def test_strptime_exception_context(self):
# additional check for IndexError branch (issue #19545)
with self.assertRaises(ValueError) as e:
time.strptime('19', '%Y %')
- self.assertIs(e.exception.__suppress_context__, True)
+ self.assertIsNone(e.exception.__context__)
def test_asctime(self):
time.asctime(time.gmtime(self.t))
diff --git a/Misc/NEWS.d/next/Library/2024-10-13-20-21-35.gh-issue-53203.Rz1c8A.rst b/Misc/NEWS.d/next/Library/2024-10-13-20-21-35.gh-issue-53203.Rz1c8A.rst
new file mode 100644
index 00000000000000..cdfa8c191e8242
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-13-20-21-35.gh-issue-53203.Rz1c8A.rst
@@ -0,0 +1,2 @@
+Fix :func:`time.strptime` for ``%c``, ``%x`` and ``%X`` formats in many
+locales that use non-ASCII digits, like Persian, Burmese, Odia and Shan.
From 86296bbc178aa2f7094c8d7ab9eb4175f78d105f Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 14 Oct 2024 23:34:38 +0200
Subject: [PATCH 063/269] [3.12] Fix idlelib typos (GH-125484) (#125488)
Fix idlelib typos (GH-125484)
Propagate fixes in Doc/library/idle.rst to help.html.
Change 'interruptable' to 'interruptible' in run.py.
The latter was reported by ember91 in PR 125473.
(cherry picked from commit 3fea1d000ef0a74062fd3fe218ad94618b08d9f2)
Co-authored-by: Terry Jan Reedy
---
Lib/idlelib/help.html | 21 ++++++++++++---------
Lib/idlelib/run.py | 12 ++++++------
2 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html
index 827d230b54e159..2a4adc6a4d395f 100644
--- a/Lib/idlelib/help.html
+++ b/Lib/idlelib/help.html
@@ -5,7 +5,7 @@
- IDLE — Python 3.13.0a2 documentation
+ IDLE — Python 3.14.0a0 documentation
@@ -18,7 +18,7 @@
@@ -26,6 +26,7 @@
+
@@ -45,6 +46,8 @@
+
+
@@ -184,7 +187,7 @@ Navigation
- 3.13.0a2 Documentation »
+ 3.14.0a0 Documentation »
The Python Standard Library »
@@ -554,7 +557,7 @@ Key bindings
@@ -694,7 +697,7 @@ Shell windowFound a bug ?
diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py
index 476a7b26c004b5..2faeba678c7262 100644
--- a/Lib/idlelib/run.py
+++ b/Lib/idlelib/run.py
@@ -101,11 +101,11 @@ def handle_tk_events(tcl=tcl):
# Thread shared globals: Establish a queue between a subthread (which handles
# the socket) and the main thread (which runs user code), plus global
-# completion, exit and interruptable (the main thread) flags:
+# completion, exit and interruptible (the main thread) flags:
exit_now = False
quitting = False
-interruptable = False
+interruptible = False
def main(del_exitfunc=False):
"""Start the Python execution server in a subprocess
@@ -575,14 +575,14 @@ def __init__(self, rpchandler):
self.locals = {}
def runcode(self, code):
- global interruptable
+ global interruptible
try:
self.user_exc_info = None
- interruptable = True
+ interruptible = True
try:
exec(code, self.locals)
finally:
- interruptable = False
+ interruptible = False
except SystemExit as e:
if e.args: # SystemExit called with an argument.
ob = e.args[0]
@@ -608,7 +608,7 @@ def runcode(self, code):
flush_stdout()
def interrupt_the_server(self):
- if interruptable:
+ if interruptible:
thread.interrupt_main()
def start_the_debugger(self, gui_adap_oid):
From 449f2c98bdd04694a987639a91f4fab366c2e9d4 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 15 Oct 2024 00:09:23 +0200
Subject: [PATCH 064/269] [3.12] gh-85453: Improve variable mark up for
datetime.rst (GH-120702) (#125491)
Variables and literals are marked up using backticks.
(cherry picked from commit 2a5cdb251674ce8d9a824c102f7cd846d944cfa4)
Co-authored-by: edson duarte
---
Doc/library/datetime.rst | 34 +++++++++++++++++-----------------
1 file changed, 17 insertions(+), 17 deletions(-)
diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst
index 671554f2cf3d0d..54ab05e4e66549 100644
--- a/Doc/library/datetime.rst
+++ b/Doc/library/datetime.rst
@@ -180,19 +180,19 @@ Objects of the :class:`date` type are always naive.
An object of type :class:`.time` or :class:`.datetime` may be aware or naive.
-A :class:`.datetime` object *d* is aware if both of the following hold:
+A :class:`.datetime` object ``d`` is aware if both of the following hold:
1. ``d.tzinfo`` is not ``None``
2. ``d.tzinfo.utcoffset(d)`` does not return ``None``
-Otherwise, *d* is naive.
+Otherwise, ``d`` is naive.
-A :class:`.time` object *t* is aware if both of the following hold:
+A :class:`.time` object ``t`` is aware if both of the following hold:
1. ``t.tzinfo`` is not ``None``
2. ``t.tzinfo.utcoffset(None)`` does not return ``None``.
-Otherwise, *t* is naive.
+Otherwise, ``t`` is naive.
The distinction between aware and naive doesn't apply to :class:`timedelta`
objects.
@@ -358,8 +358,8 @@ Supported operations:
+--------------------------------+-----------------------------------------------+
| ``q, r = divmod(t1, t2)`` | Computes the quotient and the remainder: |
| | ``q = t1 // t2`` (3) and ``r = t1 % t2``. |
-| | q is an integer and r is a :class:`timedelta` |
-| | object. |
+| | ``q`` is an integer and ``r`` is a |
+| | :class:`timedelta` object. |
+--------------------------------+-----------------------------------------------+
| ``+t1`` | Returns a :class:`timedelta` object with the |
| | same value. (2) |
@@ -526,7 +526,7 @@ Other constructors, all class methods:
January 1 of year 1 has ordinal 1.
:exc:`ValueError` is raised unless ``1 <= ordinal <=
- date.max.toordinal()``. For any date *d*,
+ date.max.toordinal()``. For any date ``d``,
``date.fromordinal(d.toordinal()) == d``.
@@ -678,7 +678,7 @@ Instance methods:
.. method:: date.toordinal()
Return the proleptic Gregorian ordinal of the date, where January 1 of year 1
- has ordinal 1. For any :class:`date` object *d*,
+ has ordinal 1. For any :class:`date` object ``d``,
``date.fromordinal(d.toordinal()) == d``.
@@ -730,7 +730,7 @@ Instance methods:
.. method:: date.__str__()
- For a date *d*, ``str(d)`` is equivalent to ``d.isoformat()``.
+ For a date ``d``, ``str(d)`` is equivalent to ``d.isoformat()``.
.. method:: date.ctime()
@@ -1011,7 +1011,7 @@ Other constructors, all class methods:
is used. If the *date* argument is a :class:`.datetime` object, its time components
and :attr:`.tzinfo` attributes are ignored.
- For any :class:`.datetime` object *d*,
+ For any :class:`.datetime` object ``d``,
``d == datetime.combine(d.date(), d.time(), d.tzinfo)``.
.. versionchanged:: 3.6
@@ -1200,11 +1200,11 @@ Supported operations:
If both are naive, or both are aware and have the same :attr:`~.datetime.tzinfo` attribute,
the :attr:`~.datetime.tzinfo` attributes are ignored, and the result is a :class:`timedelta`
- object *t* such that ``datetime2 + t == datetime1``. No time zone adjustments
+ object ``t`` such that ``datetime2 + t == datetime1``. No time zone adjustments
are done in this case.
If both are aware and have different :attr:`~.datetime.tzinfo` attributes, ``a-b`` acts
- as if *a* and *b* were first converted to naive UTC datetimes. The
+ as if ``a`` and ``b`` were first converted to naive UTC datetimes. The
result is ``(a.replace(tzinfo=None) - a.utcoffset()) - (b.replace(tzinfo=None)
- b.utcoffset())`` except that the implementation never overflows.
@@ -1377,11 +1377,11 @@ Instance methods:
.. method:: datetime.utctimetuple()
- If :class:`.datetime` instance *d* is naive, this is the same as
+ If :class:`.datetime` instance ``d`` is naive, this is the same as
``d.timetuple()`` except that :attr:`~.time.struct_time.tm_isdst` is forced to 0 regardless of what
``d.dst()`` returns. DST is never in effect for a UTC time.
- If *d* is aware, *d* is normalized to UTC time, by subtracting
+ If ``d`` is aware, ``d`` is normalized to UTC time, by subtracting
``d.utcoffset()``, and a :class:`time.struct_time` for the
normalized time is returned. :attr:`!tm_isdst` is forced to 0. Note
that an :exc:`OverflowError` may be raised if ``d.year`` was
@@ -1529,7 +1529,7 @@ Instance methods:
.. method:: datetime.__str__()
- For a :class:`.datetime` instance *d*, ``str(d)`` is equivalent to
+ For a :class:`.datetime` instance ``d``, ``str(d)`` is equivalent to
``d.isoformat(' ')``.
@@ -1776,7 +1776,7 @@ Instance attributes (read-only):
.. versionadded:: 3.6
:class:`.time` objects support equality and order comparisons,
-where *a* is considered less than *b* when *a* precedes *b* in time.
+where ``a`` is considered less than ``b`` when ``a`` precedes ``b`` in time.
Naive and aware :class:`!time` objects are never equal.
Order comparison between naive and aware :class:`!time` objects raises
@@ -1904,7 +1904,7 @@ Instance methods:
.. method:: time.__str__()
- For a time *t*, ``str(t)`` is equivalent to ``t.isoformat()``.
+ For a time ``t``, ``str(t)`` is equivalent to ``t.isoformat()``.
.. method:: time.strftime(format)
From 79422bc13b17331211619354b7d8acaccafcc79a Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 15 Oct 2024 09:36:17 +0200
Subject: [PATCH 065/269] [3.12] Doc: Update CVE URL (GH-125489) (#125504)
Co-authored-by: Mariusz Felisiak
---
Doc/conf.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Doc/conf.py b/Doc/conf.py
index 36f6243969c8b2..fb55c5c65c2f35 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -593,7 +593,7 @@
# Sphinx 8.1 has in-built CVE and CWE roles.
extlinks |= {
"cve": (
- "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-%s",
+ "https://www.cve.org/CVERecord?id=CVE-%s",
"CVE-%s",
),
"cwe": ("https://cwe.mitre.org/data/definitions/%s.html", "CWE-%s"),
From 26725d1756424b8abff896ffed7b017dd59241fc Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 15 Oct 2024 16:28:23 +0200
Subject: [PATCH 066/269] [3.12] gh-125514: fix bug in test_traceback utility.
Specify exception types in except: clauses (GH-125516) (#125525)
gh-125514: fix bug in test_traceback utility. Specify exception types in except: clauses (GH-125516)
(cherry picked from commit 55c4f4c30b49734ce35dc88139b8b4fdc94c66fd)
Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com>
---
Lib/test/test_traceback.py | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py
index d12b559cf076d6..119143e4f3a009 100644
--- a/Lib/test/test_traceback.py
+++ b/Lib/test/test_traceback.py
@@ -138,7 +138,7 @@ def test_no_caret_with_no_debug_ranges_flag_python_traceback(self):
import traceback
try:
x = 1 / 0
- except:
+ except ZeroDivisionError:
traceback.print_exc()
""")
try:
@@ -386,9 +386,10 @@ class PurePythonExceptionFormattingMixin:
def get_exception(self, callable, slice_start=0, slice_end=-1):
try:
callable()
- self.fail("No exception thrown.")
- except:
+ except BaseException:
return traceback.format_exc().splitlines()[slice_start:slice_end]
+ else:
+ self.fail("No exception thrown.")
callable_line = get_exception.__code__.co_firstlineno + 2
@@ -1490,7 +1491,7 @@ def test_context_suppression(self):
try:
try:
raise Exception
- except:
+ except Exception:
raise ZeroDivisionError from None
except ZeroDivisionError as _:
e = _
@@ -1838,9 +1839,9 @@ def exc():
try:
try:
raise EG("eg1", [ValueError(1), TypeError(2)])
- except:
+ except EG:
raise EG("eg2", [ValueError(3), TypeError(4)])
- except:
+ except EG:
raise ImportError(5)
expected = (
@@ -1889,7 +1890,7 @@ def exc():
except Exception as e:
exc = e
raise EG("eg", [VE(1), exc, VE(4)])
- except:
+ except EG:
raise EG("top", [VE(5)])
expected = (f' + Exception Group Traceback (most recent call last):\n'
@@ -2642,7 +2643,7 @@ def test_long_context_chain(self):
def f():
try:
1/0
- except:
+ except ZeroDivisionError:
f()
try:
@@ -2731,7 +2732,7 @@ def test_comparison_params_variations(self):
def raise_exc():
try:
raise ValueError('bad value')
- except:
+ except ValueError:
raise
def raise_with_locals():
From 0a82c4cbd8c0b74e6111370d3c498142fdf9f085 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 15 Oct 2024 17:26:41 +0200
Subject: [PATCH 067/269] [3.12] gh-125422: Don't set the caller's f_trace if
it's botframe (GH-125427) (#125531)
gh-125422: Don't set the caller's f_trace if it's botframe (GH-125427)
(cherry picked from commit 703227dd021491ceb9343f69fa48f4b6a05adbb3)
Co-authored-by: Tian Gao
---
Lib/bdb.py | 5 +++--
Lib/test/test_bdb.py | 13 +++++++++++++
Lib/test/test_pdb.py | 14 ++++++++++++++
.../2024-10-14-04-44-12.gh-issue-125422.MlVuC6.rst | 1 +
4 files changed, 31 insertions(+), 2 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-14-04-44-12.gh-issue-125422.MlVuC6.rst
diff --git a/Lib/bdb.py b/Lib/bdb.py
index 564d6c5e5324ed..196e6b178cb9fd 100644
--- a/Lib/bdb.py
+++ b/Lib/bdb.py
@@ -295,9 +295,10 @@ def _set_caller_tracefunc(self, current_frame):
# Issue #13183: pdb skips frames after hitting a breakpoint and running
# step commands.
# Restore the trace function in the caller (that may not have been set
- # for performance reasons) when returning from the current frame.
+ # for performance reasons) when returning from the current frame, unless
+ # the caller is the botframe.
caller_frame = current_frame.f_back
- if caller_frame and not caller_frame.f_trace:
+ if caller_frame and not caller_frame.f_trace and caller_frame is not self.botframe:
caller_frame.f_trace = self.trace_dispatch
# Derived classes and clients can call the following methods
diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py
index 568c88e326c087..33e28592f59e1b 100644
--- a/Lib/test/test_bdb.py
+++ b/Lib/test/test_bdb.py
@@ -1203,6 +1203,19 @@ def main():
with TracerRun(self) as tracer:
tracer.runcall(tfunc_import)
+ def test_next_to_botframe(self):
+ # gh-125422
+ # Check that next command won't go to the bottom frame.
+ code = """
+ lno = 2
+ """
+ self.expect_set = [
+ ('line', 2, ''), ('step', ),
+ ('return', 2, ''), ('next', ),
+ ]
+ with TracerRun(self) as tracer:
+ tracer.run(compile(textwrap.dedent(code), '', 'exec'))
+
class TestRegressions(unittest.TestCase):
def test_format_stack_entry_no_lineno(self):
diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py
index 8a7e41b281165e..6da82fe04c0eaa 100644
--- a/Lib/test/test_pdb.py
+++ b/Lib/test/test_pdb.py
@@ -2283,6 +2283,20 @@ def test_issue26053(self):
self.assertRegex(res, "Restarting .* with arguments:\na b c")
self.assertRegex(res, "Restarting .* with arguments:\nd e f")
+ def test_step_into_botframe(self):
+ # gh-125422
+ # pdb should not be able to step into the botframe (bdb.py)
+ script = "x = 1"
+ commands = """
+ step
+ step
+ step
+ quit
+ """
+ stdout, _ = self.run_pdb_script(script, commands)
+ self.assertIn("The program finished", stdout)
+ self.assertNotIn("bdb.py", stdout)
+
def test_pdbrc_basic(self):
script = textwrap.dedent("""
a = 1
diff --git a/Misc/NEWS.d/next/Library/2024-10-14-04-44-12.gh-issue-125422.MlVuC6.rst b/Misc/NEWS.d/next/Library/2024-10-14-04-44-12.gh-issue-125422.MlVuC6.rst
new file mode 100644
index 00000000000000..c890ecec8beaf8
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-14-04-44-12.gh-issue-125422.MlVuC6.rst
@@ -0,0 +1 @@
+Fixed the bug where :mod:`pdb` and :mod:`bdb` can step into the bottom caller frame.
From 1cab726622985a5596529521f471aea8588747c3 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 15 Oct 2024 18:09:55 +0200
Subject: [PATCH 068/269] [3.12] gh-89819: Add argument_default and
conflict_handler to add_argument_group() docs (GH-125379) (GH-125539)
(cherry picked from commit c9826c11db25e81b1a90c837f84074879f1b1126)
Co-authored-by: Savannah Ostrowski
---
Doc/library/argparse.rst | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index a1f08ea272934f..87d9a45539a1dd 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -1740,7 +1740,8 @@ FileType objects
Argument groups
^^^^^^^^^^^^^^^
-.. method:: ArgumentParser.add_argument_group(title=None, description=None)
+.. method:: ArgumentParser.add_argument_group(title=None, description=None, *, \
+ [argument_default], [conflict_handler])
By default, :class:`ArgumentParser` groups command-line arguments into
"positional arguments" and "options" when displaying help
@@ -1785,6 +1786,11 @@ Argument groups
--bar BAR bar help
+ The optional, keyword-only parameters argument_default_ and conflict_handler_
+ allow for finer-grained control of the behavior of the argument group. These
+ parameters have the same meaning as in the :class:`ArgumentParser` constructor,
+ but apply specifically to the argument group rather than the entire parser.
+
Note that any arguments not in your user-defined groups will end up back
in the usual "positional arguments" and "optional arguments" sections.
From 90b1406b881aa0a57ce3a73c402b73b79138b7e0 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 15 Oct 2024 18:29:05 +0200
Subject: [PATCH 069/269] [3.12] gh-100141: Allow pdb to deal with empty file
(GH-125425) (#125537)
gh-100141: Allow pdb to deal with empty file (GH-125425)
(cherry picked from commit bb9604b62ae7f043594ffea9287f9213067cc7fb)
Co-authored-by: Tian Gao
---
Lib/pdb.py | 3 +--
Lib/test/test_pdb.py | 10 ++++++++++
.../2024-10-14-02-27-03.gh-issue-100141.NuAcwa.rst | 1 +
3 files changed, 12 insertions(+), 2 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-14-02-27-03.gh-issue-100141.NuAcwa.rst
diff --git a/Lib/pdb.py b/Lib/pdb.py
index 89cf975164ac04..559517813cf600 100755
--- a/Lib/pdb.py
+++ b/Lib/pdb.py
@@ -321,8 +321,7 @@ def user_call(self, frame, argument_list):
def user_line(self, frame):
"""This function is called when we stop or break at this line."""
if self._wait_for_mainpyfile:
- if (self.mainpyfile != self.canonic(frame.f_code.co_filename)
- or frame.f_lineno <= 0):
+ if (self.mainpyfile != self.canonic(frame.f_code.co_filename)):
return
self._wait_for_mainpyfile = False
if self.bp_commands(frame):
diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py
index 6da82fe04c0eaa..1542bb3bee1aea 100644
--- a/Lib/test/test_pdb.py
+++ b/Lib/test/test_pdb.py
@@ -2754,6 +2754,16 @@ def _create_fake_frozen_module():
# verify that pdb found the source of the "frozen" function
self.assertIn('x = "Sentinel string for gh-93696"', stdout, "Sentinel statement not found")
+ def test_empty_file(self):
+ script = ''
+ commands = 'q\n'
+ # We check that pdb stopped at line 0, but anything reasonable
+ # is acceptable here, as long as it does not halt
+ stdout, _ = self.run_pdb_script(script, commands)
+ self.assertIn('main.py(0)', stdout)
+ stdout, _ = self.run_pdb_module(script, commands)
+ self.assertIn('__main__.py(0)', stdout)
+
def test_non_utf8_encoding(self):
script_dir = os.path.join(os.path.dirname(__file__), 'encoded_modules')
for filename in os.listdir(script_dir):
diff --git a/Misc/NEWS.d/next/Library/2024-10-14-02-27-03.gh-issue-100141.NuAcwa.rst b/Misc/NEWS.d/next/Library/2024-10-14-02-27-03.gh-issue-100141.NuAcwa.rst
new file mode 100644
index 00000000000000..c366b0ad4040d3
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-14-02-27-03.gh-issue-100141.NuAcwa.rst
@@ -0,0 +1 @@
+Fixed the bug where :mod:`pdb` will be stuck in an infinite loop when debugging an empty file.
From cbd50a4bdc7ec474221334324a09e5f2053adea6 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 16 Oct 2024 14:44:37 +0200
Subject: [PATCH 070/269] [3.12] gh-125041: test_zlib: For s390x HW
acceleration, only skip checking the compressed bytes (GH-125042) (GH-125526)
(cherry picked from commit cc5a225cdc2a5d4e035dd08d59cef39182c10a6c)
Co-authored-by: Petr Viktorin
---
Lib/test/support/__init__.py | 6 ++---
Lib/test/test_zlib.py | 25 ++++++++++++-------
...-10-07-14-13-38.gh-issue-125041.PKLWDf.rst | 3 +++
3 files changed, 22 insertions(+), 12 deletions(-)
create mode 100644 Misc/NEWS.d/next/Tests/2024-10-07-14-13-38.gh-issue-125041.PKLWDf.rst
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 78f410e1455852..ba57eb307c0b8f 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -2450,9 +2450,9 @@ def adjust_int_max_str_digits(max_digits):
else:
C_RECURSION_LIMIT = 10000
-#Windows doesn't have os.uname() but it doesn't support s390x.
-skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',
- 'skipped on s390x')
+# Windows doesn't have os.uname() but it doesn't support s390x.
+is_s390x = hasattr(os, 'uname') and os.uname().machine == 's390x'
+skip_on_s390x = unittest.skipIf(is_s390x, 'skipped on s390x')
_BASE_COPY_SRC_DIR_IGNORED_NAMES = frozenset({
# SRC_DIR/.git
diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py
index 0a13986a847f0a..8654b93ec64ac8 100644
--- a/Lib/test/test_zlib.py
+++ b/Lib/test/test_zlib.py
@@ -7,7 +7,7 @@
import pickle
import random
import sys
-from test.support import bigmemtest, _1G, _4G, skip_on_s390x
+from test.support import bigmemtest, _1G, _4G, is_s390x
zlib = import_helper.import_module('zlib')
@@ -34,8 +34,9 @@ def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION):
ZLIB_RUNTIME_VERSION_TUPLE = _zlib_runtime_version_tuple()
-# bpo-46623: On s390x, when a hardware accelerator is used, using different
-# ways to compress data with zlib can produce different compressed data.
+# bpo-46623: When a hardware accelerator is used (currently only on s390x),
+# using different ways to compress data with zlib can produce different
+# compressed data.
# Simplified test_pair() code:
#
# def func1(data):
@@ -58,8 +59,10 @@ def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION):
#
# zlib.decompress(func1(data)) == zlib.decompress(func2(data)) == data
#
-# Make the assumption that s390x always has an accelerator to simplify the skip
-# condition.
+# To simplify the skip condition, make the assumption that s390x always has an
+# accelerator, and nothing else has it.
+HW_ACCELERATED = is_s390x
+
class VersionTestCase(unittest.TestCase):
@@ -224,12 +227,14 @@ def test_keywords(self):
bufsize=zlib.DEF_BUF_SIZE),
HAMLET_SCENE)
- @skip_on_s390x
def test_speech128(self):
# compress more data
data = HAMLET_SCENE * 128
x = zlib.compress(data)
- self.assertEqual(zlib.compress(bytearray(data)), x)
+ # With hardware acceleration, the compressed bytes
+ # might not be identical.
+ if not HW_ACCELERATED:
+ self.assertEqual(zlib.compress(bytearray(data)), x)
for ob in x, bytearray(x):
self.assertEqual(zlib.decompress(ob), data)
@@ -276,7 +281,6 @@ def test_64bit_compress(self, size):
class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
# Test compression object
- @skip_on_s390x
def test_pair(self):
# straightforward compress/decompress objects
datasrc = HAMLET_SCENE * 128
@@ -287,7 +291,10 @@ def test_pair(self):
x1 = co.compress(data)
x2 = co.flush()
self.assertRaises(zlib.error, co.flush) # second flush should not work
- self.assertEqual(x1 + x2, datazip)
+ # With hardware acceleration, the compressed bytes might not
+ # be identical.
+ if not HW_ACCELERATED:
+ self.assertEqual(x1 + x2, datazip)
for v1, v2 in ((x1, x2), (bytearray(x1), bytearray(x2))):
dco = zlib.decompressobj()
y1 = dco.decompress(v1 + v2)
diff --git a/Misc/NEWS.d/next/Tests/2024-10-07-14-13-38.gh-issue-125041.PKLWDf.rst b/Misc/NEWS.d/next/Tests/2024-10-07-14-13-38.gh-issue-125041.PKLWDf.rst
new file mode 100644
index 00000000000000..c7181eb9c1f3a9
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2024-10-07-14-13-38.gh-issue-125041.PKLWDf.rst
@@ -0,0 +1,3 @@
+Re-enable skipped tests for :mod:`zlib` on the s390x architecture: only skip
+checks of the compressed bytes, which can be different between zlib's
+software implementation and the hardware-accelerated implementation.
From 4256847190e3f87ec357a1a4e8d9eb5c57367d5e Mon Sep 17 00:00:00 2001
From: Sam Gross
Date: Wed, 16 Oct 2024 14:03:32 -0400
Subject: [PATCH 071/269] [3.12] gh-125451: Fix deadlock in ProcessPoolExecutor
shutdown (GH-125492) (#125599)
There was a deadlock when `ProcessPoolExecutor` shuts down at the same
time that a queueing thread handles an error processing a task.
Don't use `_shutdown_lock` to protect the `_ThreadWakeup` pipes -- use
an internal lock instead. This fixes the ordering deadlock where the
`ExecutorManagerThread` holds the `_shutdown_lock` and joins the
queueing thread, while the queueing thread is attempting to acquire the
`_shutdown_lock` while closing the `_ThreadWakeup`.
(cherry picked from commit 760872efecb95017db8e38a8eda614bf23d2a22c)
---
Lib/concurrent/futures/process.py | 52 ++++++++-----------
...-10-14-17-29-34.gh-issue-125451.fmP3T9.rst | 2 +
2 files changed, 24 insertions(+), 30 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-14-17-29-34.gh-issue-125451.fmP3T9.rst
diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py
index 0e452883963c17..ff7c17efaab694 100644
--- a/Lib/concurrent/futures/process.py
+++ b/Lib/concurrent/futures/process.py
@@ -68,27 +68,31 @@
class _ThreadWakeup:
def __init__(self):
self._closed = False
+ self._lock = threading.Lock()
self._reader, self._writer = mp.Pipe(duplex=False)
def close(self):
- # Please note that we do not take the shutdown lock when
+ # Please note that we do not take the self._lock when
# calling clear() (to avoid deadlocking) so this method can
# only be called safely from the same thread as all calls to
- # clear() even if you hold the shutdown lock. Otherwise we
+ # clear() even if you hold the lock. Otherwise we
# might try to read from the closed pipe.
- if not self._closed:
- self._closed = True
- self._writer.close()
- self._reader.close()
+ with self._lock:
+ if not self._closed:
+ self._closed = True
+ self._writer.close()
+ self._reader.close()
def wakeup(self):
- if not self._closed:
- self._writer.send_bytes(b"")
+ with self._lock:
+ if not self._closed:
+ self._writer.send_bytes(b"")
def clear(self):
- if not self._closed:
- while self._reader.poll():
- self._reader.recv_bytes()
+ if self._closed:
+ raise RuntimeError('operation on closed _ThreadWakeup')
+ while self._reader.poll():
+ self._reader.recv_bytes()
def _python_exit():
@@ -167,10 +171,8 @@ def __init__(self, work_id, fn, args, kwargs):
class _SafeQueue(Queue):
"""Safe Queue set exception to the future object linked to a job"""
- def __init__(self, max_size=0, *, ctx, pending_work_items, shutdown_lock,
- thread_wakeup):
+ def __init__(self, max_size=0, *, ctx, pending_work_items, thread_wakeup):
self.pending_work_items = pending_work_items
- self.shutdown_lock = shutdown_lock
self.thread_wakeup = thread_wakeup
super().__init__(max_size, ctx=ctx)
@@ -179,8 +181,7 @@ def _on_queue_feeder_error(self, e, obj):
tb = format_exception(type(e), e, e.__traceback__)
e.__cause__ = _RemoteTraceback('\n"""\n{}"""'.format(''.join(tb)))
work_item = self.pending_work_items.pop(obj.work_id, None)
- with self.shutdown_lock:
- self.thread_wakeup.wakeup()
+ self.thread_wakeup.wakeup()
# work_item can be None if another process terminated. In this
# case, the executor_manager_thread fails all work_items
# with BrokenProcessPool
@@ -305,12 +306,10 @@ def __init__(self, executor):
# will wake up the queue management thread so that it can terminate
# if there is no pending work item.
def weakref_cb(_,
- thread_wakeup=self.thread_wakeup,
- shutdown_lock=self.shutdown_lock):
+ thread_wakeup=self.thread_wakeup):
mp.util.debug('Executor collected: triggering callback for'
' QueueManager wakeup')
- with shutdown_lock:
- thread_wakeup.wakeup()
+ thread_wakeup.wakeup()
self.executor_reference = weakref.ref(executor, weakref_cb)
@@ -438,11 +437,6 @@ def wait_result_broken_or_wakeup(self):
elif wakeup_reader in ready:
is_broken = False
- # No need to hold the _shutdown_lock here because:
- # 1. we're the only thread to use the wakeup reader
- # 2. we're also the only thread to call thread_wakeup.close()
- # 3. we want to avoid a possible deadlock when both reader and writer
- # would block (gh-105829)
self.thread_wakeup.clear()
return result_item, is_broken, cause
@@ -740,10 +734,9 @@ def __init__(self, max_workers=None, mp_context=None,
# as it could result in a deadlock if a worker process dies with the
# _result_queue write lock still acquired.
#
- # _shutdown_lock must be locked to access _ThreadWakeup.close() and
- # .wakeup(). Care must also be taken to not call clear or close from
- # more than one thread since _ThreadWakeup.clear() is not protected by
- # the _shutdown_lock
+ # Care must be taken to only call clear and close from the
+ # executor_manager_thread, since _ThreadWakeup.clear() is not protected
+ # by a lock.
self._executor_manager_thread_wakeup = _ThreadWakeup()
# Create communication channels for the executor
@@ -754,7 +747,6 @@ def __init__(self, max_workers=None, mp_context=None,
self._call_queue = _SafeQueue(
max_size=queue_size, ctx=self._mp_context,
pending_work_items=self._pending_work_items,
- shutdown_lock=self._shutdown_lock,
thread_wakeup=self._executor_manager_thread_wakeup)
# Killed worker processes can produce spurious "broken pipe"
# tracebacks in the queue's own worker thread. But we detect killed
diff --git a/Misc/NEWS.d/next/Library/2024-10-14-17-29-34.gh-issue-125451.fmP3T9.rst b/Misc/NEWS.d/next/Library/2024-10-14-17-29-34.gh-issue-125451.fmP3T9.rst
new file mode 100644
index 00000000000000..589988d4d6273f
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-14-17-29-34.gh-issue-125451.fmP3T9.rst
@@ -0,0 +1,2 @@
+Fix deadlock when :class:`concurrent.futures.ProcessPoolExecutor` shuts down
+concurrently with an error when feeding a job to a worker process.
From 42b8e52de41fc82f2985ddda9d40112b6e28a80c Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 17 Oct 2024 00:25:16 +0200
Subject: [PATCH 072/269] gh-125550: Enable py.exe to detect Store installs of
3.14 (GH-125551)
(cherry picked from commit 8e7b2a1161744c7d3d90966a65ed6ae1019a65cb)
Co-authored-by: Steve Dower
---
.../2024-10-15-21-28-43.gh-issue-125550.hmGWCP.rst | 2 ++
PC/launcher2.c | 9 ++++++---
2 files changed, 8 insertions(+), 3 deletions(-)
create mode 100644 Misc/NEWS.d/next/Windows/2024-10-15-21-28-43.gh-issue-125550.hmGWCP.rst
diff --git a/Misc/NEWS.d/next/Windows/2024-10-15-21-28-43.gh-issue-125550.hmGWCP.rst b/Misc/NEWS.d/next/Windows/2024-10-15-21-28-43.gh-issue-125550.hmGWCP.rst
new file mode 100644
index 00000000000000..c3ae00c74b3d91
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2024-10-15-21-28-43.gh-issue-125550.hmGWCP.rst
@@ -0,0 +1,2 @@
+Enable the :ref:`launcher` to detect Python 3.14 installs from the Windows
+Store.
diff --git a/PC/launcher2.c b/PC/launcher2.c
index f331aab3f51e56..7a78ed1655a9ee 100644
--- a/PC/launcher2.c
+++ b/PC/launcher2.c
@@ -1938,6 +1938,7 @@ struct AppxSearchInfo {
struct AppxSearchInfo APPX_SEARCH[] = {
// Releases made through the Store
+ { L"PythonSoftwareFoundation.Python.3.14_qbz5n2kfra8p0", L"3.14", 10 },
{ L"PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0", L"3.13", 10 },
{ L"PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0", L"3.12", 10 },
{ L"PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0", L"3.11", 10 },
@@ -1946,8 +1947,9 @@ struct AppxSearchInfo APPX_SEARCH[] = {
{ L"PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0", L"3.8", 10 },
// Side-loadable releases. Note that the publisher ID changes whenever we
- // renew our code-signing certificate, so the newer ID has a higher
- // priority (lower sortKey)
+ // change our code signing certificate subject, so the newer IDs have higher
+ // priorities (lower sortKey)
+ { L"PythonSoftwareFoundation.Python.3.14_3847v3x7pw1km", L"3.14", 11 },
{ L"PythonSoftwareFoundation.Python.3.13_3847v3x7pw1km", L"3.13", 11 },
{ L"PythonSoftwareFoundation.Python.3.12_3847v3x7pw1km", L"3.12", 11 },
{ L"PythonSoftwareFoundation.Python.3.11_3847v3x7pw1km", L"3.11", 11 },
@@ -2030,7 +2032,8 @@ struct StoreSearchInfo {
struct StoreSearchInfo STORE_SEARCH[] = {
- { L"3", /* 3.12 */ L"9NCVDN91XZQP" },
+ { L"3", /* 3.13 */ L"9PNRBTZXMB4Z" },
+ { L"3.14", L"9NTRHQCBBPR8" },
{ L"3.13", L"9PNRBTZXMB4Z" },
{ L"3.12", L"9NCVDN91XZQP" },
{ L"3.11", L"9NRWMJP3717K" },
From 32d457941e8b39c0300e02632f932d1556b7beee Mon Sep 17 00:00:00 2001
From: Thomas Grainger
Date: Thu, 17 Oct 2024 05:45:59 +0100
Subject: [PATCH 073/269] [3.12] gh-124958: fix asyncio.TaskGroup and _PyFuture
refcycles (#124959) (#125466)
gh-124958: fix asyncio.TaskGroup and _PyFuture refcycles (#124959)
---
Lib/asyncio/futures.py | 6 +-
Lib/asyncio/taskgroups.py | 41 +++++++--
Lib/test/test_asyncio/test_futures.py | 22 +++++
Lib/test/test_asyncio/test_taskgroups.py | 92 ++++++++++++++++++-
...-10-04-08-46-00.gh-issue-124958.rea9-x.rst | 1 +
5 files changed, 147 insertions(+), 15 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-04-08-46-00.gh-issue-124958.rea9-x.rst
diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py
index fd486f02c67c8e..0c530bbdbcf2d8 100644
--- a/Lib/asyncio/futures.py
+++ b/Lib/asyncio/futures.py
@@ -194,8 +194,7 @@ def result(self):
the future is done and has an exception set, this exception is raised.
"""
if self._state == _CANCELLED:
- exc = self._make_cancelled_error()
- raise exc
+ raise self._make_cancelled_error()
if self._state != _FINISHED:
raise exceptions.InvalidStateError('Result is not ready.')
self.__log_traceback = False
@@ -212,8 +211,7 @@ def exception(self):
InvalidStateError.
"""
if self._state == _CANCELLED:
- exc = self._make_cancelled_error()
- raise exc
+ raise self._make_cancelled_error()
if self._state != _FINISHED:
raise exceptions.InvalidStateError('Exception is not set.')
self.__log_traceback = False
diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py
index d264e51f1fd4e6..aada3ffa8e0f29 100644
--- a/Lib/asyncio/taskgroups.py
+++ b/Lib/asyncio/taskgroups.py
@@ -66,6 +66,20 @@ async def __aenter__(self):
return self
async def __aexit__(self, et, exc, tb):
+ tb = None
+ try:
+ return await self._aexit(et, exc)
+ finally:
+ # Exceptions are heavy objects that can have object
+ # cycles (bad for GC); let's not keep a reference to
+ # a bunch of them. It would be nicer to use a try/finally
+ # in __aexit__ directly but that introduced some diff noise
+ self._parent_task = None
+ self._errors = None
+ self._base_error = None
+ exc = None
+
+ async def _aexit(self, et, exc):
self._exiting = True
if (exc is not None and
@@ -126,25 +140,34 @@ async def __aexit__(self, et, exc, tb):
assert not self._tasks
if self._base_error is not None:
- raise self._base_error
+ try:
+ raise self._base_error
+ finally:
+ exc = None
# Propagate CancelledError if there is one, except if there
# are other errors -- those have priority.
- if propagate_cancellation_error and not self._errors:
- raise propagate_cancellation_error
+ try:
+ if propagate_cancellation_error and not self._errors:
+ try:
+ raise propagate_cancellation_error
+ finally:
+ exc = None
+ finally:
+ propagate_cancellation_error = None
if et is not None and et is not exceptions.CancelledError:
self._errors.append(exc)
if self._errors:
- # Exceptions are heavy objects that can have object
- # cycles (bad for GC); let's not keep a reference to
- # a bunch of them.
try:
- me = BaseExceptionGroup('unhandled errors in a TaskGroup', self._errors)
- raise me from None
+ raise BaseExceptionGroup(
+ 'unhandled errors in a TaskGroup',
+ self._errors,
+ ) from None
finally:
- self._errors = None
+ exc = None
+
def create_task(self, coro, *, name=None, context=None):
"""Create a new task in this group and return it.
diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py
index 47daa0e9f410a8..050d33f4fab3ed 100644
--- a/Lib/test/test_asyncio/test_futures.py
+++ b/Lib/test/test_asyncio/test_futures.py
@@ -640,6 +640,28 @@ def __del__(self):
fut = self._new_future(loop=self.loop)
fut.set_result(Evil())
+ def test_future_cancelled_result_refcycles(self):
+ f = self._new_future(loop=self.loop)
+ f.cancel()
+ exc = None
+ try:
+ f.result()
+ except asyncio.CancelledError as e:
+ exc = e
+ self.assertIsNotNone(exc)
+ self.assertListEqual(gc.get_referrers(exc), [])
+
+ def test_future_cancelled_exception_refcycles(self):
+ f = self._new_future(loop=self.loop)
+ f.cancel()
+ exc = None
+ try:
+ f.exception()
+ except asyncio.CancelledError as e:
+ exc = e
+ self.assertIsNotNone(exc)
+ self.assertListEqual(gc.get_referrers(exc), [])
+
@unittest.skipUnless(hasattr(futures, '_CFuture'),
'requires the C _asyncio module')
diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py
index 7a18362b54e469..236bfaaccf88fa 100644
--- a/Lib/test/test_asyncio/test_taskgroups.py
+++ b/Lib/test/test_asyncio/test_taskgroups.py
@@ -1,7 +1,7 @@
# Adapted with permission from the EdgeDB project;
# license: PSFL.
-
+import gc
import asyncio
import contextvars
import contextlib
@@ -10,7 +10,6 @@
from test.test_asyncio.utils import await_without_task
-
# To prevent a warning "test altered the execution environment"
def tearDownModule():
asyncio.set_event_loop_policy(None)
@@ -824,6 +823,95 @@ async def test_taskgroup_without_parent_task(self):
# We still have to await coro to avoid a warning
await coro
+ async def test_exception_refcycles_direct(self):
+ """Test that TaskGroup doesn't keep a reference to the raised ExceptionGroup"""
+ tg = asyncio.TaskGroup()
+ exc = None
+
+ class _Done(Exception):
+ pass
+
+ try:
+ async with tg:
+ raise _Done
+ except ExceptionGroup as e:
+ exc = e
+
+ self.assertIsNotNone(exc)
+ self.assertListEqual(gc.get_referrers(exc), [])
+
+
+ async def test_exception_refcycles_errors(self):
+ """Test that TaskGroup deletes self._errors, and __aexit__ args"""
+ tg = asyncio.TaskGroup()
+ exc = None
+
+ class _Done(Exception):
+ pass
+
+ try:
+ async with tg:
+ raise _Done
+ except* _Done as excs:
+ exc = excs.exceptions[0]
+
+ self.assertIsInstance(exc, _Done)
+ self.assertListEqual(gc.get_referrers(exc), [])
+
+
+ async def test_exception_refcycles_parent_task(self):
+ """Test that TaskGroup deletes self._parent_task"""
+ tg = asyncio.TaskGroup()
+ exc = None
+
+ class _Done(Exception):
+ pass
+
+ async def coro_fn():
+ async with tg:
+ raise _Done
+
+ try:
+ async with asyncio.TaskGroup() as tg2:
+ tg2.create_task(coro_fn())
+ except* _Done as excs:
+ exc = excs.exceptions[0].exceptions[0]
+
+ self.assertIsInstance(exc, _Done)
+ self.assertListEqual(gc.get_referrers(exc), [])
+
+ async def test_exception_refcycles_propagate_cancellation_error(self):
+ """Test that TaskGroup deletes propagate_cancellation_error"""
+ tg = asyncio.TaskGroup()
+ exc = None
+
+ try:
+ async with asyncio.timeout(-1):
+ async with tg:
+ await asyncio.sleep(0)
+ except TimeoutError as e:
+ exc = e.__cause__
+
+ self.assertIsInstance(exc, asyncio.CancelledError)
+ self.assertListEqual(gc.get_referrers(exc), [])
+
+ async def test_exception_refcycles_base_error(self):
+ """Test that TaskGroup deletes self._base_error"""
+ class MyKeyboardInterrupt(KeyboardInterrupt):
+ pass
+
+ tg = asyncio.TaskGroup()
+ exc = None
+
+ try:
+ async with tg:
+ raise MyKeyboardInterrupt
+ except MyKeyboardInterrupt as e:
+ exc = e
+
+ self.assertIsNotNone(exc)
+ self.assertListEqual(gc.get_referrers(exc), [])
+
if __name__ == "__main__":
unittest.main()
diff --git a/Misc/NEWS.d/next/Library/2024-10-04-08-46-00.gh-issue-124958.rea9-x.rst b/Misc/NEWS.d/next/Library/2024-10-04-08-46-00.gh-issue-124958.rea9-x.rst
new file mode 100644
index 00000000000000..534d5bb8c898da
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-04-08-46-00.gh-issue-124958.rea9-x.rst
@@ -0,0 +1 @@
+Fix refcycles in exceptions raised from :class:`asyncio.TaskGroup` and the python implementation of :class:`asyncio.Future`
From fb1b92b3279ba5741bc9dc9809b10e995c3dab4d Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 17 Oct 2024 14:42:55 +0200
Subject: [PATCH 074/269] [3.12] gh-95836: Add custom type converter examples
to argparse tutorial (GH-125376) (GH-125642)
(cherry picked from commit dbcc5ac4709dfd8dfaf323d51f135f2218d14068)
Co-authored-by: Savannah Ostrowski
---
Doc/howto/argparse.rst | 47 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 47 insertions(+)
diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst
index ae5bab90bf8131..ac2a0465b2283b 100644
--- a/Doc/howto/argparse.rst
+++ b/Doc/howto/argparse.rst
@@ -841,6 +841,53 @@ translated messages.
To translate your own strings in the :mod:`argparse` output, use :mod:`gettext`.
+Custom type converters
+======================
+
+The :mod:`argparse` module allows you to specify custom type converters for
+your command-line arguments. This allows you to modify user input before it's
+stored in the :class:`argparse.Namespace`. This can be useful when you need to
+pre-process the input before it is used in your program.
+
+When using a custom type converter, you can use any callable that takes a
+single string argument (the argument value) and returns the converted value.
+However, if you need to handle more complex scenarios, you can use a custom
+action class with the **action** parameter instead.
+
+For example, let's say you want to handle arguments with different prefixes and
+process them accordingly::
+
+ import argparse
+
+ parser = argparse.ArgumentParser(prefix_chars='-+')
+
+ parser.add_argument('-a', metavar='', action='append',
+ type=lambda x: ('-', x))
+ parser.add_argument('+a', metavar='', action='append',
+ type=lambda x: ('+', x))
+
+ args = parser.parse_args()
+ print(args)
+
+Output:
+
+.. code-block:: shell-session
+
+ $ python prog.py -a value1 +a value2
+ Namespace(a=[('-', 'value1'), ('+', 'value2')])
+
+In this example, we:
+
+* Created a parser with custom prefix characters using the ``prefix_chars``
+ parameter.
+
+* Defined two arguments, ``-a`` and ``+a``, which used the ``type`` parameter to
+ create custom type converters to store the value in a tuple with the prefix.
+
+Without the custom type converters, the arguments would have treated the ``-a``
+and ``+a`` as the same argument, which would have been undesirable. By using custom
+type converters, we were able to differentiate between the two arguments.
+
Conclusion
==========
From 8f72f33d69dbf74728c713356ec4d160fd7d79f5 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 17 Oct 2024 18:01:55 +0200
Subject: [PATCH 075/269] [3.12] gh-125625: Check for `py -3.13` in
PCbuild/find_python.bat (GH-125649)
(cherry picked from commit 0cb20f2e7e867d5c34fc17dd5b8e51e8b0020bb3)
Co-authored-by: Wulian
---
PCbuild/find_python.bat | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat
index d8c7cec15ef5fe..0af367a3efafad 100644
--- a/PCbuild/find_python.bat
+++ b/PCbuild/find_python.bat
@@ -44,7 +44,7 @@
@rem If py.exe finds a recent enough version, use that one
@rem It is fine to add new versions to this list when they have released,
@rem but we do not use prerelease builds here.
-@for %%p in (3.12 3.11 3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found
+@for %%p in (3.13 3.12 3.11 3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found
@if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%"
@set _Py_NUGET=%NUGET%
From dc0a1763218f251d286912a6736b27c9ea887668 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 17 Oct 2024 18:59:01 +0200
Subject: [PATCH 076/269] [3.12] gh-113570: reprlib.repr does not use builtin
__repr__ for reshadowed builtins (GH-113577) (GH-125655)
(cherry picked from commit 04d6dd23e2d8a3132772cf7ce928676e26313585)
Co-authored-by: George Pittock <66332098+georgepittock@users.noreply.github.com>
---
Lib/reprlib.py | 31 ++++++++++---
Lib/test/test_reprlib.py | 44 +++++++++++++++++++
...-12-30-00-21-45.gh-issue-113570._XQgsW.rst | 1 +
3 files changed, 71 insertions(+), 5 deletions(-)
create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-12-30-00-21-45.gh-issue-113570._XQgsW.rst
diff --git a/Lib/reprlib.py b/Lib/reprlib.py
index a7b37630a4edb9..85c1b94a0ea6a9 100644
--- a/Lib/reprlib.py
+++ b/Lib/reprlib.py
@@ -35,6 +35,17 @@ def wrapper(self):
return decorating_function
class Repr:
+ _lookup = {
+ 'tuple': 'builtins',
+ 'list': 'builtins',
+ 'array': 'array',
+ 'set': 'builtins',
+ 'frozenset': 'builtins',
+ 'deque': 'collections',
+ 'dict': 'builtins',
+ 'str': 'builtins',
+ 'int': 'builtins'
+ }
def __init__(
self, *, maxlevel=6, maxtuple=6, maxlist=6, maxarray=5, maxdict=4,
@@ -59,14 +70,24 @@ def repr(self, x):
return self.repr1(x, self.maxlevel)
def repr1(self, x, level):
- typename = type(x).__name__
+ cls = type(x)
+ typename = cls.__name__
+
if ' ' in typename:
parts = typename.split()
typename = '_'.join(parts)
- if hasattr(self, 'repr_' + typename):
- return getattr(self, 'repr_' + typename)(x, level)
- else:
- return self.repr_instance(x, level)
+
+ method = getattr(self, 'repr_' + typename, None)
+ if method:
+ # not defined in this class
+ if typename not in self._lookup:
+ return method(x, level)
+ module = getattr(cls, '__module__', None)
+ # defined in this class and is the module intended
+ if module == self._lookup[typename]:
+ return method(x, level)
+
+ return self.repr_instance(x, level)
def _join(self, pieces, level):
if self.indent is None:
diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py
index 4a896db2002047..e2a67ef344d920 100644
--- a/Lib/test/test_reprlib.py
+++ b/Lib/test/test_reprlib.py
@@ -580,6 +580,50 @@ def test_invalid_indent(self):
with self.assertRaisesRegex(expected_error, expected_msg):
r.repr(test_object)
+ def test_shadowed_stdlib_array(self):
+ # Issue #113570: repr() should not be fooled by an array
+ class array:
+ def __repr__(self):
+ return "not array.array"
+
+ self.assertEqual(r(array()), "not array.array")
+
+ def test_shadowed_builtin(self):
+ # Issue #113570: repr() should not be fooled
+ # by a shadowed builtin function
+ class list:
+ def __repr__(self):
+ return "not builtins.list"
+
+ self.assertEqual(r(list()), "not builtins.list")
+
+ def test_custom_repr(self):
+ class MyRepr(Repr):
+
+ def repr_TextIOWrapper(self, obj, level):
+ if obj.name in {'', '', ''}:
+ return obj.name
+ return repr(obj)
+
+ aRepr = MyRepr()
+ self.assertEqual(aRepr.repr(sys.stdin), "")
+
+ def test_custom_repr_class_with_spaces(self):
+ class TypeWithSpaces:
+ pass
+
+ t = TypeWithSpaces()
+ type(t).__name__ = "type with spaces"
+ self.assertEqual(type(t).__name__, "type with spaces")
+
+ class MyRepr(Repr):
+ def repr_type_with_spaces(self, obj, level):
+ return "Type With Spaces"
+
+
+ aRepr = MyRepr()
+ self.assertEqual(aRepr.repr(t), "Type With Spaces")
+
def write_file(path, text):
with open(path, 'w', encoding='ASCII') as fp:
fp.write(text)
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-30-00-21-45.gh-issue-113570._XQgsW.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-30-00-21-45.gh-issue-113570._XQgsW.rst
new file mode 100644
index 00000000000000..6e0f0afe05369b
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-30-00-21-45.gh-issue-113570._XQgsW.rst
@@ -0,0 +1 @@
+Fixed a bug in ``reprlib.repr`` where it incorrectly called the repr method on shadowed Python built-in types.
From 5e62d9bd54b5657b7e50da9966e4cbc6ed3b520b Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 17 Oct 2024 19:09:05 +0200
Subject: [PATCH 077/269] [3.12] [3.13] gh-123370: Fix the canvas not clearing
after running turtledemo.clock (gh-123457) (GH-125653) (#125656)
Rewriting the day and date every tick somehow prevented them from being removed either by clicking STOP or loading another example. The solution is to rewrite them only when they change.
(cherry picked from commit c124577ebe915a00de4033c0f7fa7c47621d79e0)
(cherry picked from commit 30d7e9e721e8201bc7fb23f08a7d88d51eed827d)
Co-authored-by: Terry Jan Reedy
Co-authored-by: Wulian
---
Lib/turtledemo/clock.py | 33 +++++++++++--------
...-08-28-19-27-35.gh-issue-123370.SPZ9Ux.rst | 1 +
2 files changed, 21 insertions(+), 13 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-08-28-19-27-35.gh-issue-123370.SPZ9Ux.rst
diff --git a/Lib/turtledemo/clock.py b/Lib/turtledemo/clock.py
index 9f8585bd11e053..318f126006333f 100755
--- a/Lib/turtledemo/clock.py
+++ b/Lib/turtledemo/clock.py
@@ -1,8 +1,7 @@
#!/usr/bin/env python3
-# -*- coding: cp1252 -*-
""" turtle-example-suite:
- tdemo_clock.py
+ turtledemo/clock.py
Enhanced clock-program, showing date
and time
@@ -13,6 +12,9 @@
from turtle import *
from datetime import datetime
+dtfont = "TkFixedFont", 14, "bold"
+current_day = None
+
def jump(distanz, winkel=0):
penup()
right(winkel)
@@ -53,11 +55,23 @@ def clockface(radius):
jump(-radius)
rt(6)
+def display_date_time():
+ global current_day
+ writer.clear()
+ now = datetime.now()
+ current_day = now.day
+ writer.home()
+ writer.forward(distance=65)
+ writer.write(wochentag(now), align="center", font=dtfont)
+ writer.back(distance=150)
+ writer.write(datum(now), align="center", font=dtfont)
+ writer.forward(distance=85)
+
def setup():
global second_hand, minute_hand, hour_hand, writer
mode("logo")
make_hand_shape("second_hand", 125, 25)
- make_hand_shape("minute_hand", 130, 25)
+ make_hand_shape("minute_hand", 115, 25)
make_hand_shape("hour_hand", 90, 25)
clockface(160)
second_hand = Turtle()
@@ -75,10 +89,10 @@ def setup():
hand.speed(0)
ht()
writer = Turtle()
- #writer.mode("logo")
writer.ht()
writer.pu()
writer.bk(85)
+ display_date_time()
def wochentag(t):
wochentag = ["Monday", "Tuesday", "Wednesday",
@@ -100,18 +114,11 @@ def tick():
stunde = t.hour + minute/60.0
try:
tracer(False) # Terminator can occur here
- writer.clear()
- writer.home()
- writer.forward(65)
- writer.write(wochentag(t),
- align="center", font=("Courier", 14, "bold"))
- writer.back(150)
- writer.write(datum(t),
- align="center", font=("Courier", 14, "bold"))
- writer.forward(85)
second_hand.setheading(6*sekunde) # or here
minute_hand.setheading(6*minute)
hour_hand.setheading(30*stunde)
+ if t.day != current_day:
+ display_date_time()
tracer(True)
ontimer(tick, 100)
except Terminator:
diff --git a/Misc/NEWS.d/next/Library/2024-08-28-19-27-35.gh-issue-123370.SPZ9Ux.rst b/Misc/NEWS.d/next/Library/2024-08-28-19-27-35.gh-issue-123370.SPZ9Ux.rst
new file mode 100644
index 00000000000000..1fd5cc54eaf3e7
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-08-28-19-27-35.gh-issue-123370.SPZ9Ux.rst
@@ -0,0 +1 @@
+Fix the canvas not clearing after running turtledemo clock.
From 232b303e4ca47892f544294bf42e31dc34f0ec72 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 17 Oct 2024 22:04:48 +0200
Subject: [PATCH 078/269] [3.12] gh-52551: Fix encoding issues in strftime()
(GH-125193) (GH-125657) (GH-125661)
Fix time.strftime(), the strftime() method and formatting of the
datetime classes datetime, date and time.
* Characters not encodable in the current locale are now acceptable in
the format string.
* Surrogate pairs and sequence of surrogatescape-encoded bytes are no
longer recombinated.
* Embedded null character no longer terminates the format string.
This fixes also gh-78662 and gh-124531.
(cherry picked from commit 08ccbb9b3f5e20a7a0c4cf9995e172b59fb6067b)
(cherry picked from commit ad3eac1963a5f195ef9b2c1dbb5e44fa3cce4c72)
Co-authored-by: Serhiy Storchaka
---
Lib/test/datetimetester.py | 63 ++++-
Lib/test/test_time.py | 29 ++-
...4-10-09-17-07-33.gh-issue-52551.PBakSY.rst | 8 +
Modules/_datetimemodule.c | 169 +++++--------
Modules/timemodule.c | 235 ++++++++++--------
5 files changed, 293 insertions(+), 211 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-09-17-07-33.gh-issue-52551.PBakSY.rst
diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py
index 0528e0701fa982..903a43aad81994 100644
--- a/Lib/test/datetimetester.py
+++ b/Lib/test/datetimetester.py
@@ -2816,11 +2816,32 @@ def test_more_strftime(self):
self.assertEqual(t.strftime("%z"), "-0200" + z)
self.assertEqual(t.strftime("%:z"), "-02:00:" + z)
- # bpo-34482: Check that surrogates don't cause a crash.
- try:
- t.strftime('%y\ud800%m %H\ud800%M')
- except UnicodeEncodeError:
- pass
+ def test_strftime_special(self):
+ t = self.theclass(2004, 12, 31, 6, 22, 33, 47)
+ s1 = t.strftime('%c')
+ s2 = t.strftime('%B')
+ # gh-52551, gh-78662: Unicode strings should pass through strftime,
+ # independently from locale.
+ self.assertEqual(t.strftime('\U0001f40d'), '\U0001f40d')
+ self.assertEqual(t.strftime('\U0001f4bb%c\U0001f40d%B'), f'\U0001f4bb{s1}\U0001f40d{s2}')
+ self.assertEqual(t.strftime('%c\U0001f4bb%B\U0001f40d'), f'{s1}\U0001f4bb{s2}\U0001f40d')
+ # Lone surrogates should pass through.
+ self.assertEqual(t.strftime('\ud83d'), '\ud83d')
+ self.assertEqual(t.strftime('\udc0d'), '\udc0d')
+ self.assertEqual(t.strftime('\ud83d%c\udc0d%B'), f'\ud83d{s1}\udc0d{s2}')
+ self.assertEqual(t.strftime('%c\ud83d%B\udc0d'), f'{s1}\ud83d{s2}\udc0d')
+ self.assertEqual(t.strftime('%c\udc0d%B\ud83d'), f'{s1}\udc0d{s2}\ud83d')
+ # Surrogate pairs should not recombine.
+ self.assertEqual(t.strftime('\ud83d\udc0d'), '\ud83d\udc0d')
+ self.assertEqual(t.strftime('%c\ud83d\udc0d%B'), f'{s1}\ud83d\udc0d{s2}')
+ # Surrogate-escaped bytes should not recombine.
+ self.assertEqual(t.strftime('\udcf0\udc9f\udc90\udc8d'), '\udcf0\udc9f\udc90\udc8d')
+ self.assertEqual(t.strftime('%c\udcf0\udc9f\udc90\udc8d%B'), f'{s1}\udcf0\udc9f\udc90\udc8d{s2}')
+ # gh-124531: The null character should not terminate the format string.
+ self.assertEqual(t.strftime('\0'), '\0')
+ self.assertEqual(t.strftime('\0'*1000), '\0'*1000)
+ self.assertEqual(t.strftime('\0%c\0%B'), f'\0{s1}\0{s2}')
+ self.assertEqual(t.strftime('%c\0%B\0'), f'{s1}\0{s2}\0')
def test_extract(self):
dt = self.theclass(2002, 3, 4, 18, 45, 3, 1234)
@@ -3573,6 +3594,33 @@ def test_strftime(self):
# gh-85432: The parameter was named "fmt" in the pure-Python impl.
t.strftime(format="%f")
+ def test_strftime_special(self):
+ t = self.theclass(1, 2, 3, 4)
+ s1 = t.strftime('%I%p%Z')
+ s2 = t.strftime('%X')
+ # gh-52551, gh-78662: Unicode strings should pass through strftime,
+ # independently from locale.
+ self.assertEqual(t.strftime('\U0001f40d'), '\U0001f40d')
+ self.assertEqual(t.strftime('\U0001f4bb%I%p%Z\U0001f40d%X'), f'\U0001f4bb{s1}\U0001f40d{s2}')
+ self.assertEqual(t.strftime('%I%p%Z\U0001f4bb%X\U0001f40d'), f'{s1}\U0001f4bb{s2}\U0001f40d')
+ # Lone surrogates should pass through.
+ self.assertEqual(t.strftime('\ud83d'), '\ud83d')
+ self.assertEqual(t.strftime('\udc0d'), '\udc0d')
+ self.assertEqual(t.strftime('\ud83d%I%p%Z\udc0d%X'), f'\ud83d{s1}\udc0d{s2}')
+ self.assertEqual(t.strftime('%I%p%Z\ud83d%X\udc0d'), f'{s1}\ud83d{s2}\udc0d')
+ self.assertEqual(t.strftime('%I%p%Z\udc0d%X\ud83d'), f'{s1}\udc0d{s2}\ud83d')
+ # Surrogate pairs should not recombine.
+ self.assertEqual(t.strftime('\ud83d\udc0d'), '\ud83d\udc0d')
+ self.assertEqual(t.strftime('%I%p%Z\ud83d\udc0d%X'), f'{s1}\ud83d\udc0d{s2}')
+ # Surrogate-escaped bytes should not recombine.
+ self.assertEqual(t.strftime('\udcf0\udc9f\udc90\udc8d'), '\udcf0\udc9f\udc90\udc8d')
+ self.assertEqual(t.strftime('%I%p%Z\udcf0\udc9f\udc90\udc8d%X'), f'{s1}\udcf0\udc9f\udc90\udc8d{s2}')
+ # gh-124531: The null character should not terminate the format string.
+ self.assertEqual(t.strftime('\0'), '\0')
+ self.assertEqual(t.strftime('\0'*1000), '\0'*1000)
+ self.assertEqual(t.strftime('\0%I%p%Z\0%X'), f'\0{s1}\0{s2}')
+ self.assertEqual(t.strftime('%I%p%Z\0%X\0'), f'{s1}\0{s2}\0')
+
def test_format(self):
t = self.theclass(1, 2, 3, 4)
self.assertEqual(t.__format__(''), str(t))
@@ -4002,9 +4050,8 @@ def tzname(self, dt): return self.tz
self.assertRaises(TypeError, t.strftime, "%Z")
# Issue #6697:
- if '_Fast' in self.__class__.__name__:
- Badtzname.tz = '\ud800'
- self.assertRaises(ValueError, t.strftime, "%Z")
+ Badtzname.tz = '\ud800'
+ self.assertEqual(t.strftime("%Z"), '\ud800')
def test_hash_edge_cases(self):
# Offsets that overflow a basic time.
diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py
index ca40415f0cdd76..b020787dacac1c 100644
--- a/Lib/test/test_time.py
+++ b/Lib/test/test_time.py
@@ -177,8 +177,33 @@ def test_strftime(self):
self.fail('conversion specifier: %r failed.' % format)
self.assertRaises(TypeError, time.strftime, b'%S', tt)
- # embedded null character
- self.assertRaises(ValueError, time.strftime, '%S\0', tt)
+
+ def test_strftime_special(self):
+ tt = time.gmtime(self.t)
+ s1 = time.strftime('%c', tt)
+ s2 = time.strftime('%B', tt)
+ # gh-52551, gh-78662: Unicode strings should pass through strftime,
+ # independently from locale.
+ self.assertEqual(time.strftime('\U0001f40d', tt), '\U0001f40d')
+ self.assertEqual(time.strftime('\U0001f4bb%c\U0001f40d%B', tt), f'\U0001f4bb{s1}\U0001f40d{s2}')
+ self.assertEqual(time.strftime('%c\U0001f4bb%B\U0001f40d', tt), f'{s1}\U0001f4bb{s2}\U0001f40d')
+ # Lone surrogates should pass through.
+ self.assertEqual(time.strftime('\ud83d', tt), '\ud83d')
+ self.assertEqual(time.strftime('\udc0d', tt), '\udc0d')
+ self.assertEqual(time.strftime('\ud83d%c\udc0d%B', tt), f'\ud83d{s1}\udc0d{s2}')
+ self.assertEqual(time.strftime('%c\ud83d%B\udc0d', tt), f'{s1}\ud83d{s2}\udc0d')
+ self.assertEqual(time.strftime('%c\udc0d%B\ud83d', tt), f'{s1}\udc0d{s2}\ud83d')
+ # Surrogate pairs should not recombine.
+ self.assertEqual(time.strftime('\ud83d\udc0d', tt), '\ud83d\udc0d')
+ self.assertEqual(time.strftime('%c\ud83d\udc0d%B', tt), f'{s1}\ud83d\udc0d{s2}')
+ # Surrogate-escaped bytes should not recombine.
+ self.assertEqual(time.strftime('\udcf0\udc9f\udc90\udc8d', tt), '\udcf0\udc9f\udc90\udc8d')
+ self.assertEqual(time.strftime('%c\udcf0\udc9f\udc90\udc8d%B', tt), f'{s1}\udcf0\udc9f\udc90\udc8d{s2}')
+ # gh-124531: The null character should not terminate the format string.
+ self.assertEqual(time.strftime('\0', tt), '\0')
+ self.assertEqual(time.strftime('\0'*1000, tt), '\0'*1000)
+ self.assertEqual(time.strftime('\0%c\0%B', tt), f'\0{s1}\0{s2}')
+ self.assertEqual(time.strftime('%c\0%B\0', tt), f'{s1}\0{s2}\0')
def _bounds_checking(self, func):
# Make sure that strftime() checks the bounds of the various parts
diff --git a/Misc/NEWS.d/next/Library/2024-10-09-17-07-33.gh-issue-52551.PBakSY.rst b/Misc/NEWS.d/next/Library/2024-10-09-17-07-33.gh-issue-52551.PBakSY.rst
new file mode 100644
index 00000000000000..edc9ac5bb23117
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-09-17-07-33.gh-issue-52551.PBakSY.rst
@@ -0,0 +1,8 @@
+Fix encoding issues in :func:`time.strftime`, the
+:meth:`~datetime.datetime.strftime` method of the :mod:`datetime` classes
+:class:`~datetime.datetime`, :class:`~datetime.date` and
+:class:`~datetime.time` and formatting of these classes. Characters not
+encodable in the current locale are now acceptable in the format string.
+Surrogate pairs and sequence of surrogatescape-encoded bytes are no longer
+recombinated. Embedded null character no longer terminates the format
+string.
diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c
index 5a062b9c8c0e2c..8535811a61b9e6 100644
--- a/Modules/_datetimemodule.c
+++ b/Modules/_datetimemodule.c
@@ -1501,7 +1501,7 @@ make_somezreplacement(PyObject *object, char *sep, PyObject *tzinfoarg)
PyObject *tzinfo = get_tzinfo_member(object);
if (tzinfo == Py_None || tzinfo == NULL) {
- return PyBytes_FromStringAndSize(NULL, 0);
+ return PyUnicode_FromStringAndSize(NULL, 0);
}
assert(tzinfoarg != NULL);
@@ -1512,7 +1512,7 @@ make_somezreplacement(PyObject *object, char *sep, PyObject *tzinfoarg)
tzinfoarg) < 0)
return NULL;
- return PyBytes_FromStringAndSize(buf, strlen(buf));
+ return PyUnicode_FromString(buf);
}
static PyObject *
@@ -1569,7 +1569,7 @@ make_freplacement(PyObject *object)
else
sprintf(freplacement, "%06d", 0);
- return PyBytes_FromStringAndSize(freplacement, strlen(freplacement));
+ return PyUnicode_FromString(freplacement);
}
/* I sure don't want to reproduce the strftime code from the time module,
@@ -1590,79 +1590,60 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple,
PyObject *Zreplacement = NULL; /* py string, replacement for %Z */
PyObject *freplacement = NULL; /* py string, replacement for %f */
- const char *pin; /* pointer to next char in input format */
- Py_ssize_t flen; /* length of input format */
- char ch; /* next char in input format */
-
- PyObject *newfmt = NULL; /* py string, the output format */
- char *pnew; /* pointer to available byte in output format */
- size_t totalnew; /* number bytes total in output format buffer,
- exclusive of trailing \0 */
- size_t usednew; /* number bytes used so far in output format buffer */
-
- const char *ptoappend; /* ptr to string to append to output buffer */
- Py_ssize_t ntoappend; /* # of bytes to append to output buffer */
-
assert(object && format && timetuple);
assert(PyUnicode_Check(format));
- /* Convert the input format to a C string and size */
- pin = PyUnicode_AsUTF8AndSize(format, &flen);
- if (!pin)
+
+ PyObject *strftime = _PyImport_GetModuleAttrString("time", "strftime");
+ if (strftime == NULL) {
return NULL;
+ }
/* Scan the input format, looking for %z/%Z/%f escapes, building
* a new format. Since computing the replacements for those codes
* is expensive, don't unless they're actually used.
*/
- if (flen > INT_MAX - 1) {
- PyErr_NoMemory();
- goto Done;
- }
- totalnew = flen + 1; /* realistic if no %z/%Z */
- newfmt = PyBytes_FromStringAndSize(NULL, totalnew);
- if (newfmt == NULL) goto Done;
- pnew = PyBytes_AsString(newfmt);
- usednew = 0;
-
- while ((ch = *pin++) != '\0') {
- if (ch != '%') {
- ptoappend = pin - 1;
- ntoappend = 1;
+ _PyUnicodeWriter writer;
+ _PyUnicodeWriter_Init(&writer);
+ writer.overallocate = 1;
+
+ Py_ssize_t flen = PyUnicode_GET_LENGTH(format);
+ Py_ssize_t i = 0;
+ Py_ssize_t start = 0;
+ Py_ssize_t end = 0;
+ while (i != flen) {
+ i = PyUnicode_FindChar(format, '%', i, flen, 1);
+ if (i < 0) {
+ assert(!PyErr_Occurred());
+ break;
}
- else if ((ch = *pin++) == '\0') {
- /* Null byte follows %, copy only '%'.
- *
- * Back the pin up one char so that we catch the null check
- * the next time through the loop.*/
- pin--;
- ptoappend = pin - 1;
- ntoappend = 1;
+ end = i;
+ i++;
+ if (i == flen) {
+ break;
}
+ Py_UCS4 ch = PyUnicode_READ_CHAR(format, i);
+ i++;
/* A % has been seen and ch is the character after it. */
- else if (ch == 'z') {
+ PyObject *replacement = NULL;
+ if (ch == 'z') {
/* %z -> +HHMM */
if (zreplacement == NULL) {
zreplacement = make_somezreplacement(object, "", tzinfoarg);
if (zreplacement == NULL)
- goto Done;
+ goto Error;
}
- assert(zreplacement != NULL);
- assert(PyBytes_Check(zreplacement));
- ptoappend = PyBytes_AS_STRING(zreplacement);
- ntoappend = PyBytes_GET_SIZE(zreplacement);
+ replacement = zreplacement;
}
- else if (ch == ':' && *pin == 'z' && pin++) {
+ else if (ch == ':' && i < flen && PyUnicode_READ_CHAR(format, i) == 'z') {
/* %:z -> +HH:MM */
+ i++;
if (colonzreplacement == NULL) {
colonzreplacement = make_somezreplacement(object, ":", tzinfoarg);
if (colonzreplacement == NULL)
- goto Done;
+ goto Error;
}
- assert(colonzreplacement != NULL);
- assert(PyBytes_Check(colonzreplacement));
- ptoappend = PyBytes_AS_STRING(colonzreplacement);
- ntoappend = PyBytes_GET_SIZE(colonzreplacement);
+ replacement = colonzreplacement;
}
else if (ch == 'Z') {
/* format tzname */
@@ -1670,79 +1651,63 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple,
Zreplacement = make_Zreplacement(object,
tzinfoarg);
if (Zreplacement == NULL)
- goto Done;
+ goto Error;
}
- assert(Zreplacement != NULL);
- assert(PyUnicode_Check(Zreplacement));
- ptoappend = PyUnicode_AsUTF8AndSize(Zreplacement,
- &ntoappend);
- if (ptoappend == NULL)
- goto Done;
+ replacement = Zreplacement;
}
else if (ch == 'f') {
/* format microseconds */
if (freplacement == NULL) {
freplacement = make_freplacement(object);
if (freplacement == NULL)
- goto Done;
+ goto Error;
}
- assert(freplacement != NULL);
- assert(PyBytes_Check(freplacement));
- ptoappend = PyBytes_AS_STRING(freplacement);
- ntoappend = PyBytes_GET_SIZE(freplacement);
+ replacement = freplacement;
}
else {
/* percent followed by something else */
- ptoappend = pin - 2;
- ntoappend = 2;
- }
-
- /* Append the ntoappend chars starting at ptoappend to
- * the new format.
- */
- if (ntoappend == 0)
continue;
- assert(ptoappend != NULL);
- assert(ntoappend > 0);
- while (usednew + ntoappend > totalnew) {
- if (totalnew > (PY_SSIZE_T_MAX >> 1)) { /* overflow */
- PyErr_NoMemory();
- goto Done;
- }
- totalnew <<= 1;
- if (_PyBytes_Resize(&newfmt, totalnew) < 0)
- goto Done;
- pnew = PyBytes_AsString(newfmt) + usednew;
}
- memcpy(pnew, ptoappend, ntoappend);
- pnew += ntoappend;
- usednew += ntoappend;
- assert(usednew <= totalnew);
+ assert(replacement != NULL);
+ assert(PyUnicode_Check(replacement));
+ if (_PyUnicodeWriter_WriteSubstring(&writer, format, start, end) < 0) {
+ goto Error;
+ }
+ start = i;
+ if (_PyUnicodeWriter_WriteStr(&writer, replacement) < 0) {
+ goto Error;
+ }
} /* end while() */
- if (_PyBytes_Resize(&newfmt, usednew) < 0)
- goto Done;
- {
- PyObject *format;
- PyObject *strftime = _PyImport_GetModuleAttrString("time", "strftime");
-
- if (strftime == NULL)
+ PyObject *newformat;
+ if (start == 0) {
+ _PyUnicodeWriter_Dealloc(&writer);
+ newformat = Py_NewRef(format);
+ }
+ else {
+ if (_PyUnicodeWriter_WriteSubstring(&writer, format, start, flen) < 0) {
+ goto Error;
+ }
+ newformat = _PyUnicodeWriter_Finish(&writer);
+ if (newformat == NULL) {
goto Done;
- format = PyUnicode_FromString(PyBytes_AS_STRING(newfmt));
- if (format != NULL) {
- result = PyObject_CallFunctionObjArgs(strftime,
- format, timetuple, NULL);
- Py_DECREF(format);
}
- Py_DECREF(strftime);
}
+ result = PyObject_CallFunctionObjArgs(strftime,
+ newformat, timetuple, NULL);
+ Py_DECREF(newformat);
+
Done:
Py_XDECREF(freplacement);
Py_XDECREF(zreplacement);
Py_XDECREF(colonzreplacement);
Py_XDECREF(Zreplacement);
- Py_XDECREF(newfmt);
+ Py_XDECREF(strftime);
return result;
+
+ Error:
+ _PyUnicodeWriter_Dealloc(&writer);
+ goto Done;
}
/* ---------------------------------------------------------------------------
diff --git a/Modules/timemodule.c b/Modules/timemodule.c
index b10f8c7c939550..59078263571f7e 100644
--- a/Modules/timemodule.c
+++ b/Modules/timemodule.c
@@ -795,27 +795,100 @@ the C library strftime function.\n"
#endif
static PyObject *
-time_strftime(PyObject *module, PyObject *args)
+time_strftime1(time_char **outbuf, size_t *bufsize,
+ time_char *format, size_t fmtlen,
+ struct tm *tm)
{
- PyObject *tup = NULL;
- struct tm buf;
- const time_char *fmt;
+ size_t buflen;
+#if defined(MS_WINDOWS) && !defined(HAVE_WCSFTIME)
+ /* check that the format string contains only valid directives */
+ for (const time_char *f = strchr(format, '%');
+ f != NULL;
+ f = strchr(f + 2, '%'))
+ {
+ if (f[1] == '#')
+ ++f; /* not documented by python, */
+ if (f[1] == '\0')
+ break;
+ if ((f[1] == 'y') && tm->tm_year < 0) {
+ PyErr_SetString(PyExc_ValueError,
+ "format %y requires year >= 1900 on Windows");
+ return NULL;
+ }
+ }
+#elif (defined(_AIX) || (defined(__sun) && defined(__SVR4))) && defined(HAVE_WCSFTIME)
+ for (const time_char *f = wcschr(format, '%');
+ f != NULL;
+ f = wcschr(f + 2, '%'))
+ {
+ if (f[1] == L'\0')
+ break;
+ /* Issue #19634: On AIX, wcsftime("y", (1899, 1, 1, 0, 0, 0, 0, 0, 0))
+ returns "0/" instead of "99" */
+ if (f[1] == L'y' && tm->tm_year < 0) {
+ PyErr_SetString(PyExc_ValueError,
+ "format %y requires year >= 1900 on AIX");
+ return NULL;
+ }
+ }
+#endif
+
+ /* I hate these functions that presume you know how big the output
+ * will be ahead of time...
+ */
+ while (1) {
+ if (*bufsize > PY_SSIZE_T_MAX/sizeof(time_char)) {
+ PyErr_NoMemory();
+ return NULL;
+ }
+ *outbuf = (time_char *)PyMem_Realloc(*outbuf,
+ *bufsize*sizeof(time_char));
+ if (*outbuf == NULL) {
+ PyErr_NoMemory();
+ return NULL;
+ }
+#if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__)
+ errno = 0;
+#endif
+ _Py_BEGIN_SUPPRESS_IPH
+ buflen = format_time(*outbuf, *bufsize, format, tm);
+ _Py_END_SUPPRESS_IPH
+#if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__)
+ /* VisualStudio .NET 2005 does this properly */
+ if (buflen == 0 && errno == EINVAL) {
+ PyErr_SetString(PyExc_ValueError, "Invalid format string");
+ return NULL;
+ }
+#endif
+ if (buflen == 0 && *bufsize < 256 * fmtlen) {
+ *bufsize += *bufsize;
+ continue;
+ }
+ /* If the buffer is 256 times as long as the format,
+ it's probably not failing for lack of room!
+ More likely, the format yields an empty result,
+ e.g. an empty format, or %Z when the timezone
+ is unknown. */
#ifdef HAVE_WCSFTIME
- wchar_t *format;
+ return PyUnicode_FromWideChar(*outbuf, buflen);
#else
- PyObject *format;
+ return PyUnicode_DecodeLocaleAndSize(*outbuf, buflen, "surrogateescape");
#endif
+ }
+}
+
+static PyObject *
+time_strftime(PyObject *module, PyObject *args)
+{
+ PyObject *tup = NULL;
+ struct tm buf;
PyObject *format_arg;
- size_t fmtlen, buflen;
- time_char *outbuf = NULL;
- size_t i;
- PyObject *ret = NULL;
+ Py_ssize_t format_size;
+ time_char *format, *outbuf = NULL;
+ size_t fmtlen, bufsize = 1024;
memset((void *) &buf, '\0', sizeof(buf));
- /* Will always expect a unicode string to be passed as format.
- Given that there's no str type anymore in py3k this seems safe.
- */
if (!PyArg_ParseTuple(args, "U|O:strftime", &format_arg, &tup))
return NULL;
@@ -848,101 +921,65 @@ time_strftime(PyObject *module, PyObject *args)
else if (buf.tm_isdst > 1)
buf.tm_isdst = 1;
-#ifdef HAVE_WCSFTIME
- format = PyUnicode_AsWideCharString(format_arg, NULL);
- if (format == NULL)
+ format_size = PyUnicode_GET_LENGTH(format_arg);
+ if ((size_t)format_size > PY_SSIZE_T_MAX/sizeof(time_char) - 1) {
+ PyErr_NoMemory();
return NULL;
- fmt = format;
-#else
- /* Convert the unicode string to an ascii one */
- format = PyUnicode_EncodeLocale(format_arg, "surrogateescape");
- if (format == NULL)
+ }
+ format = PyMem_Malloc((format_size + 1)*sizeof(time_char));
+ if (format == NULL) {
+ PyErr_NoMemory();
return NULL;
- fmt = PyBytes_AS_STRING(format);
-#endif
-
-#if defined(MS_WINDOWS) && !defined(HAVE_WCSFTIME)
- /* check that the format string contains only valid directives */
- for (outbuf = strchr(fmt, '%');
- outbuf != NULL;
- outbuf = strchr(outbuf+2, '%'))
- {
- if (outbuf[1] == '#')
- ++outbuf; /* not documented by python, */
- if (outbuf[1] == '\0')
- break;
- if ((outbuf[1] == 'y') && buf.tm_year < 0) {
- PyErr_SetString(PyExc_ValueError,
- "format %y requires year >= 1900 on Windows");
- Py_DECREF(format);
- return NULL;
- }
}
-#elif (defined(_AIX) || (defined(__sun) && defined(__SVR4))) && defined(HAVE_WCSFTIME)
- for (outbuf = wcschr(fmt, '%');
- outbuf != NULL;
- outbuf = wcschr(outbuf+2, '%'))
- {
- if (outbuf[1] == L'\0')
- break;
- /* Issue #19634: On AIX, wcsftime("y", (1899, 1, 1, 0, 0, 0, 0, 0, 0))
- returns "0/" instead of "99" */
- if (outbuf[1] == L'y' && buf.tm_year < 0) {
- PyErr_SetString(PyExc_ValueError,
- "format %y requires year >= 1900 on AIX");
- PyMem_Free(format);
- return NULL;
+ _PyUnicodeWriter writer;
+ _PyUnicodeWriter_Init(&writer);
+ writer.overallocate = 1;
+ Py_ssize_t i = 0;
+ while (i < format_size) {
+ fmtlen = 0;
+ for (; i < format_size; i++) {
+ Py_UCS4 c = PyUnicode_READ_CHAR(format_arg, i);
+ if (!c || c > 127) {
+ break;
+ }
+ format[fmtlen++] = (char)c;
}
- }
-#endif
-
- fmtlen = time_strlen(fmt);
-
- /* I hate these functions that presume you know how big the output
- * will be ahead of time...
- */
- for (i = 1024; ; i += i) {
- outbuf = (time_char *)PyMem_Malloc(i*sizeof(time_char));
- if (outbuf == NULL) {
- PyErr_NoMemory();
- break;
+ if (fmtlen) {
+ format[fmtlen] = 0;
+ PyObject *unicode = time_strftime1(&outbuf, &bufsize,
+ format, fmtlen, &buf);
+ if (unicode == NULL) {
+ goto error;
+ }
+ if (_PyUnicodeWriter_WriteStr(&writer, unicode) < 0) {
+ Py_DECREF(unicode);
+ goto error;
+ }
+ Py_DECREF(unicode);
}
-#if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__)
- errno = 0;
-#endif
- _Py_BEGIN_SUPPRESS_IPH
- buflen = format_time(outbuf, i, fmt, &buf);
- _Py_END_SUPPRESS_IPH
-#if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__)
- /* VisualStudio .NET 2005 does this properly */
- if (buflen == 0 && errno == EINVAL) {
- PyErr_SetString(PyExc_ValueError, "Invalid format string");
- PyMem_Free(outbuf);
- break;
+
+ Py_ssize_t start = i;
+ for (; i < format_size; i++) {
+ Py_UCS4 c = PyUnicode_READ_CHAR(format_arg, i);
+ if (c == '%') {
+ break;
+ }
}
-#endif
- if (buflen > 0 || i >= 256 * fmtlen) {
- /* If the buffer is 256 times as long as the format,
- it's probably not failing for lack of room!
- More likely, the format yields an empty result,
- e.g. an empty format, or %Z when the timezone
- is unknown. */
-#ifdef HAVE_WCSFTIME
- ret = PyUnicode_FromWideChar(outbuf, buflen);
-#else
- ret = PyUnicode_DecodeLocaleAndSize(outbuf, buflen, "surrogateescape");
-#endif
- PyMem_Free(outbuf);
- break;
+ if (start < i) {
+ if (_PyUnicodeWriter_WriteSubstring(&writer, format_arg, start, i) < 0) {
+ goto error;
+ }
}
- PyMem_Free(outbuf);
}
-#ifdef HAVE_WCSFTIME
+
+ PyMem_Free(outbuf);
PyMem_Free(format);
-#else
- Py_DECREF(format);
-#endif
- return ret;
+ return _PyUnicodeWriter_Finish(&writer);
+error:
+ PyMem_Free(outbuf);
+ PyMem_Free(format);
+ _PyUnicodeWriter_Dealloc(&writer);
+ return NULL;
}
#undef time_char
From aa9faee686d1a3773778c087e39b146271deed24 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Fri, 18 Oct 2024 04:25:00 +0200
Subject: [PATCH 079/269] [3.12] gh-125620: Skip check_resource_tracker_death
on NetBSD due to long wait for SIGKILL process termination (GH-125621)
(#125673)
gh-125620: Skip check_resource_tracker_death on NetBSD due to long wait for SIGKILL process termination (GH-125621)
* Skip test_resource_tracker_sigkill on NetBSD
(cherry picked from commit a0f5c8e6272a1fd5422892d773923b138e77ae5f)
Co-authored-by: Furkan Onder
---
Lib/test/_test_multiprocessing.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index aac74ea311f19b..607bfc02b12303 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -5655,6 +5655,8 @@ def test_resource_tracker_sigterm(self):
# Catchable signal (ignored by semaphore tracker)
self.check_resource_tracker_death(signal.SIGTERM, False)
+ @unittest.skipIf(sys.platform.startswith("netbsd"),
+ "gh-125620: Skip on NetBSD due to long wait for SIGKILL process termination.")
def test_resource_tracker_sigkill(self):
# Uncatchable signal.
self.check_resource_tracker_death(signal.SIGKILL, True)
From 93933782d999ef58682a8713e5cd3cf92166048e Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Fri, 18 Oct 2024 16:13:31 +0200
Subject: [PATCH 080/269] [3.12] Add tests for time.strftime() with invalid
format string (GH-125696) (GH-125701)
(cherry picked from commit 2e950e341930ea79549137d4d3771d5edb940e65)
Co-authored-by: Serhiy Storchaka
---
Lib/test/test_time.py | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py
index b020787dacac1c..9463adda88db0d 100644
--- a/Lib/test/test_time.py
+++ b/Lib/test/test_time.py
@@ -14,7 +14,7 @@
except ImportError:
_testcapi = None
-from test.support import skip_if_buggy_ucrt_strfptime
+from test.support import skip_if_buggy_ucrt_strfptime, SuppressCrashReport
# Max year is only limited by the size of C int.
SIZEOF_INT = sysconfig.get_config_var('SIZEOF_INT') or 4
@@ -178,6 +178,17 @@ def test_strftime(self):
self.assertRaises(TypeError, time.strftime, b'%S', tt)
+ def test_strftime_invalid_format(self):
+ tt = time.gmtime(self.t)
+ with SuppressCrashReport():
+ for i in range(1, 128):
+ format = ' %' + chr(i)
+ with self.subTest(format=format):
+ try:
+ time.strftime(format, tt)
+ except ValueError as exc:
+ self.assertEqual(str(exc), 'Invalid format string')
+
def test_strftime_special(self):
tt = time.gmtime(self.t)
s1 = time.strftime('%c', tt)
From 2ce10b17293c67e2ceeef9ced3719a49c61aa80c Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sat, 19 Oct 2024 06:27:44 +0200
Subject: [PATCH 081/269] [3.12] GH-125277: Increase minimum supported Sphinx
to 7.2.6 (GH-125368) (#125721)
---
.github/workflows/reusable-docs.yml | 2 +-
Doc/conf.py | 2 +-
Doc/requirements-oldest-sphinx.txt | 30 +++++++++----------
...-10-10-23-46-54.gh-issue-125277.QAby09.rst | 2 ++
4 files changed, 19 insertions(+), 17 deletions(-)
create mode 100644 Misc/NEWS.d/next/Documentation/2024-10-10-23-46-54.gh-issue-125277.QAby09.rst
diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml
index 4d1dc04e8b638a..d575963e3cec4a 100644
--- a/.github/workflows/reusable-docs.yml
+++ b/.github/workflows/reusable-docs.yml
@@ -81,7 +81,7 @@ jobs:
- name: 'Set up Python'
uses: actions/setup-python@v5
with:
- python-version: '3.12' # known to work with Sphinx 6.2.1
+ python-version: '3.13' # known to work with Sphinx 7.2.6
cache: 'pip'
cache-dependency-path: 'Doc/requirements-oldest-sphinx.txt'
- name: 'Install build dependencies'
diff --git a/Doc/conf.py b/Doc/conf.py
index fb55c5c65c2f35..f8e13cd109d7f4 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -82,7 +82,7 @@
highlight_language = 'python3'
# Minimum version of sphinx required
-needs_sphinx = '6.2.1'
+needs_sphinx = '7.2.6'
# Create table of contents entries for domain objects (e.g. functions, classes,
# attributes, etc.). Default is True.
diff --git a/Doc/requirements-oldest-sphinx.txt b/Doc/requirements-oldest-sphinx.txt
index 068fe0cb426ecd..3483faea6b56cb 100644
--- a/Doc/requirements-oldest-sphinx.txt
+++ b/Doc/requirements-oldest-sphinx.txt
@@ -7,29 +7,29 @@ blurb
python-docs-theme>=2022.1
# Generated from:
-# pip install "Sphinx~=6.2.1"
+# pip install "Sphinx~=7.2.6"
# pip freeze
#
-# Sphinx 6.2.1 comes from ``needs_sphinx = '6.2.1'`` in ``Doc/conf.py``.
+# Sphinx 7.2.6 comes from ``needs_sphinx = '7.2.6'`` in ``Doc/conf.py``.
alabaster==0.7.16
-Babel==2.15.0
-certifi==2024.7.4
-charset-normalizer==3.3.2
-docutils==0.19
-idna==3.7
+Babel==2.16.0
+certifi==2024.8.30
+charset-normalizer==3.4.0
+docutils==0.20.1
+idna==3.10
imagesize==1.4.1
Jinja2==3.1.4
-MarkupSafe==2.1.5
+MarkupSafe==3.0.1
packaging==24.1
Pygments==2.18.0
requests==2.32.3
snowballstemmer==2.2.0
-Sphinx==6.2.1
-sphinxcontrib-applehelp==1.0.8
-sphinxcontrib-devhelp==1.0.6
-sphinxcontrib-htmlhelp==2.0.5
+Sphinx==7.2.6
+sphinxcontrib-applehelp==2.0.0
+sphinxcontrib-devhelp==2.0.0
+sphinxcontrib-htmlhelp==2.1.0
sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.7
-sphinxcontrib-serializinghtml==1.1.10
-urllib3==2.2.2
+sphinxcontrib-qthelp==2.0.0
+sphinxcontrib-serializinghtml==2.0.0
+urllib3==2.2.3
diff --git a/Misc/NEWS.d/next/Documentation/2024-10-10-23-46-54.gh-issue-125277.QAby09.rst b/Misc/NEWS.d/next/Documentation/2024-10-10-23-46-54.gh-issue-125277.QAby09.rst
new file mode 100644
index 00000000000000..fcd6e22c27b5f4
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2024-10-10-23-46-54.gh-issue-125277.QAby09.rst
@@ -0,0 +1,2 @@
+Require Sphinx 7.2.6 or later to build the Python documentation.
+Patch by Adam Turner.
From 5f6e1120e3ba21d5140e91737386e46a22ffb5ea Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sat, 19 Oct 2024 14:07:16 +0200
Subject: [PATCH 082/269] [3.12] gh-125522: Fix bare except in
test_math.testTan (GH-125544) (#125727)
gh-125522: Fix bare except in test_math.testTan (GH-125544)
(cherry picked from commit 4b421e8aca7f2dccc5ac8604b78589941dd7974c)
Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com>
---
Lib/test/test_math.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py
index 7e403f7ef25891..600946eeb8272d 100644
--- a/Lib/test/test_math.py
+++ b/Lib/test/test_math.py
@@ -1886,7 +1886,7 @@ def testTan(self):
try:
self.assertTrue(math.isnan(math.tan(INF)))
self.assertTrue(math.isnan(math.tan(NINF)))
- except:
+ except ValueError:
self.assertRaises(ValueError, math.tan, INF)
self.assertRaises(ValueError, math.tan, NINF)
self.assertTrue(math.isnan(math.tan(NAN)))
From 9a22ec735fb2f0ec850e70dd119e8422c97567e5 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 21 Oct 2024 02:43:26 +0200
Subject: [PATCH 083/269] [3.12] gh-99030: Added documentation links for types
and exceptions (GH-123857) (GH-125765)
gh-99030: Added documentation links for types and exceptions (GH-123857)
* Added documentation links for types and exceptions
* Shortened description sentences
* Change content
* Change documentation
* Move seealso
* Add a spaces
(cherry picked from commit 9256be7ff0ab035cfd262127d893c9bc88b3c84c)
Co-authored-by: RUANG (Roy James)
---
Doc/library/builtins.rst | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/Doc/library/builtins.rst b/Doc/library/builtins.rst
index 644344e7fef29a..c4979db52d2aed 100644
--- a/Doc/library/builtins.rst
+++ b/Doc/library/builtins.rst
@@ -7,10 +7,7 @@
--------------
This module provides direct access to all 'built-in' identifiers of Python; for
-example, ``builtins.open`` is the full name for the built-in function
-:func:`open`. See :ref:`built-in-funcs` and :ref:`built-in-consts` for
-documentation.
-
+example, ``builtins.open`` is the full name for the built-in function :func:`open`.
This module is not normally accessed explicitly by most applications, but can be
useful in modules that provide objects with the same name as a built-in value,
@@ -40,3 +37,10 @@ available as part of their globals. The value of ``__builtins__`` is normally
either this module or the value of this module's :attr:`~object.__dict__` attribute.
Since this is an implementation detail, it may not be used by alternate
implementations of Python.
+
+.. seealso::
+
+ * :ref:`built-in-consts`
+ * :ref:`bltin-exceptions`
+ * :ref:`built-in-funcs`
+ * :ref:`bltin-types`
From 8cc8d7d61ff15b077fc389fbcb0bee60979cea82 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 21 Oct 2024 10:20:42 +0200
Subject: [PATCH 084/269] [3.12] gh-125519: Improve traceback if
`importlib.reload()` is called with a non-module object (GH-125520) (#125769)
gh-125519: Improve traceback if `importlib.reload()` is called with a non-module object (GH-125520)
(cherry picked from commit c5c21fee7ae1ea689a351caa454c98e716a6e537)
Co-authored-by: Alex Waygood
---
Lib/importlib/__init__.py | 2 +-
Lib/test/test_importlib/test_api.py | 16 ++++++++++++++++
...024-10-15-14-01-03.gh-issue-125519.TqGh6a.rst | 2 ++
3 files changed, 19 insertions(+), 1 deletion(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-15-14-01-03.gh-issue-125519.TqGh6a.rst
diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py
index 707c081cb2c5b6..5b92442fc379e0 100644
--- a/Lib/importlib/__init__.py
+++ b/Lib/importlib/__init__.py
@@ -105,7 +105,7 @@ def reload(module):
try:
name = module.__name__
except AttributeError:
- raise TypeError("reload() argument must be a module")
+ raise TypeError("reload() argument must be a module") from None
if sys.modules.get(name) is not module:
raise ImportError(f"module {name} not in sys.modules", name=name)
diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py
index ecf2c47c462e23..ef01a3f6de09bf 100644
--- a/Lib/test/test_importlib/test_api.py
+++ b/Lib/test/test_importlib/test_api.py
@@ -8,6 +8,8 @@
import sys
from test.support import import_helper
from test.support import os_helper
+from test import support
+import traceback
import types
import unittest
import warnings
@@ -354,6 +356,20 @@ def test_module_missing_spec(self):
with self.assertRaises(ModuleNotFoundError):
self.init.reload(module)
+ def test_reload_traceback_with_non_str(self):
+ # gh-125519
+ with support.captured_stdout() as stdout:
+ try:
+ self.init.reload("typing")
+ except TypeError as exc:
+ traceback.print_exception(exc, file=stdout)
+ else:
+ self.fail("Expected TypeError to be raised")
+ printed_traceback = stdout.getvalue()
+ self.assertIn("TypeError", printed_traceback)
+ self.assertNotIn("AttributeError", printed_traceback)
+ self.assertNotIn("module.__spec__.name", printed_traceback)
+
(Frozen_ReloadTests,
Source_ReloadTests
diff --git a/Misc/NEWS.d/next/Library/2024-10-15-14-01-03.gh-issue-125519.TqGh6a.rst b/Misc/NEWS.d/next/Library/2024-10-15-14-01-03.gh-issue-125519.TqGh6a.rst
new file mode 100644
index 00000000000000..e6062625104590
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-15-14-01-03.gh-issue-125519.TqGh6a.rst
@@ -0,0 +1,2 @@
+Improve traceback if :func:`importlib.reload` is called with an object that
+is not a module. Patch by Alex Waygood.
From 2bc2aae386f7d426022441bf9b1ce8b85bfbb46d Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland"
Date: Mon, 21 Oct 2024 12:23:41 +0200
Subject: [PATCH 085/269] [3.12] gh-125741: Update check_generated_files CI to
use our published container image (gh-125744) (#125760)
(cherry picked from commit ed24702bd0f9925908ce48584c31dfad732208b2)
Co-authored-by: Donghee Na
---
.github/workflows/build.yml | 53 ++++++++++++++++++++++++++++++-------
1 file changed, 44 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4c8fed552b5d04..a6daf0a436a459 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -91,6 +91,50 @@ jobs:
name: abi-data
path: ./Doc/data/*.abi
+ check_autoconf_regen:
+ name: 'Check if Autoconf files are up to date'
+ # Don't use ubuntu-latest but a specific version to make the job
+ # reproducible: to get the same tools versions (autoconf, aclocal, ...)
+ runs-on: ubuntu-24.04
+ container:
+ image: ghcr.io/python/autoconf:2024.10.11.11293396815
+ timeout-minutes: 60
+ needs: check_source
+ if: needs.check_source.outputs.run_tests == 'true'
+ steps:
+ - name: Install Git
+ run: |
+ apt install git -yq
+ git config --global --add safe.directory "$GITHUB_WORKSPACE"
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 1
+ - name: Runner image version
+ run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
+ - name: Check Autoconf and aclocal versions
+ run: |
+ grep "Generated by GNU Autoconf 2.71" configure
+ grep "aclocal 1.16.5" aclocal.m4
+ grep -q "runstatedir" configure
+ grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4
+ - name: Regenerate autoconf files
+ # Same command used by Tools/build/regen-configure.sh ($AUTORECONF)
+ run: autoreconf -ivf -Werror
+ - name: Check for changes
+ run: |
+ git add -u
+ changes=$(git status --porcelain)
+ # Check for changes in regenerated files
+ if test -n "$changes"; then
+ echo "Generated files not up to date."
+ echo "Perhaps you forgot to run make regen-all or build.bat --regen. ;)"
+ echo "configure files must be regenerated with a specific version of autoconf."
+ echo "$changes"
+ echo ""
+ git diff --staged || true
+ exit 1
+ fi
+
check_generated_files:
name: 'Check if generated files are up to date'
# Don't use ubuntu-latest but a specific version to make the job
@@ -119,19 +163,10 @@ jobs:
uses: hendrikmuhs/ccache-action@v1.2
with:
save: false
- - name: Check Autoconf and aclocal versions
- run: |
- grep "Generated by GNU Autoconf 2.71" configure
- grep "aclocal 1.16.5" aclocal.m4
- grep -q "runstatedir" configure
- grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4
- name: Configure CPython
run: |
# Build Python with the libpython dynamic library
./configure --config-cache --with-pydebug --enable-shared
- - name: Regenerate autoconf files
- # Same command used by Tools/build/regen-configure.sh ($AUTORECONF)
- run: autoreconf -ivf -Werror
- name: Build CPython
run: |
# Deepfreeze will usually cause global objects to be added or removed,
From 18196fea89af7d1f68386e5f6ccc63de04b75a98 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 21 Oct 2024 15:06:02 +0200
Subject: [PATCH 086/269] [3.12] gh-125682: Reject non-ASCII digits in the
Python implementation of JSON decoder (GH-125687) (GH-125693)
(cherry picked from commit d358425e6968858e52908794d15f37e62abc74ec)
Co-authored-by: Nice Zombies
---
Lib/json/scanner.py | 2 +-
Lib/test/test_json/test_decode.py | 6 ++++++
.../Library/2024-10-18-09-51-29.gh-issue-125682.vsj4cU.rst | 2 ++
3 files changed, 9 insertions(+), 1 deletion(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-18-09-51-29.gh-issue-125682.vsj4cU.rst
diff --git a/Lib/json/scanner.py b/Lib/json/scanner.py
index 7a61cfc2d24dce..090897515fe2f3 100644
--- a/Lib/json/scanner.py
+++ b/Lib/json/scanner.py
@@ -9,7 +9,7 @@
__all__ = ['make_scanner']
NUMBER_RE = re.compile(
- r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
+ r'(-?(?:0|[1-9][0-9]*))(\.[0-9]+)?([eE][-+]?[0-9]+)?',
(re.VERBOSE | re.MULTILINE | re.DOTALL))
def py_make_scanner(context):
diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py
index 79fb239b35d3f2..2250af964c022b 100644
--- a/Lib/test/test_json/test_decode.py
+++ b/Lib/test/test_json/test_decode.py
@@ -16,6 +16,12 @@ def test_float(self):
self.assertIsInstance(rval, float)
self.assertEqual(rval, 1.0)
+ def test_nonascii_digits_rejected(self):
+ # JSON specifies only ascii digits, see gh-125687
+ for num in ["1\uff10", "0.\uff10", "0e\uff10"]:
+ with self.assertRaises(self.JSONDecodeError):
+ self.loads(num)
+
def test_bytes(self):
self.assertEqual(self.loads(b"1"), 1)
diff --git a/Misc/NEWS.d/next/Library/2024-10-18-09-51-29.gh-issue-125682.vsj4cU.rst b/Misc/NEWS.d/next/Library/2024-10-18-09-51-29.gh-issue-125682.vsj4cU.rst
new file mode 100644
index 00000000000000..3eb2905ad8d810
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-18-09-51-29.gh-issue-125682.vsj4cU.rst
@@ -0,0 +1,2 @@
+Reject non-ASCII digits in the Python implementation of :func:`json.loads`
+conforming to the JSON specification.
From 2746ec4ce2b55efcd160002012147ce20686e47e Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 21 Oct 2024 15:08:10 +0200
Subject: [PATCH 087/269] [3.12] gh-125660: Reject invalid unicode escapes for
Python implementation of JSON decoder (GH-125683) (GH-125695)
(cherry picked from commit df751363e386d1f77c5ba9515a5539902457d386)
Co-authored-by: Nice Zombies
---
Lib/json/decoder.py | 9 +++++----
Lib/test/test_json/test_scanstring.py | 10 ++++++++++
.../2024-10-18-08-58-10.gh-issue-125660.sDdDqO.rst | 1 +
3 files changed, 16 insertions(+), 4 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-18-08-58-10.gh-issue-125660.sDdDqO.rst
diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py
index c5d9ae2d0d5d04..5e5effeac02551 100644
--- a/Lib/json/decoder.py
+++ b/Lib/json/decoder.py
@@ -50,17 +50,18 @@ def __reduce__(self):
}
+HEXDIGITS = re.compile(r'[0-9A-Fa-f]{4}', FLAGS)
STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
BACKSLASH = {
'"': '"', '\\': '\\', '/': '/',
'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t',
}
-def _decode_uXXXX(s, pos):
- esc = s[pos + 1:pos + 5]
- if len(esc) == 4 and esc[1] not in 'xX':
+def _decode_uXXXX(s, pos, _m=HEXDIGITS.match):
+ esc = _m(s, pos + 1)
+ if esc is not None:
try:
- return int(esc, 16)
+ return int(esc.group(), 16)
except ValueError:
pass
msg = "Invalid \\uXXXX escape"
diff --git a/Lib/test/test_json/test_scanstring.py b/Lib/test/test_json/test_scanstring.py
index 2d3ee8a8bf0f92..cca556a3b95bab 100644
--- a/Lib/test/test_json/test_scanstring.py
+++ b/Lib/test/test_json/test_scanstring.py
@@ -116,6 +116,11 @@ def test_bad_escapes(self):
'"\\u012z"',
'"\\u0x12"',
'"\\u0X12"',
+ '"\\u{0}"'.format("\uff10" * 4),
+ '"\\u 123"',
+ '"\\u-123"',
+ '"\\u+123"',
+ '"\\u1_23"',
'"\\ud834\\"',
'"\\ud834\\u"',
'"\\ud834\\ud"',
@@ -127,6 +132,11 @@ def test_bad_escapes(self):
'"\\ud834\\udd2z"',
'"\\ud834\\u0x20"',
'"\\ud834\\u0X20"',
+ '"\\ud834\\u{0}"'.format("\uff10" * 4),
+ '"\\ud834\\u 123"',
+ '"\\ud834\\u-123"',
+ '"\\ud834\\u+123"',
+ '"\\ud834\\u1_23"',
]
for s in bad_escapes:
with self.assertRaises(self.JSONDecodeError, msg=s):
diff --git a/Misc/NEWS.d/next/Library/2024-10-18-08-58-10.gh-issue-125660.sDdDqO.rst b/Misc/NEWS.d/next/Library/2024-10-18-08-58-10.gh-issue-125660.sDdDqO.rst
new file mode 100644
index 00000000000000..74d76c7bddae7d
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-18-08-58-10.gh-issue-125660.sDdDqO.rst
@@ -0,0 +1 @@
+Reject invalid unicode escapes for Python implementation of :func:`json.loads`.
From c3e705fbe5258d8d02f2e405d04b72c427bb4a74 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 21 Oct 2024 18:55:11 +0200
Subject: [PATCH 088/269] [3.12] Doc: C API: Move `tp_dealloc` paragraph to
`tp_dealloc` section (GH-125737) (#125799)
Doc: C API: Move `tp_dealloc` paragraph to `tp_dealloc` section (GH-125737)
It looks like commit 43cf44ddcce6b225f959ea2a53e4817244ca6054
(gh-31501) accidentally moved the paragraph to the `tp_finalize`
section when the intent was to move it to the `tp_dealloc` section
(according to the commit message).
(cherry picked from commit d880c83ff7fb2e464bc4f469d74cc3fc3eca082c)
Co-authored-by: Richard Hansen
---
Doc/c-api/typeobj.rst | 24 +++++++++++++-----------
1 file changed, 13 insertions(+), 11 deletions(-)
diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
index 1b28a2a3060e6e..610cb96e8885b6 100644
--- a/Doc/c-api/typeobj.rst
+++ b/Doc/c-api/typeobj.rst
@@ -704,6 +704,19 @@ and :c:data:`PyType_Type` effectively act as defaults.)
Py_DECREF(tp);
}
+ .. warning::
+
+ In a garbage collected Python, :c:member:`!tp_dealloc` may be called from
+ any Python thread, not just the thread which created the object (if the
+ object becomes part of a refcount cycle, that cycle might be collected by
+ a garbage collection on any thread). This is not a problem for Python
+ API calls, since the thread on which :c:member:`!tp_dealloc` is called
+ will own the Global Interpreter Lock (GIL). However, if the object being
+ destroyed in turn destroys objects from some other C or C++ library, care
+ should be taken to ensure that destroying those objects on the thread
+ which called :c:member:`!tp_dealloc` will not violate any assumptions of
+ the library.
+
**Inheritance:**
@@ -2101,17 +2114,6 @@ and :c:data:`PyType_Type` effectively act as defaults.)
PyErr_Restore(error_type, error_value, error_traceback);
}
- Also, note that, in a garbage collected Python,
- :c:member:`~PyTypeObject.tp_dealloc` may be called from
- any Python thread, not just the thread which created the object (if the object
- becomes part of a refcount cycle, that cycle might be collected by a garbage
- collection on any thread). This is not a problem for Python API calls, since
- the thread on which tp_dealloc is called will own the Global Interpreter Lock
- (GIL). However, if the object being destroyed in turn destroys objects from some
- other C or C++ library, care should be taken to ensure that destroying those
- objects on the thread which called tp_dealloc will not violate any assumptions
- of the library.
-
**Inheritance:**
This field is inherited by subtypes.
From cea93dbd99d4d19c1c8ec688d9411e59f1ba42d2 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 21 Oct 2024 18:56:29 +0200
Subject: [PATCH 089/269] [3.12] gh-125766: Docs: minor rewording of
installation on Linux section (GH-125794)
(cherry picked from commit d67bf2d89ab57f94608d7d2cf949dc4a8749485d)
Co-authored-by: partev
---
Doc/using/unix.rst | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst
index 58838c28e6eb86..a2bcdab09a9282 100644
--- a/Doc/using/unix.rst
+++ b/Doc/using/unix.rst
@@ -17,12 +17,12 @@ On Linux
Python comes preinstalled on most Linux distributions, and is available as a
package on all others. However there are certain features you might want to use
-that are not available on your distro's package. You can easily compile the
+that are not available on your distro's package. You can compile the
latest version of Python from source.
-In the event that Python doesn't come preinstalled and isn't in the repositories as
-well, you can easily make packages for your own distro. Have a look at the
-following links:
+In the event that the latest version of Python doesn't come preinstalled and isn't
+in the repositories as well, you can make packages for your own distro. Have a
+look at the following links:
.. seealso::
From 0009651d8ea5bf084da4436dcf28a374d40b066b Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 21 Oct 2024 21:24:45 +0200
Subject: [PATCH 090/269] [3.12] gh-124969: Make
locale.nl_langinfo(locale.ALT_DIGITS) returning a string again (GH-125774)
(GH-125805)
This is a follow up of GH-124974. Only Glibc needed a fix.
Now the returned value is a string consisting of semicolon-separated
symbols on all Posix platforms.
(cherry picked from commit dcc4fb2c9068f60353f0c0978948b7681f7745e6)
Co-authored-by: Serhiy Storchaka
---
Doc/library/locale.rst | 7 ++--
Lib/test/test__locale.py | 30 ++++++++++-----
...-10-08-12-09-09.gh-issue-124969._VBQLq.rst | 7 ++--
Modules/_localemodule.c | 38 +++++++++++--------
4 files changed, 52 insertions(+), 30 deletions(-)
diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst
index f0553d51fedf14..a81879a2fe48dc 100644
--- a/Doc/library/locale.rst
+++ b/Doc/library/locale.rst
@@ -158,8 +158,7 @@ The :mod:`locale` module defines the following exception and functions:
.. function:: nl_langinfo(option)
- Return some locale-specific information as a string (or a tuple for
- ``ALT_DIGITS``). This function is not
+ Return some locale-specific information as a string. This function is not
available on all systems, and the set of possible options might also vary
across platforms. The possible argument values are numbers, for which
symbolic constants are available in the locale module.
@@ -312,7 +311,9 @@ The :mod:`locale` module defines the following exception and functions:
.. data:: ALT_DIGITS
- Get a tuple of up to 100 strings used to represent the values 0 to 99.
+ Get a string consisting of up to 100 semicolon-separated symbols used
+ to represent the values 0 to 99 in a locale-specific way.
+ In most locales this is an empty string.
.. function:: getdefaultlocale([envvars])
diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py
index 5041def7216197..a680e6edb63c0e 100644
--- a/Lib/test/test__locale.py
+++ b/Lib/test/test__locale.py
@@ -26,7 +26,10 @@
'bs_BA', 'fr_LU', 'kl_GL', 'fa_IR', 'de_BE', 'sv_SE', 'it_CH', 'uk_UA',
'eu_ES', 'vi_VN', 'af_ZA', 'nb_NO', 'en_DK', 'tg_TJ', 'ps_AF', 'en_US',
'fr_FR.ISO8859-1', 'fr_FR.UTF-8', 'fr_FR.ISO8859-15@euro',
- 'ru_RU.KOI8-R', 'ko_KR.eucKR']
+ 'ru_RU.KOI8-R', 'ko_KR.eucKR',
+ 'ja_JP.UTF-8', 'lzh_TW.UTF-8', 'my_MM.UTF-8', 'or_IN.UTF-8', 'shn_MM.UTF-8',
+ 'ar_AE.UTF-8', 'bn_IN.UTF-8', 'mr_IN.UTF-8', 'th_TH.TIS620',
+]
def setUpModule():
global candidate_locales
@@ -78,11 +81,13 @@ def accept(loc):
'C': (0, {}),
'en_US': (0, {}),
'fa_IR': (100, {0: '\u06f0\u06f0', 10: '\u06f1\u06f0', 99: '\u06f9\u06f9'}),
- 'ja_JP': (100, {0: '\u3007', 10: '\u5341', 99: '\u4e5d\u5341\u4e5d'}),
+ 'ja_JP': (100, {1: '\u4e00', 10: '\u5341', 99: '\u4e5d\u5341\u4e5d'}),
'lzh_TW': (32, {0: '\u3007', 10: '\u5341', 31: '\u5345\u4e00'}),
'my_MM': (100, {0: '\u1040\u1040', 10: '\u1041\u1040', 99: '\u1049\u1049'}),
'or_IN': (100, {0: '\u0b66', 10: '\u0b67\u0b66', 99: '\u0b6f\u0b6f'}),
'shn_MM': (100, {0: '\u1090\u1090', 10: '\u1091\u1090', 99: '\u1099\u1099'}),
+ 'ar_AE': (100, {0: '\u0660', 10: '\u0661\u0660', 99: '\u0669\u0669'}),
+ 'bn_IN': (100, {0: '\u09e6', 10: '\u09e7\u09e6', 99: '\u09ef\u09ef'}),
}
if sys.platform == 'win32':
@@ -196,7 +201,7 @@ def test_lc_numeric_basic(self):
def test_alt_digits_nl_langinfo(self):
# Test nl_langinfo(ALT_DIGITS)
tested = False
- for loc, (count, samples) in known_alt_digits.items():
+ for loc in candidate_locales:
with self.subTest(locale=loc):
try:
setlocale(LC_TIME, loc)
@@ -204,14 +209,21 @@ def test_alt_digits_nl_langinfo(self):
except Error:
self.skipTest(f'no locale {loc!r}')
continue
+
with self.subTest(locale=loc):
alt_digits = nl_langinfo(locale.ALT_DIGITS)
- self.assertIsInstance(alt_digits, tuple)
- if count and not alt_digits and support.is_apple:
- self.skipTest(f'ALT_DIGITS is not set for locale {loc!r} on Apple platforms')
- self.assertEqual(len(alt_digits), count)
- for i in samples:
- self.assertEqual(alt_digits[i], samples[i])
+ self.assertIsInstance(alt_digits, str)
+ alt_digits = alt_digits.split(';') if alt_digits else []
+ if alt_digits:
+ self.assertGreaterEqual(len(alt_digits), 10, alt_digits)
+ loc1 = loc.split('.', 1)[0]
+ if loc1 in known_alt_digits:
+ count, samples = known_alt_digits[loc1]
+ if count and not alt_digits:
+ self.skipTest(f'ALT_DIGITS is not set for locale {loc!r} on this platform')
+ self.assertEqual(len(alt_digits), count, alt_digits)
+ for i in samples:
+ self.assertEqual(alt_digits[i], samples[i])
tested = True
if not tested:
self.skipTest('no suitable locales')
diff --git a/Misc/NEWS.d/next/Library/2024-10-08-12-09-09.gh-issue-124969._VBQLq.rst b/Misc/NEWS.d/next/Library/2024-10-08-12-09-09.gh-issue-124969._VBQLq.rst
index b5082b90721d42..7959ce2d1e9907 100644
--- a/Misc/NEWS.d/next/Library/2024-10-08-12-09-09.gh-issue-124969._VBQLq.rst
+++ b/Misc/NEWS.d/next/Library/2024-10-08-12-09-09.gh-issue-124969._VBQLq.rst
@@ -1,3 +1,4 @@
-Fix ``locale.nl_langinfo(locale.ALT_DIGITS)``. Now it returns a tuple of up
-to 100 strings (an empty tuple on most locales). Previously it returned the
-first item of that tuple or an empty string.
+Fix ``locale.nl_langinfo(locale.ALT_DIGITS)`` on platforms with glibc.
+Now it returns a string consisting of up to 100 semicolon-separated symbols
+(an empty string in most locales) on all Posix platforms.
+Previously it only returned the first symbol or an empty string.
diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c
index 3dea764fdaad27..53ebb57d23ae07 100644
--- a/Modules/_localemodule.c
+++ b/Modules/_localemodule.c
@@ -619,28 +619,36 @@ _locale_nl_langinfo_impl(PyObject *module, int item)
const char *result = nl_langinfo(item);
result = result != NULL ? result : "";
PyObject *pyresult;
+#ifdef __GLIBC__
#ifdef ALT_DIGITS
- if (item == ALT_DIGITS) {
- /* The result is a sequence of up to 100 NUL-separated strings. */
- const char *s = result;
+ if (item == ALT_DIGITS && *result) {
+ /* According to the POSIX specification the result must be
+ * a sequence of up to 100 semicolon-separated strings.
+ * But in Glibc they are NUL-separated. */
+ Py_ssize_t i = 0;
int count = 0;
- for (; count < 100 && *s; count++) {
- s += strlen(s) + 1;
+ for (; count < 100 && result[i]; count++) {
+ i += strlen(result + i) + 1;
}
- pyresult = PyTuple_New(count);
- if (pyresult != NULL) {
- for (int i = 0; i < count; i++) {
- PyObject *unicode = PyUnicode_DecodeLocale(result, NULL);
- if (unicode == NULL) {
- Py_CLEAR(pyresult);
- break;
- }
- PyTuple_SET_ITEM(pyresult, i, unicode);
- result += strlen(result) + 1;
+ char *buf = PyMem_Malloc(i);
+ if (buf == NULL) {
+ PyErr_NoMemory();
+ pyresult = NULL;
+ }
+ else {
+ memcpy(buf, result, i);
+ /* Replace all NULs with semicolons. */
+ i = 0;
+ while (--count) {
+ i += strlen(buf + i);
+ buf[i++] = ';';
}
+ pyresult = PyUnicode_DecodeLocale(buf, NULL);
+ PyMem_Free(buf);
}
}
else
+#endif
#endif
{
pyresult = PyUnicode_DecodeLocale(result, NULL);
From e6746d9a5d215064ae5592b520d9fd3115a023aa Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 21 Oct 2024 21:37:34 +0200
Subject: [PATCH 091/269] [3.12] gh-53203: Fix test_strptime on Solaris
(GH-125785) (GH-125807)
Use fixed timezone.
Skip roundtrip tests on locales with 2-digit year.
(cherry picked from commit 9dde4638e44639d45bd7d72e70a8d410995a585a)
Co-authored-by: Serhiy Storchaka
---
Lib/test/test_strptime.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py
index 45b5c8ed2c3051..94b16635bb023c 100644
--- a/Lib/test/test_strptime.py
+++ b/Lib/test/test_strptime.py
@@ -513,12 +513,17 @@ def test_date_time_locale(self):
# NB: Dates before 1969 do not roundtrip on some locales:
# az_IR, bo_CN, bo_IN, dz_BT, eu_ES, eu_FR, fa_IR, or_IN.
+ @support.run_with_tz('STD-1DST,M4.1.0,M10.1.0')
@run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP',
'he_IL', 'ar_AE', 'mfe_MU', 'yo_NG',
'csb_PL', 'br_FR', 'gez_ET', 'brx_IN',
'my_MM', 'shn_MM')
def test_date_time_locale2(self):
# Test %c directive
+ loc = locale.getlocale(locale.LC_TIME)[0]
+ if sys.platform.startswith('sunos'):
+ if loc in ('ar_AE',):
+ self.skipTest(f'locale {loc!r} may not work on this platform')
self.roundtrip('%c', slice(0, 6), (1900, 1, 1, 0, 0, 0, 0, 1, 0))
self.roundtrip('%c', slice(0, 6), (1800, 1, 1, 0, 0, 0, 0, 1, 0))
@@ -551,6 +556,10 @@ def test_date_locale(self):
'eu_ES', 'ar_AE', 'my_MM', 'shn_MM')
def test_date_locale2(self):
# Test %x directive
+ loc = locale.getlocale(locale.LC_TIME)[0]
+ if sys.platform.startswith('sunos'):
+ if loc in ('en_US', 'de_DE', 'ar_AE'):
+ self.skipTest(f'locale {loc!r} may not work on this platform')
self.roundtrip('%x', slice(0, 3), (1900, 1, 1, 0, 0, 0, 0, 1, 0))
self.roundtrip('%x', slice(0, 3), (1800, 1, 1, 0, 0, 0, 0, 1, 0))
From 7a2bd3d2adba2c95f316243dc477910e63f4e6ae Mon Sep 17 00:00:00 2001
From: Zachary Ware
Date: Mon, 21 Oct 2024 21:44:56 -0500
Subject: [PATCH 092/269] [3.12] gh-124448: Update Windows builds to use Tcl/Tk
8.6.15 (GH-125796)
(cherry picked from commit 9d8f2d8e08336695fdade5846da4bbcc3eb5f152)
---
...024-09-24-19-04-56.gh-issue-124448.srVT3d.rst | 1 +
Misc/externals.spdx.json | 16 ++++++++--------
PCbuild/get_externals.bat | 6 +++---
PCbuild/readme.txt | 2 +-
PCbuild/tcltk.props | 2 +-
5 files changed, 14 insertions(+), 13 deletions(-)
create mode 100644 Misc/NEWS.d/next/Windows/2024-09-24-19-04-56.gh-issue-124448.srVT3d.rst
diff --git a/Misc/NEWS.d/next/Windows/2024-09-24-19-04-56.gh-issue-124448.srVT3d.rst b/Misc/NEWS.d/next/Windows/2024-09-24-19-04-56.gh-issue-124448.srVT3d.rst
new file mode 100644
index 00000000000000..ca9845a8daea9d
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2024-09-24-19-04-56.gh-issue-124448.srVT3d.rst
@@ -0,0 +1 @@
+Updated bundled Tcl/Tk to 8.6.15.
diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json
index 6c711e74f94c79..4219cd4fe4c953 100644
--- a/Misc/externals.spdx.json
+++ b/Misc/externals.spdx.json
@@ -90,42 +90,42 @@
"checksums": [
{
"algorithm": "SHA256",
- "checksumValue": "6e33a88f116822167734cd72b693b5d30ced130a3cae6dc2ff696042f993bb42"
+ "checksumValue": "4c23f0dd3efcbe6f3a22c503a68d147617bb30c4f5290f1eb3eaacf0b460440b"
}
],
- "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.13.0.tar.gz",
+ "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.15.0.tar.gz",
"externalRefs": [
{
"referenceCategory": "SECURITY",
- "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.13.0:*:*:*:*:*:*:*",
+ "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.15.0:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
}
],
"licenseConcluded": "NOASSERTION",
"name": "tcl-core",
"primaryPackagePurpose": "SOURCE",
- "versionInfo": "8.6.13.0"
+ "versionInfo": "8.6.15.0"
},
{
"SPDXID": "SPDXRef-PACKAGE-tk",
"checksums": [
{
"algorithm": "SHA256",
- "checksumValue": "896c1f488bdd0159091bd5cce124b756dfdffa4a5350b7fd4d7d8e48421089a4"
+ "checksumValue": "0ae56d39bca92865f338529557a1e56d110594184b6dc5a91339c5675751e264"
}
],
- "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.13.0.tar.gz",
+ "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.15.0.tar.gz",
"externalRefs": [
{
"referenceCategory": "SECURITY",
- "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.13.0:*:*:*:*:*:*:*",
+ "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.15.0:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
}
],
"licenseConcluded": "NOASSERTION",
"name": "tk",
"primaryPackagePurpose": "SOURCE",
- "versionInfo": "8.6.13.0"
+ "versionInfo": "8.6.15.0"
},
{
"SPDXID": "SPDXRef-PACKAGE-tix",
diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat
index e0144bcdf18c10..fa99d03e51ffe6 100644
--- a/PCbuild/get_externals.bat
+++ b/PCbuild/get_externals.bat
@@ -55,8 +55,8 @@ set libraries=%libraries% bzip2-1.0.8
if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4
if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.15
set libraries=%libraries% sqlite-3.45.3.0
-if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.0
-if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.0
+if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0
+if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.15.0
if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tix-8.4.3.6
set libraries=%libraries% xz-5.2.5
set libraries=%libraries% zlib-1.3.1
@@ -78,7 +78,7 @@ echo.Fetching external binaries...
set binaries=
if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4
if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.15
-if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.0
+if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.2
if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06
for %%b in (%binaries%) do (
diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt
index 29edf9bf260ab1..e054a94cf578bf 100644
--- a/PCbuild/readme.txt
+++ b/PCbuild/readme.txt
@@ -193,7 +193,7 @@ _sqlite3
Homepage:
https://www.sqlite.org/
_tkinter
- Wraps version 8.6.6 of the Tk windowing system, which is downloaded
+ Wraps version 8.6.15 of the Tk windowing system, which is downloaded
from our binaries repository at
https://github.com/python/cpython-bin-deps.
diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props
index 9d5189b3b8e93d..6b01d5b78eb199 100644
--- a/PCbuild/tcltk.props
+++ b/PCbuild/tcltk.props
@@ -2,7 +2,7 @@
- 8.6.13.0
+ 8.6.15.2
$(TclVersion)
8.4.3.6
$([System.Version]::Parse($(TclVersion)).Major)
From 98a0b362b5a9fea5662003474a690ad415195bc1 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 22 Oct 2024 15:16:03 +0200
Subject: [PATCH 093/269] [3.12] Doc: Show object descriptions in the table of
contents (GH-125757) (#125841)
(cherry picked from commit 91ddde4af0c3031c84a967bcf59f6fb4f8a48c0d)
Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
---
Doc/conf.py | 3 ++-
Doc/tools/extensions/pyspecific.py | 1 +
Doc/tools/static/sidebar-wrap.css | 6 ++++++
3 files changed, 9 insertions(+), 1 deletion(-)
create mode 100644 Doc/tools/static/sidebar-wrap.css
diff --git a/Doc/conf.py b/Doc/conf.py
index f8e13cd109d7f4..f4cf4ee9547e19 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -86,7 +86,8 @@
# Create table of contents entries for domain objects (e.g. functions, classes,
# attributes, etc.). Default is True.
-toc_object_entries = False
+toc_object_entries = True
+toc_object_entries_show_parents = 'hide'
# Ignore any .rst files in the includes/ directory;
# they're embedded in pages but not rendered individually.
diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py
index d3a4835edd58a4..8cb5eaffebfad0 100644
--- a/Doc/tools/extensions/pyspecific.py
+++ b/Doc/tools/extensions/pyspecific.py
@@ -412,5 +412,6 @@ def setup(app):
app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod)
app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod)
app.add_directive('miscnews', MiscNews)
+ app.add_css_file('sidebar-wrap.css')
app.connect('env-check-consistency', patch_pairindextypes)
return {'version': '1.0', 'parallel_read_safe': True}
diff --git a/Doc/tools/static/sidebar-wrap.css b/Doc/tools/static/sidebar-wrap.css
new file mode 100644
index 00000000000000..0a80f516f28349
--- /dev/null
+++ b/Doc/tools/static/sidebar-wrap.css
@@ -0,0 +1,6 @@
+div.sphinxsidebarwrapper {
+ overflow-x: scroll;
+}
+div.sphinxsidebarwrapper li code {
+ overflow-wrap: normal;
+}
From da3d81d3ea94002bc81af2713f6f891ac8211289 Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka
Date: Tue, 22 Oct 2024 16:23:30 +0300
Subject: [PATCH 094/269] [3.12] gh-125355: Rewrite parse_intermixed_args() in
argparse (GH-125356) (GH-125839)
* The parser no longer changes temporarily during parsing.
* Default values are not processed twice.
* Required mutually exclusive groups containing positional arguments are
now supported.
* The missing arguments report now includes the names of all required
optional and positional arguments.
* Unknown options can be intermixed with positional arguments in
parse_known_intermixed_args().
(cherry picked from commit 759a54d28ffe7eac8c23917f5d3dfad8309856be)
---
Lib/argparse.py | 99 +++++++------------
Lib/test/test_argparse.py | 52 +++++++---
...-10-22-13-28-00.gh-issue-125355.zssHm_.rst | 7 ++
3 files changed, 81 insertions(+), 77 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-22-13-28-00.gh-issue-125355.zssHm_.rst
diff --git a/Lib/argparse.py b/Lib/argparse.py
index 22c9b07db18a4f..2a8b501a44e855 100644
--- a/Lib/argparse.py
+++ b/Lib/argparse.py
@@ -1911,6 +1911,9 @@ def parse_args(self, args=None, namespace=None):
return args
def parse_known_args(self, args=None, namespace=None):
+ return self._parse_known_args2(args, namespace, intermixed=False)
+
+ def _parse_known_args2(self, args, namespace, intermixed):
if args is None:
# args default to the system args
args = _sys.argv[1:]
@@ -1937,18 +1940,18 @@ def parse_known_args(self, args=None, namespace=None):
# parse the arguments and exit if there are any errors
if self.exit_on_error:
try:
- namespace, args = self._parse_known_args(args, namespace)
+ namespace, args = self._parse_known_args(args, namespace, intermixed)
except ArgumentError as err:
self.error(str(err))
else:
- namespace, args = self._parse_known_args(args, namespace)
+ namespace, args = self._parse_known_args(args, namespace, intermixed)
if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
return namespace, args
- def _parse_known_args(self, arg_strings, namespace):
+ def _parse_known_args(self, arg_strings, namespace, intermixed):
# replace arg strings that are file references
if self.fromfile_prefix_chars is not None:
arg_strings = self._read_args_from_files(arg_strings)
@@ -2038,6 +2041,7 @@ def consume_optional(start_index):
# if we found no optional action, skip it
if action is None:
extras.append(arg_strings[start_index])
+ extras_pattern.append('O')
return start_index + 1
# if there is an explicit argument, try to match the
@@ -2073,6 +2077,7 @@ def consume_optional(start_index):
sep = ''
else:
extras.append(char + explicit_arg)
+ extras_pattern.append('O')
stop = start_index + 1
break
# if the action expect exactly one argument, we've
@@ -2143,6 +2148,7 @@ def consume_positionals(start_index):
# consume Positionals and Optionals alternately, until we have
# passed the last option string
extras = []
+ extras_pattern = []
start_index = 0
if option_string_indices:
max_option_string_index = max(option_string_indices)
@@ -2155,7 +2161,7 @@ def consume_positionals(start_index):
index
for index in option_string_indices
if index >= start_index])
- if start_index != next_option_string_index:
+ if not intermixed and start_index != next_option_string_index:
positionals_end_index = consume_positionals(start_index)
# only try to parse the next optional if we didn't consume
@@ -2171,16 +2177,35 @@ def consume_positionals(start_index):
if start_index not in option_string_indices:
strings = arg_strings[start_index:next_option_string_index]
extras.extend(strings)
+ extras_pattern.extend(arg_strings_pattern[start_index:next_option_string_index])
start_index = next_option_string_index
# consume the next optional and any arguments for it
start_index = consume_optional(start_index)
- # consume any positionals following the last Optional
- stop_index = consume_positionals(start_index)
+ if not intermixed:
+ # consume any positionals following the last Optional
+ stop_index = consume_positionals(start_index)
- # if we didn't consume all the argument strings, there were extras
- extras.extend(arg_strings[stop_index:])
+ # if we didn't consume all the argument strings, there were extras
+ extras.extend(arg_strings[stop_index:])
+ else:
+ extras.extend(arg_strings[start_index:])
+ extras_pattern.extend(arg_strings_pattern[start_index:])
+ extras_pattern = ''.join(extras_pattern)
+ assert len(extras_pattern) == len(extras)
+ # consume all positionals
+ arg_strings = [s for s, c in zip(extras, extras_pattern) if c != 'O']
+ arg_strings_pattern = extras_pattern.replace('O', '')
+ stop_index = consume_positionals(0)
+ # leave unknown optionals and non-consumed positionals in extras
+ for i, c in enumerate(extras_pattern):
+ if not stop_index:
+ break
+ if c != 'O':
+ stop_index -= 1
+ extras[i] = None
+ extras = [s for s in extras if s is not None]
# make sure all required actions were present and also convert
# action defaults which were not given as arguments
@@ -2446,10 +2471,6 @@ def parse_known_intermixed_args(self, args=None, namespace=None):
# are then parsed. If the parser definition is incompatible with the
# intermixed assumptions (e.g. use of REMAINDER, subparsers) a
# TypeError is raised.
- #
- # positionals are 'deactivated' by setting nargs and default to
- # SUPPRESS. This blocks the addition of that positional to the
- # namespace
positionals = self._get_positional_actions()
a = [action for action in positionals
@@ -2458,59 +2479,7 @@ def parse_known_intermixed_args(self, args=None, namespace=None):
raise TypeError('parse_intermixed_args: positional arg'
' with nargs=%s'%a[0].nargs)
- if [action.dest for group in self._mutually_exclusive_groups
- for action in group._group_actions if action in positionals]:
- raise TypeError('parse_intermixed_args: positional in'
- ' mutuallyExclusiveGroup')
-
- try:
- save_usage = self.usage
- try:
- if self.usage is None:
- # capture the full usage for use in error messages
- self.usage = self.format_usage()[7:]
- for action in positionals:
- # deactivate positionals
- action.save_nargs = action.nargs
- # action.nargs = 0
- action.nargs = SUPPRESS
- action.save_default = action.default
- action.default = SUPPRESS
- namespace, remaining_args = self.parse_known_args(args,
- namespace)
- for action in positionals:
- # remove the empty positional values from namespace
- if (hasattr(namespace, action.dest)
- and getattr(namespace, action.dest)==[]):
- from warnings import warn
- warn('Do not expect %s in %s' % (action.dest, namespace))
- delattr(namespace, action.dest)
- finally:
- # restore nargs and usage before exiting
- for action in positionals:
- action.nargs = action.save_nargs
- action.default = action.save_default
- optionals = self._get_optional_actions()
- try:
- # parse positionals. optionals aren't normally required, but
- # they could be, so make sure they aren't.
- for action in optionals:
- action.save_required = action.required
- action.required = False
- for group in self._mutually_exclusive_groups:
- group.save_required = group.required
- group.required = False
- namespace, extras = self.parse_known_args(remaining_args,
- namespace)
- finally:
- # restore parser values before exiting
- for action in optionals:
- action.required = action.save_required
- for group in self._mutually_exclusive_groups:
- group.required = group.save_required
- finally:
- self.usage = save_usage
- return namespace, extras
+ return self._parse_known_args2(args, namespace, intermixed=True)
# ========================
# Value conversion methods
diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py
index d8a4a00292d230..7f3c74be0ce6dd 100644
--- a/Lib/test/test_argparse.py
+++ b/Lib/test/test_argparse.py
@@ -5897,12 +5897,23 @@ def test_basic(self):
# cannot parse the '1,2,3'
self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1]), args)
self.assertEqual(["2", "3"], extras)
+ args, extras = parser.parse_known_intermixed_args(argv)
+ self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args)
+ self.assertEqual([], extras)
+ # unknown optionals go into extras
+ argv = 'cmd --foo x --error 1 2 --bar y 3'.split()
+ args, extras = parser.parse_known_intermixed_args(argv)
+ self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args)
+ self.assertEqual(['--error'], extras)
argv = 'cmd --foo x 1 --error 2 --bar y 3'.split()
args, extras = parser.parse_known_intermixed_args(argv)
- # unknown optionals go into extras
- self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1]), args)
- self.assertEqual(['--error', '2', '3'], extras)
+ self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args)
+ self.assertEqual(['--error'], extras)
+ argv = 'cmd --foo x 1 2 --error --bar y 3'.split()
+ args, extras = parser.parse_known_intermixed_args(argv)
+ self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args)
+ self.assertEqual(['--error'], extras)
# restores attributes that were temporarily changed
self.assertIsNone(parser.usage)
@@ -5921,28 +5932,45 @@ def test_remainder(self):
parser.parse_intermixed_args(argv)
self.assertRegex(str(cm.exception), r'\.\.\.')
- def test_exclusive(self):
- # mutually exclusive group; intermixed works fine
- parser = ErrorRaisingArgumentParser(prog='PROG')
+ def test_required_exclusive(self):
+ # required mutually exclusive group; intermixed works fine
+ parser = argparse.ArgumentParser(prog='PROG', exit_on_error=False)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--foo', action='store_true', help='FOO')
group.add_argument('--spam', help='SPAM')
parser.add_argument('badger', nargs='*', default='X', help='BADGER')
+ args = parser.parse_intermixed_args('--foo 1 2'.split())
+ self.assertEqual(NS(badger=['1', '2'], foo=True, spam=None), args)
args = parser.parse_intermixed_args('1 --foo 2'.split())
self.assertEqual(NS(badger=['1', '2'], foo=True, spam=None), args)
- self.assertRaises(ArgumentParserError, parser.parse_intermixed_args, '1 2'.split())
+ self.assertRaisesRegex(argparse.ArgumentError,
+ 'one of the arguments --foo --spam is required',
+ parser.parse_intermixed_args, '1 2'.split())
self.assertEqual(group.required, True)
- def test_exclusive_incompatible(self):
- # mutually exclusive group including positional - fail
- parser = ErrorRaisingArgumentParser(prog='PROG')
+ def test_required_exclusive_with_positional(self):
+ # required mutually exclusive group with positional argument
+ parser = argparse.ArgumentParser(prog='PROG', exit_on_error=False)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--foo', action='store_true', help='FOO')
group.add_argument('--spam', help='SPAM')
group.add_argument('badger', nargs='*', default='X', help='BADGER')
- self.assertRaises(TypeError, parser.parse_intermixed_args, [])
+ args = parser.parse_intermixed_args(['--foo'])
+ self.assertEqual(NS(foo=True, spam=None, badger='X'), args)
+ args = parser.parse_intermixed_args(['a', 'b'])
+ self.assertEqual(NS(foo=False, spam=None, badger=['a', 'b']), args)
+ self.assertRaisesRegex(argparse.ArgumentError,
+ 'one of the arguments --foo --spam badger is required',
+ parser.parse_intermixed_args, [])
+ self.assertRaisesRegex(argparse.ArgumentError,
+ 'argument badger: not allowed with argument --foo',
+ parser.parse_intermixed_args, ['--foo', 'a', 'b'])
+ self.assertRaisesRegex(argparse.ArgumentError,
+ 'argument badger: not allowed with argument --foo',
+ parser.parse_intermixed_args, ['a', '--foo', 'b'])
self.assertEqual(group.required, True)
+
class TestIntermixedMessageContentError(TestCase):
# case where Intermixed gives different error message
# error is raised by 1st parsing step
@@ -5960,7 +5988,7 @@ def test_missing_argument_name_in_message(self):
with self.assertRaises(ArgumentParserError) as cm:
parser.parse_intermixed_args([])
msg = str(cm.exception)
- self.assertNotRegex(msg, 'req_pos')
+ self.assertRegex(msg, 'req_pos')
self.assertRegex(msg, 'req_opt')
# ==========================
diff --git a/Misc/NEWS.d/next/Library/2024-10-22-13-28-00.gh-issue-125355.zssHm_.rst b/Misc/NEWS.d/next/Library/2024-10-22-13-28-00.gh-issue-125355.zssHm_.rst
new file mode 100644
index 00000000000000..fd67f697641d92
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-22-13-28-00.gh-issue-125355.zssHm_.rst
@@ -0,0 +1,7 @@
+Fix several bugs in :meth:`argparse.ArgumentParser.parse_intermixed_args`.
+
+* The parser no longer changes temporarily during parsing.
+* Default values are not processed twice.
+* Required mutually exclusive groups containing positional arguments are now supported.
+* The missing arguments report now includes the names of all required optional and positional arguments.
+* Unknown options can be intermixed with positional arguments in parse_known_intermixed_args().
From 440ed18e08887b958ad50db1b823e692a747b671 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 23 Oct 2024 21:34:11 +0200
Subject: [PATCH 095/269] [3.12] gh-124858: fix happy eyeballs refcyles
(GH-124859) (#124913)
gh-124858: fix happy eyeballs refcyles (GH-124859)
(cherry picked from commit c066bf553577d1000e208eb078d9e758c3e41186)
Co-authored-by: Thomas Grainger
---
Lib/asyncio/base_events.py | 18 ++++++++++++------
Lib/asyncio/staggered.py | 1 +
Lib/test/test_asyncio/test_streams.py | 18 ++++++++++++++++++
...4-10-01-17-12-20.gh-issue-124858.Zy0tvT.rst | 1 +
4 files changed, 32 insertions(+), 6 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-01-17-12-20.gh-issue-124858.Zy0tvT.rst
diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py
index cb037fd472c5aa..3146f7f3f65503 100644
--- a/Lib/asyncio/base_events.py
+++ b/Lib/asyncio/base_events.py
@@ -17,7 +17,6 @@
import collections.abc
import concurrent.futures
import errno
-import functools
import heapq
import itertools
import os
@@ -1106,11 +1105,18 @@ async def create_connection(
except OSError:
continue
else: # using happy eyeballs
- sock, _, _ = await staggered.staggered_race(
- (functools.partial(self._connect_sock,
- exceptions, addrinfo, laddr_infos)
- for addrinfo in infos),
- happy_eyeballs_delay, loop=self)
+ sock = (await staggered.staggered_race(
+ (
+ # can't use functools.partial as it keeps a reference
+ # to exceptions
+ lambda addrinfo=addrinfo: self._connect_sock(
+ exceptions, addrinfo, laddr_infos
+ )
+ for addrinfo in infos
+ ),
+ happy_eyeballs_delay,
+ loop=self,
+ ))[0] # can't use sock, _, _ as it keeks a reference to exceptions
if sock is None:
exceptions = [exc for sub in exceptions for exc in sub]
diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py
index 7aafcea4d885eb..0f4df8855a80b9 100644
--- a/Lib/asyncio/staggered.py
+++ b/Lib/asyncio/staggered.py
@@ -144,6 +144,7 @@ async def run_one_coro(ok_to_start, previous_failed) -> None:
raise d.exception()
return winner_result, winner_index, exceptions
finally:
+ del exceptions
# Make sure no tasks are left running if we leave this function
for t in running_tasks:
t.cancel()
diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py
index 3c8cc5f3649180..686fef8377f4e5 100644
--- a/Lib/test/test_asyncio/test_streams.py
+++ b/Lib/test/test_asyncio/test_streams.py
@@ -1166,6 +1166,24 @@ async def handle_echo(reader, writer):
messages = self._basetest_unhandled_exceptions(handle_echo)
self.assertEqual(messages, [])
+ def test_open_connection_happy_eyeball_refcycles(self):
+ port = socket_helper.find_unused_port()
+ async def main():
+ exc = None
+ try:
+ await asyncio.open_connection(
+ host="localhost",
+ port=port,
+ happy_eyeballs_delay=0.25,
+ )
+ except* OSError as excs:
+ # can't use assertRaises because that clears frames
+ exc = excs.exceptions[0]
+ self.assertIsNotNone(exc)
+ self.assertListEqual(gc.get_referrers(exc), [])
+
+ asyncio.run(main())
+
if __name__ == '__main__':
unittest.main()
diff --git a/Misc/NEWS.d/next/Library/2024-10-01-17-12-20.gh-issue-124858.Zy0tvT.rst b/Misc/NEWS.d/next/Library/2024-10-01-17-12-20.gh-issue-124858.Zy0tvT.rst
new file mode 100644
index 00000000000000..c05d24a7c5aacb
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-01-17-12-20.gh-issue-124858.Zy0tvT.rst
@@ -0,0 +1 @@
+Fix reference cycles left in tracebacks in :func:`asyncio.open_connection` when used with ``happy_eyeballs_delay``
From 05b11ba059caced565facb95422138968c36c4d2 Mon Sep 17 00:00:00 2001
From: Tian Gao
Date: Wed, 23 Oct 2024 15:36:40 -0700
Subject: [PATCH 096/269] =?UTF-8?q?[3.12]=20gh-125884:=20Support=20breakpo?=
=?UTF-8?q?int=20on=20functions=20with=20annotations=20(G=E2=80=A6=20(#125?=
=?UTF-8?q?903)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* [3.12] gh-125884: Support breakpoint on functions with annotations (GH-125892)
(cherry picked from commit 8f2c0f7a03b71485b5635cb47c000e4e8ace8800)
Co-authored-by: Tian Gao
---
Lib/pdb.py | 2 +-
Lib/test/test_pdb.py | 36 +++++++++++++++++++
...-10-23-17-45-40.gh-issue-125884.41E_PD.rst | 1 +
3 files changed, 38 insertions(+), 1 deletion(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-23-17-45-40.gh-issue-125884.41E_PD.rst
diff --git a/Lib/pdb.py b/Lib/pdb.py
index 559517813cf600..1e1b5ea4f0a184 100755
--- a/Lib/pdb.py
+++ b/Lib/pdb.py
@@ -96,7 +96,7 @@ class Restart(Exception):
"post_mortem", "help"]
def find_function(funcname, filename):
- cre = re.compile(r'def\s+%s\s*[(]' % re.escape(funcname))
+ cre = re.compile(r'def\s+%s(\s*\[.+\])?\s*[(]' % re.escape(funcname))
try:
fp = tokenize.open(filename)
except OSError:
diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py
index 1542bb3bee1aea..bd61de0ad3494c 100644
--- a/Lib/test/test_pdb.py
+++ b/Lib/test/test_pdb.py
@@ -352,6 +352,42 @@ def test_pdb_breakpoint_commands():
4
"""
+def test_pdb_breakpoint_on_annotated_function_def():
+ """Test breakpoints on function definitions with annotation.
+
+ >>> def foo[T]():
+ ... return 0
+
+ >>> def bar() -> int:
+ ... return 0
+
+ >>> def foobar[T]() -> int:
+ ... return 0
+
+ >>> reset_Breakpoint()
+
+ >>> def test_function():
+ ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace()
+ ... pass
+
+ >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE
+ ... 'break foo',
+ ... 'break bar',
+ ... 'break foobar',
+ ... 'continue',
+ ... ]):
+ ... test_function()
+ > (3)test_function()
+ -> pass
+ (Pdb) break foo
+ Breakpoint 1 at :1
+ (Pdb) break bar
+ Breakpoint 2 at :1
+ (Pdb) break foobar
+ Breakpoint 3 at :1
+ (Pdb) continue
+ """
+
def test_pdb_breakpoints_preserved_across_interactive_sessions():
"""Breakpoints are remembered between interactive sessions
diff --git a/Misc/NEWS.d/next/Library/2024-10-23-17-45-40.gh-issue-125884.41E_PD.rst b/Misc/NEWS.d/next/Library/2024-10-23-17-45-40.gh-issue-125884.41E_PD.rst
new file mode 100644
index 00000000000000..684b1f282b143e
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-23-17-45-40.gh-issue-125884.41E_PD.rst
@@ -0,0 +1 @@
+Fixed the bug for :mod:`pdb` where it can't set breakpoints on functions with certain annotations.
From 869119e36d1b1367eff7003b04dfbe2ce7129a20 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 24 Oct 2024 18:50:16 +0200
Subject: [PATCH 097/269] [3.12] gh-125909: Avoid a redirect when linking to
the devguide (GH-125826) (#125930)
(cherry picked from commit 5003ad5c5ea508f0dde1b374cd8bc6a481ad5c5d)
Co-authored-by: partev
---
Doc/tools/templates/indexcontent.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Doc/tools/templates/indexcontent.html b/Doc/tools/templates/indexcontent.html
index f2e9fbb0106452..2686f48dad2a95 100644
--- a/Doc/tools/templates/indexcontent.html
+++ b/Doc/tools/templates/indexcontent.html
@@ -59,7 +59,7 @@ {{ docstitle|e }}
{% trans %}Reporting issues{% endtrans %}
- {% trans %}Contributing to Docs{% endtrans %}
+ {% trans %}Contributing to Docs{% endtrans %}
{% trans %}Download the documentation{% endtrans %}
{% trans %}History and license of Python{% endtrans %}
From 31ff9e5ac685dff58456f7fa93d1872e8cda604f Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 24 Oct 2024 23:00:45 +0200
Subject: [PATCH 098/269] [3.12] gh-125933: Add ARIA labels to select elements
in the version switcher (GH-125934) (#125939)
gh-125933: Add ARIA labels to select elements in the version switcher (GH-125934)
(cherry picked from commit 1306f33c84b2745aa8af5e3e8f680aa80b836c0e)
Co-authored-by: Kerim Kabirov
Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
---
Doc/tools/static/rtd_switcher.js | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/Doc/tools/static/rtd_switcher.js b/Doc/tools/static/rtd_switcher.js
index f5dc7045a0dbc4..2bf01a002db90c 100644
--- a/Doc/tools/static/rtd_switcher.js
+++ b/Doc/tools/static/rtd_switcher.js
@@ -7,7 +7,7 @@
document.addEventListener("readthedocs-addons-data-ready", function(event) {
const config = event.detail.data()
const versionSelect = `
-
+
${ config.versions.active.map(
(version) => `
a.language.name.localeCompare(b.language.name));
const languageSelect = `
-
+
${ languages.map(
(translation) => `
Date: Fri, 25 Oct 2024 16:02:46 +0200
Subject: [PATCH 099/269] [3.12] GH-125789: fix `fut._callbacks` to always
return a copy of callbacks (GH-125922) (#125977)
GH-125789: fix `fut._callbacks` to always return a copy of callbacks (GH-125922)
Fix `asyncio.Future._callbacks` to always return a copy of the internal list of callbacks to avoid mutation from user code affecting the internal state.
Co-authored-by: Kumar Aditya
(cherry picked from commit cae853e3b44cd5cb033b904e163c490dd28bc30a)
---
Lib/test/test_asyncio/test_futures.py | 18 +++++++
...-10-24-14-08-10.gh-issue-125789.eaiAMw.rst | 1 +
Modules/_asynciomodule.c | 53 +++++++++----------
3 files changed, 44 insertions(+), 28 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-24-14-08-10.gh-issue-125789.eaiAMw.rst
diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py
index 050d33f4fab3ed..60e93a268ab00d 100644
--- a/Lib/test/test_asyncio/test_futures.py
+++ b/Lib/test/test_asyncio/test_futures.py
@@ -686,6 +686,24 @@ def test_future_iter_get_referents_segfault(self):
evil = gc.get_referents(_asyncio)
gc.collect()
+ def test_callbacks_copy(self):
+ # See https://github.com/python/cpython/issues/125789
+ # In C implementation, the `_callbacks` attribute
+ # always returns a new list to avoid mutations of internal state
+
+ fut = self._new_future(loop=self.loop)
+ f1 = lambda _: 1
+ f2 = lambda _: 2
+ fut.add_done_callback(f1)
+ fut.add_done_callback(f2)
+ callbacks = fut._callbacks
+ self.assertIsNot(callbacks, fut._callbacks)
+ fut.remove_done_callback(f1)
+ callbacks = fut._callbacks
+ self.assertIsNot(callbacks, fut._callbacks)
+ fut.remove_done_callback(f2)
+ self.assertIsNone(fut._callbacks)
+
@unittest.skipUnless(hasattr(futures, '_CFuture'),
'requires the C _asyncio module')
diff --git a/Misc/NEWS.d/next/Library/2024-10-24-14-08-10.gh-issue-125789.eaiAMw.rst b/Misc/NEWS.d/next/Library/2024-10-24-14-08-10.gh-issue-125789.eaiAMw.rst
new file mode 100644
index 00000000000000..964a006bb47b7b
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-24-14-08-10.gh-issue-125789.eaiAMw.rst
@@ -0,0 +1 @@
+Fix possible crash when mutating list of callbacks returned by :attr:`!asyncio.Future._callbacks`. It now always returns a new copy in C implementation :mod:`!_asyncio`. Patch by Kumar Aditya.
diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c
index da44bb6b714578..bd4eaf083f7fed 100644
--- a/Modules/_asynciomodule.c
+++ b/Modules/_asynciomodule.c
@@ -1290,52 +1290,49 @@ static PyObject *
FutureObj_get_callbacks(FutureObj *fut, void *Py_UNUSED(ignored))
{
asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut);
- Py_ssize_t i;
-
ENSURE_FUTURE_ALIVE(state, fut)
- if (fut->fut_callback0 == NULL) {
- if (fut->fut_callbacks == NULL) {
- Py_RETURN_NONE;
- }
-
- return Py_NewRef(fut->fut_callbacks);
+ Py_ssize_t len = 0;
+ if (fut->fut_callback0 != NULL) {
+ len++;
}
-
- Py_ssize_t len = 1;
if (fut->fut_callbacks != NULL) {
len += PyList_GET_SIZE(fut->fut_callbacks);
}
-
- PyObject *new_list = PyList_New(len);
- if (new_list == NULL) {
- return NULL;
+ if (len == 0) {
+ Py_RETURN_NONE;
}
- PyObject *tup0 = PyTuple_New(2);
- if (tup0 == NULL) {
- Py_DECREF(new_list);
+ PyObject *callbacks = PyList_New(len);
+ if (callbacks == NULL) {
return NULL;
}
- Py_INCREF(fut->fut_callback0);
- PyTuple_SET_ITEM(tup0, 0, fut->fut_callback0);
- assert(fut->fut_context0 != NULL);
- Py_INCREF(fut->fut_context0);
- PyTuple_SET_ITEM(tup0, 1, (PyObject *)fut->fut_context0);
-
- PyList_SET_ITEM(new_list, 0, tup0);
+ Py_ssize_t i = 0;
+ if (fut->fut_callback0 != NULL) {
+ PyObject *tup0 = PyTuple_New(2);
+ if (tup0 == NULL) {
+ Py_DECREF(callbacks);
+ return NULL;
+ }
+ PyTuple_SET_ITEM(tup0, 0, Py_NewRef(fut->fut_callback0));
+ assert(fut->fut_context0 != NULL);
+ PyTuple_SET_ITEM(tup0, 1, Py_NewRef(fut->fut_context0));
+ PyList_SET_ITEM(callbacks, i, tup0);
+ i++;
+ }
if (fut->fut_callbacks != NULL) {
- for (i = 0; i < PyList_GET_SIZE(fut->fut_callbacks); i++) {
- PyObject *cb = PyList_GET_ITEM(fut->fut_callbacks, i);
+ for (Py_ssize_t j = 0; j < PyList_GET_SIZE(fut->fut_callbacks); j++) {
+ PyObject *cb = PyList_GET_ITEM(fut->fut_callbacks, j);
Py_INCREF(cb);
- PyList_SET_ITEM(new_list, i + 1, cb);
+ PyList_SET_ITEM(callbacks, i, cb);
+ i++;
}
}
- return new_list;
+ return callbacks;
}
static PyObject *
From f6682fbe49dfb4126f9cd8db86a0c83ed53e4391 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Fri, 25 Oct 2024 20:38:59 +0200
Subject: [PATCH 100/269] [3.12] gh-125969: fix OOB in
`future_schedule_callbacks` due to an evil `call_soon` (GH-125970) (#125992)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
gh-125969: fix OOB in `future_schedule_callbacks` due to an evil `call_soon` (GH-125970)
(cherry picked from commit c5b99f5c2c5347d66b9da362773969c531fb6c85)
Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
Co-authored-by: Andrew Svetlov
---
Lib/test/test_asyncio/test_futures.py | 33 +++++++++++++++++++
...-10-25-11-13-24.gh-issue-125969.YvbrTr.rst | 2 ++
Modules/_asynciomodule.c | 29 +++++++---------
3 files changed, 47 insertions(+), 17 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-25-11-13-24.gh-issue-125969.YvbrTr.rst
diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py
index 60e93a268ab00d..396ca75c7d3fab 100644
--- a/Lib/test/test_asyncio/test_futures.py
+++ b/Lib/test/test_asyncio/test_futures.py
@@ -936,6 +936,39 @@ def __eq__(self, other):
fut.remove_done_callback(evil())
+ def test_evil_call_soon_list_mutation(self):
+ called_on_fut_callback0 = False
+
+ pad = lambda: ...
+
+ def evil_call_soon(*args, **kwargs):
+ nonlocal called_on_fut_callback0
+ if called_on_fut_callback0:
+ # Called when handling fut->fut_callbacks[0]
+ # and mutates the length fut->fut_callbacks.
+ fut.remove_done_callback(int)
+ fut.remove_done_callback(pad)
+ else:
+ called_on_fut_callback0 = True
+
+ fake_event_loop = lambda: ...
+ fake_event_loop.call_soon = evil_call_soon
+ fake_event_loop.get_debug = lambda: False # suppress traceback
+
+ with mock.patch.object(self, 'loop', fake_event_loop):
+ fut = self._new_future()
+ self.assertIs(fut.get_loop(), fake_event_loop)
+
+ fut.add_done_callback(str) # sets fut->fut_callback0
+ fut.add_done_callback(int) # sets fut->fut_callbacks[0]
+ fut.add_done_callback(pad) # sets fut->fut_callbacks[1]
+ fut.add_done_callback(pad) # sets fut->fut_callbacks[2]
+ fut.set_result("boom")
+
+ # When there are no more callbacks, the Python implementation
+ # returns an empty list but the C implementation returns None.
+ self.assertIn(fut._callbacks, (None, []))
+
@unittest.skipUnless(hasattr(futures, '_CFuture'),
'requires the C _asyncio module')
diff --git a/Misc/NEWS.d/next/Library/2024-10-25-11-13-24.gh-issue-125969.YvbrTr.rst b/Misc/NEWS.d/next/Library/2024-10-25-11-13-24.gh-issue-125969.YvbrTr.rst
new file mode 100644
index 00000000000000..dc99adff7416c5
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-25-11-13-24.gh-issue-125969.YvbrTr.rst
@@ -0,0 +1,2 @@
+Fix an out-of-bounds crash when an evil :meth:`asyncio.loop.call_soon`
+mutates the length of the internal callbacks list. Patch by Bénédikt Tran.
diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c
index bd4eaf083f7fed..25d7ab8aa49f24 100644
--- a/Modules/_asynciomodule.c
+++ b/Modules/_asynciomodule.c
@@ -430,9 +430,6 @@ future_ensure_alive(FutureObj *fut)
static int
future_schedule_callbacks(asyncio_state *state, FutureObj *fut)
{
- Py_ssize_t len;
- Py_ssize_t i;
-
if (fut->fut_callback0 != NULL) {
/* There's a 1st callback */
@@ -458,27 +455,25 @@ future_schedule_callbacks(asyncio_state *state, FutureObj *fut)
return 0;
}
- len = PyList_GET_SIZE(fut->fut_callbacks);
- if (len == 0) {
- /* The list of callbacks was empty; clear it and return. */
- Py_CLEAR(fut->fut_callbacks);
- return 0;
- }
-
- for (i = 0; i < len; i++) {
- PyObject *cb_tup = PyList_GET_ITEM(fut->fut_callbacks, i);
+ // Beware: An evil call_soon could change fut->fut_callbacks.
+ // The idea is to transfer the ownership of the callbacks list
+ // so that external code is not able to mutate the list during
+ // the iteration.
+ PyObject *callbacks = fut->fut_callbacks;
+ fut->fut_callbacks = NULL;
+ Py_ssize_t n = PyList_GET_SIZE(callbacks);
+ for (Py_ssize_t i = 0; i < n; i++) {
+ assert(PyList_GET_SIZE(callbacks) == n);
+ PyObject *cb_tup = PyList_GET_ITEM(callbacks, i);
PyObject *cb = PyTuple_GET_ITEM(cb_tup, 0);
PyObject *ctx = PyTuple_GET_ITEM(cb_tup, 1);
if (call_soon(state, fut->fut_loop, cb, (PyObject *)fut, ctx)) {
- /* If an error occurs in pure-Python implementation,
- all callbacks are cleared. */
- Py_CLEAR(fut->fut_callbacks);
+ Py_DECREF(callbacks);
return -1;
}
}
-
- Py_CLEAR(fut->fut_callbacks);
+ Py_DECREF(callbacks);
return 0;
}
From d0b2fa4804eb7edf08fd6d97fa615e190d679d81 Mon Sep 17 00:00:00 2001
From: Ethan Furman
Date: Fri, 25 Oct 2024 16:03:52 -0700
Subject: [PATCH 101/269] [3.12] gh-125259: Fix error notes removal in enum
initialization (GH-125647) (GH-125953)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
(cherry picked from commit 34653bba644aa5481613f398153757d7357e39ea)
Co-authored-by: Mario Šaško
---
Lib/enum.py | 16 +++++-----------
Lib/test/test_enum.py | 19 +++++++++++++++++++
...-10-17-16-10-29.gh-issue-125259.oMew0c.rst | 1 +
3 files changed, 25 insertions(+), 11 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-17-16-10-29.gh-issue-125259.oMew0c.rst
diff --git a/Lib/enum.py b/Lib/enum.py
index d9859b3c0a9f98..eaa517e2fbc39b 100644
--- a/Lib/enum.py
+++ b/Lib/enum.py
@@ -592,19 +592,13 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k
classdict['_all_bits_'] = 0
classdict['_inverted_'] = None
try:
- exc = None
enum_class = super().__new__(metacls, cls, bases, classdict, **kwds)
except Exception as e:
- # since 3.12 the line "Error calling __set_name__ on '_proto_member' instance ..."
- # is tacked on to the error instead of raising a RuntimeError
- # recreate the exception to discard
- exc = type(e)(str(e))
- exc.__cause__ = e.__cause__
- exc.__context__ = e.__context__
- tb = e.__traceback__
- if exc is not None:
- raise exc.with_traceback(tb)
- #
+ # since 3.12 the note "Error calling __set_name__ on '_proto_member' instance ..."
+ # is tacked on to the error instead of raising a RuntimeError, so discard it
+ if hasattr(e, '__notes__'):
+ del e.__notes__
+ raise
# update classdict with any changes made by __init_subclass__
classdict.update(enum_class.__dict__)
#
diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py
index aff5b7f68f01bc..2e50ae0fe96586 100644
--- a/Lib/test/test_enum.py
+++ b/Lib/test/test_enum.py
@@ -1852,6 +1852,25 @@ def test_wrong_inheritance_order(self):
class Wrong(Enum, str):
NotHere = 'error before this point'
+ def test_raise_custom_error_on_creation(self):
+ class InvalidRgbColorError(ValueError):
+ def __init__(self, r, g, b):
+ self.r = r
+ self.g = g
+ self.b = b
+ super().__init__(f'({r}, {g}, {b}) is not a valid RGB color')
+
+ with self.assertRaises(InvalidRgbColorError):
+ class RgbColor(Enum):
+ RED = (255, 0, 0)
+ GREEN = (0, 255, 0)
+ BLUE = (0, 0, 255)
+ INVALID = (256, 0, 0)
+
+ def __init__(self, r, g, b):
+ if not all(0 <= val <= 255 for val in (r, g, b)):
+ raise InvalidRgbColorError(r, g, b)
+
def test_intenum_transitivity(self):
class number(IntEnum):
one = 1
diff --git a/Misc/NEWS.d/next/Library/2024-10-17-16-10-29.gh-issue-125259.oMew0c.rst b/Misc/NEWS.d/next/Library/2024-10-17-16-10-29.gh-issue-125259.oMew0c.rst
new file mode 100644
index 00000000000000..4fa6330abea512
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-17-16-10-29.gh-issue-125259.oMew0c.rst
@@ -0,0 +1 @@
+Fix the notes removal logic for errors thrown in enum initialization.
From fe4585a674daaf79f0978429daabd36b3bffb212 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sat, 26 Oct 2024 17:27:38 +0200
Subject: [PATCH 102/269] [3.12] gh-125698: Autoconf: Sync EXEEXT and ac_exeext
(GH-125995) (#126007)
(cherry picked from commit 8b7cdc5e0c3ee9903d41a2bfc494148db32dc379)
Co-authored-by: Erlend E. Aasland
---
configure | 3 +++
configure.ac | 3 +++
2 files changed, 6 insertions(+)
diff --git a/configure b/configure
index 1c75810d9e8c4b..8cca82b6bc87b4 100755
--- a/configure
+++ b/configure
@@ -7252,6 +7252,9 @@ fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $EXEEXT" >&5
printf "%s\n" "$EXEEXT" >&6; }
+# Make sure we keep EXEEXT and ac_exeext sync'ed.
+ac_exeext=$EXEEXT
+
# Test whether we're running on a non-case-sensitive system, in which
# case we give a warning if no ext is given
diff --git a/configure.ac b/configure.ac
index d0d54050286cd8..d3f0fb4d88f389 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1308,6 +1308,9 @@ AC_ARG_WITH([suffix],
])
AC_MSG_RESULT([$EXEEXT])
+# Make sure we keep EXEEXT and ac_exeext sync'ed.
+AS_VAR_SET([ac_exeext], [$EXEEXT])
+
# Test whether we're running on a non-case-sensitive system, in which
# case we give a warning if no ext is given
AC_SUBST([BUILDEXEEXT])
From 6a8e8f48b4f9be5d15ceec6869bbaef55e99f18d Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sat, 26 Oct 2024 18:32:07 +0200
Subject: [PATCH 103/269] [3.12] gh-118950: Fix SSLProtocol.connection_lost not
being called when OSError is thrown (GH-118960) (#125932)
gh-118950: Fix SSLProtocol.connection_lost not being called when OSError is thrown (GH-118960)
(cherry picked from commit 3f24bde0b6689b8f05872a8118a97908b5a94659)
Co-authored-by: Javad Shafique
Co-authored-by: Kumar Aditya
---
Lib/asyncio/sslproto.py | 5 +-
Lib/test/test_asyncio/test_sslproto.py | 48 +++++++++++++++++++
...-05-12-03-10-36.gh-issue-118950.5Wc4vp.rst | 1 +
3 files changed, 53 insertions(+), 1 deletion(-)
create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-12-03-10-36.gh-issue-118950.5Wc4vp.rst
diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py
index e51669a2ab2af6..29e72b1fd9aa24 100644
--- a/Lib/asyncio/sslproto.py
+++ b/Lib/asyncio/sslproto.py
@@ -101,7 +101,7 @@ def get_protocol(self):
return self._ssl_protocol._app_protocol
def is_closing(self):
- return self._closed
+ return self._closed or self._ssl_protocol._is_transport_closing()
def close(self):
"""Close the transport.
@@ -379,6 +379,9 @@ def _get_app_transport(self):
self._app_transport_created = True
return self._app_transport
+ def _is_transport_closing(self):
+ return self._transport is not None and self._transport.is_closing()
+
def connection_made(self, transport):
"""Called when the low-level connection is made.
diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py
index f5f0afeab51c9e..761904c5146b6a 100644
--- a/Lib/test/test_asyncio/test_sslproto.py
+++ b/Lib/test/test_asyncio/test_sslproto.py
@@ -109,6 +109,54 @@ def test_connection_lost(self):
test_utils.run_briefly(self.loop)
self.assertIsInstance(waiter.exception(), ConnectionAbortedError)
+ def test_connection_lost_when_busy(self):
+ # gh-118950: SSLProtocol.connection_lost not being called when OSError
+ # is thrown on asyncio.write.
+ sock = mock.Mock()
+ sock.fileno = mock.Mock(return_value=12345)
+ sock.send = mock.Mock(side_effect=BrokenPipeError)
+
+ # construct StreamWriter chain that contains loop dependant logic this emulates
+ # what _make_ssl_transport() does in BaseSelectorEventLoop
+ reader = asyncio.StreamReader(limit=2 ** 16, loop=self.loop)
+ protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop)
+ ssl_proto = self.ssl_protocol(proto=protocol)
+
+ # emulate reading decompressed data
+ sslobj = mock.Mock()
+ sslobj.read.side_effect = ssl.SSLWantReadError
+ sslobj.write.side_effect = ssl.SSLWantReadError
+ ssl_proto._sslobj = sslobj
+
+ # emulate outgoing data
+ data = b'An interesting message'
+
+ outgoing = mock.Mock()
+ outgoing.read = mock.Mock(return_value=data)
+ outgoing.pending = len(data)
+ ssl_proto._outgoing = outgoing
+
+ # use correct socket transport to initialize the SSLProtocol
+ self.loop._make_socket_transport(sock, ssl_proto)
+
+ transport = ssl_proto._app_transport
+ writer = asyncio.StreamWriter(transport, protocol, reader, self.loop)
+
+ async def main():
+ # writes data to transport
+ async def write():
+ writer.write(data)
+ await writer.drain()
+
+ # try to write for the first time
+ await write()
+ # try to write for the second time, this raises as the connection_lost
+ # callback should be done with error
+ with self.assertRaises(ConnectionResetError):
+ await write()
+
+ self.loop.run_until_complete(main())
+
def test_close_during_handshake(self):
# bpo-29743 Closing transport during handshake process leaks socket
waiter = self.loop.create_future()
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-12-03-10-36.gh-issue-118950.5Wc4vp.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-12-03-10-36.gh-issue-118950.5Wc4vp.rst
new file mode 100644
index 00000000000000..82be975f4d808d
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-12-03-10-36.gh-issue-118950.5Wc4vp.rst
@@ -0,0 +1 @@
+Fix bug where SSLProtocol.connection_lost wasn't getting called when OSError was thrown on writing to socket.
From 67b270142d58b8f9a64f8ae2b84a195e7e49adda Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sat, 26 Oct 2024 20:37:42 +0200
Subject: [PATCH 104/269] [3.12] gh-84545: Clarify the 'extend' action
documentation in argparse (GH-125870) (GH-125965)
(cherry picked from commit da8673da362a2135cd621ac619d3aced6bb55100)
Co-authored-by: Serhiy Storchaka
---
Doc/library/argparse.rst | 26 +++++++++++++++-----------
1 file changed, 15 insertions(+), 11 deletions(-)
diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index 87d9a45539a1dd..f72bd14d56fe0d 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -690,6 +690,21 @@ how the command-line arguments should be handled. The supplied actions are:
>>> parser.parse_args('--str --int'.split())
Namespace(types=[, ])
+* ``'extend'`` - This stores a list and appends each item from the multi-value
+ argument list to it.
+ The ``'extend'`` action is typically used with the nargs_ keyword argument
+ value ``'+'`` or ``'*'``.
+ Note that when nargs_ is ``None`` (the default) or ``'?'``, each
+ character of the argument string will be appended to the list.
+ Example usage::
+
+ >>> parser = argparse.ArgumentParser()
+ >>> parser.add_argument("--foo", action="extend", nargs="+", type=str)
+ >>> parser.parse_args(["--foo", "f1", "--foo", "f2", "f3", "f4"])
+ Namespace(foo=['f1', 'f2', 'f3', 'f4'])
+
+ .. versionadded:: 3.8
+
* ``'count'`` - This counts the number of times a keyword argument occurs. For
example, this is useful for increasing verbosity levels::
@@ -715,17 +730,6 @@ how the command-line arguments should be handled. The supplied actions are:
>>> parser.parse_args(['--version'])
PROG 2.0
-* ``'extend'`` - This stores a list, and extends each argument value to the
- list.
- Example usage::
-
- >>> parser = argparse.ArgumentParser()
- >>> parser.add_argument("--foo", action="extend", nargs="+", type=str)
- >>> parser.parse_args(["--foo", "f1", "--foo", "f2", "f3", "f4"])
- Namespace(foo=['f1', 'f2', 'f3', 'f4'])
-
- .. versionadded:: 3.8
-
Only actions that consume command-line arguments (e.g. ``'store'``,
``'append'`` or ``'extend'``) can be used with positional arguments.
From fdedb2618a7ed12977ec8c06b82d7296c18bd84c Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sun, 27 Oct 2024 16:23:07 +0100
Subject: [PATCH 105/269] [3.12] gh-125984: fix use-after-free on
`fut->fut_{callback,context}0` due to an evil `loop.__getattribute__`
(GH-126003) (#126044)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
gh-125984: fix use-after-free on `fut->fut_{callback,context}0` due to an evil `loop.__getattribute__` (GH-126003)
(cherry picked from commit f819d4301d7c75f02be1187fda017f0e7b608816)
Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
---
Lib/test/test_asyncio/test_futures.py | 73 ++++++++++++++++++-
...-10-26-12-50-48.gh-issue-125984.d4vp5_.rst | 3 +
Modules/_asynciomodule.c | 19 +++--
3 files changed, 87 insertions(+), 8 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-26-12-50-48.gh-issue-125984.d4vp5_.rst
diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py
index 396ca75c7d3fab..63804256ec7726 100644
--- a/Lib/test/test_asyncio/test_futures.py
+++ b/Lib/test/test_asyncio/test_futures.py
@@ -31,6 +31,25 @@ def last_cb():
pass
+class ReachableCode(Exception):
+ """Exception to raise to indicate that some code was reached.
+
+ Use this exception if using mocks is not a good alternative.
+ """
+
+
+class SimpleEvilEventLoop(asyncio.base_events.BaseEventLoop):
+ """Base class for UAF and other evil stuff requiring an evil event loop."""
+
+ def get_debug(self): # to suppress tracebacks
+ return False
+
+ def __del__(self):
+ # Automatically close the evil event loop to avoid warnings.
+ if not self.is_closed() and not self.is_running():
+ self.close()
+
+
class DuckFuture:
# Class that does not inherit from Future but aims to be duck-type
# compatible with it.
@@ -937,6 +956,7 @@ def __eq__(self, other):
fut.remove_done_callback(evil())
def test_evil_call_soon_list_mutation(self):
+ # see: https://github.com/python/cpython/issues/125969
called_on_fut_callback0 = False
pad = lambda: ...
@@ -951,9 +971,8 @@ def evil_call_soon(*args, **kwargs):
else:
called_on_fut_callback0 = True
- fake_event_loop = lambda: ...
+ fake_event_loop = SimpleEvilEventLoop()
fake_event_loop.call_soon = evil_call_soon
- fake_event_loop.get_debug = lambda: False # suppress traceback
with mock.patch.object(self, 'loop', fake_event_loop):
fut = self._new_future()
@@ -969,6 +988,56 @@ def evil_call_soon(*args, **kwargs):
# returns an empty list but the C implementation returns None.
self.assertIn(fut._callbacks, (None, []))
+ def test_use_after_free_on_fut_callback_0_with_evil__getattribute__(self):
+ # see: https://github.com/python/cpython/issues/125984
+
+ class EvilEventLoop(SimpleEvilEventLoop):
+ def call_soon(self, *args, **kwargs):
+ super().call_soon(*args, **kwargs)
+ raise ReachableCode
+
+ def __getattribute__(self, name):
+ nonlocal fut_callback_0
+ if name == 'call_soon':
+ fut.remove_done_callback(fut_callback_0)
+ del fut_callback_0
+ return object.__getattribute__(self, name)
+
+ evil_loop = EvilEventLoop()
+ with mock.patch.object(self, 'loop', evil_loop):
+ fut = self._new_future()
+ self.assertIs(fut.get_loop(), evil_loop)
+
+ fut_callback_0 = lambda: ...
+ fut.add_done_callback(fut_callback_0)
+ self.assertRaises(ReachableCode, fut.set_result, "boom")
+
+ def test_use_after_free_on_fut_context_0_with_evil__getattribute__(self):
+ # see: https://github.com/python/cpython/issues/125984
+
+ class EvilEventLoop(SimpleEvilEventLoop):
+ def call_soon(self, *args, **kwargs):
+ super().call_soon(*args, **kwargs)
+ raise ReachableCode
+
+ def __getattribute__(self, name):
+ if name == 'call_soon':
+ # resets the future's event loop
+ fut.__init__(loop=SimpleEvilEventLoop())
+ return object.__getattribute__(self, name)
+
+ evil_loop = EvilEventLoop()
+ with mock.patch.object(self, 'loop', evil_loop):
+ fut = self._new_future()
+ self.assertIs(fut.get_loop(), evil_loop)
+
+ fut_callback_0 = mock.Mock()
+ fut_context_0 = mock.Mock()
+ fut.add_done_callback(fut_callback_0, context=fut_context_0)
+ del fut_context_0
+ del fut_callback_0
+ self.assertRaises(ReachableCode, fut.set_result, "boom")
+
@unittest.skipUnless(hasattr(futures, '_CFuture'),
'requires the C _asyncio module')
diff --git a/Misc/NEWS.d/next/Library/2024-10-26-12-50-48.gh-issue-125984.d4vp5_.rst b/Misc/NEWS.d/next/Library/2024-10-26-12-50-48.gh-issue-125984.d4vp5_.rst
new file mode 100644
index 00000000000000..7a1d7b53b11301
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-26-12-50-48.gh-issue-125984.d4vp5_.rst
@@ -0,0 +1,3 @@
+Fix use-after-free crashes on :class:`asyncio.Future` objects for which the
+underlying event loop implements an evil :meth:`~object.__getattribute__`.
+Reported by Nico-Posada. Patch by Bénédikt Tran.
diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c
index 25d7ab8aa49f24..8787fe4ac1cc51 100644
--- a/Modules/_asynciomodule.c
+++ b/Modules/_asynciomodule.c
@@ -433,12 +433,19 @@ future_schedule_callbacks(asyncio_state *state, FutureObj *fut)
if (fut->fut_callback0 != NULL) {
/* There's a 1st callback */
- int ret = call_soon(state,
- fut->fut_loop, fut->fut_callback0,
- (PyObject *)fut, fut->fut_context0);
-
- Py_CLEAR(fut->fut_callback0);
- Py_CLEAR(fut->fut_context0);
+ // Beware: An evil call_soon could alter fut_callback0 or fut_context0.
+ // Since we are anyway clearing them after the call, whether call_soon
+ // succeeds or not, the idea is to transfer ownership so that external
+ // code is not able to alter them during the call.
+ PyObject *fut_callback0 = fut->fut_callback0;
+ fut->fut_callback0 = NULL;
+ PyObject *fut_context0 = fut->fut_context0;
+ fut->fut_context0 = NULL;
+
+ int ret = call_soon(state, fut->fut_loop, fut_callback0,
+ (PyObject *)fut, fut_context0);
+ Py_CLEAR(fut_callback0);
+ Py_CLEAR(fut_context0);
if (ret) {
/* If an error occurs in pure-Python implementation,
all callbacks are cleared. */
From 4fc1da1b2757c49f9e6eda9cbc7513cc2ed5180c Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sun, 27 Oct 2024 18:32:11 +0100
Subject: [PATCH 106/269] [3.12] gh-125966: fix use-after-free on
`fut->fut_callback0` due to an evil callback's `__eq__` in asyncio
(GH-125967) (#126048)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
gh-125966: fix use-after-free on `fut->fut_callback0` due to an evil callback's `__eq__` in asyncio (GH-125967)
(cherry picked from commit ed5059eeb1aa50b481957307db5a34b937497382)
Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
---
Lib/test/test_asyncio/test_futures.py | 18 ++++++++++++++++++
...4-10-25-10-53-56.gh-issue-125966.eOCYU_.rst | 2 ++
Modules/_asynciomodule.c | 7 ++++++-
3 files changed, 26 insertions(+), 1 deletion(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst
diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py
index 63804256ec7726..8be6dcac548ec0 100644
--- a/Lib/test/test_asyncio/test_futures.py
+++ b/Lib/test/test_asyncio/test_futures.py
@@ -988,6 +988,24 @@ def evil_call_soon(*args, **kwargs):
# returns an empty list but the C implementation returns None.
self.assertIn(fut._callbacks, (None, []))
+ def test_use_after_free_on_fut_callback_0_with_evil__eq__(self):
+ # Special thanks to Nico-Posada for the original PoC.
+ # See https://github.com/python/cpython/issues/125966.
+
+ fut = self._new_future()
+
+ class cb_pad:
+ def __eq__(self, other):
+ return True
+
+ class evil(cb_pad):
+ def __eq__(self, other):
+ fut.remove_done_callback(None)
+ return NotImplemented
+
+ fut.add_done_callback(cb_pad())
+ fut.remove_done_callback(evil())
+
def test_use_after_free_on_fut_callback_0_with_evil__getattribute__(self):
# see: https://github.com/python/cpython/issues/125984
diff --git a/Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst b/Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst
new file mode 100644
index 00000000000000..9fe8795de18003
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst
@@ -0,0 +1,2 @@
+Fix a use-after-free crash in :meth:`asyncio.Future.remove_done_callback`.
+Patch by Bénédikt Tran.
diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c
index 8787fe4ac1cc51..9016c077e0d657 100644
--- a/Modules/_asynciomodule.c
+++ b/Modules/_asynciomodule.c
@@ -1044,7 +1044,12 @@ _asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls,
ENSURE_FUTURE_ALIVE(state, self)
if (self->fut_callback0 != NULL) {
- int cmp = PyObject_RichCompareBool(self->fut_callback0, fn, Py_EQ);
+ // Beware: An evil PyObject_RichCompareBool could free fut_callback0
+ // before a recursive call is made with that same arg. For details, see
+ // https://github.com/python/cpython/pull/125967#discussion_r1816593340.
+ PyObject *fut_callback0 = Py_NewRef(self->fut_callback0);
+ int cmp = PyObject_RichCompareBool(fut_callback0, fn, Py_EQ);
+ Py_DECREF(fut_callback0);
if (cmp == -1) {
return NULL;
}
From 4c039c0723d1db889269f89208751647999a7f43 Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka
Date: Sun, 27 Oct 2024 20:57:11 +0200
Subject: [PATCH 107/269] [3.12] gh-124295: Add translation tests for argparse
(GH-124803) (GH-126046) (GH-126054)
(cherry picked from commit 0922a4ae0d2803e3a4e9f3d2ccd217364cfd700e)
(cherry picked from commit ff044ed8004e31c1896ca641b81b13ab04e92837)
Co-authored-by: Tomas R.
---
Lib/test/test_argparse.py | 64 +++++++++++++++++++
Lib/test/translationdata/argparse/msgids.txt | 36 +++++++++++
Makefile.pre.in | 2 +
...-09-30-22-52-44.gh-issue-124295.VZy5kx.rst | 1 +
4 files changed, 103 insertions(+)
create mode 100644 Lib/test/translationdata/argparse/msgids.txt
create mode 100644 Misc/NEWS.d/next/Tests/2024-09-30-22-52-44.gh-issue-124295.VZy5kx.rst
diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py
index 7f3c74be0ce6dd..c8adc640f2ccaf 100644
--- a/Lib/test/test_argparse.py
+++ b/Lib/test/test_argparse.py
@@ -6,8 +6,10 @@
import io
import operator
import os
+import re
import shutil
import stat
+import subprocess
import sys
import textwrap
import tempfile
@@ -16,7 +18,15 @@
import warnings
from enum import StrEnum
+from pathlib import Path
+from test.support import REPO_ROOT
+from test.support import TEST_HOME_DIR
+from test.support import captured_stderr
+from test.support import import_helper
from test.support import os_helper
+from test.support import requires_subprocess
+from test.support import script_helper
+from test.test_tools import skip_if_missing
from unittest import mock
@@ -6388,6 +6398,56 @@ def test_os_error(self):
self.parser.parse_args, ['@no-such-file'])
+# =================
+# Translation tests
+# =================
+
+pygettext = Path(REPO_ROOT) / 'Tools' / 'i18n' / 'pygettext.py'
+snapshot_path = Path(TEST_HOME_DIR) / 'translationdata' / 'argparse' / 'msgids.txt'
+
+msgid_pattern = re.compile(r'msgid(.*?)(?:msgid_plural|msgctxt|msgstr)', re.DOTALL)
+msgid_string_pattern = re.compile(r'"((?:\\"|[^"])*)"')
+
+
+@requires_subprocess()
+class TestTranslations(unittest.TestCase):
+
+ def test_translations(self):
+ # Test messages extracted from the argparse module against a snapshot
+ skip_if_missing('i18n')
+ res = generate_po_file(stdout_only=False)
+ self.assertEqual(res.returncode, 0)
+ self.assertEqual(res.stderr, '')
+ msgids = extract_msgids(res.stdout)
+ snapshot = snapshot_path.read_text().splitlines()
+ self.assertListEqual(msgids, snapshot)
+
+
+def generate_po_file(*, stdout_only=True):
+ res = subprocess.run([sys.executable, pygettext,
+ '--no-location', '-o', '-', argparse.__file__],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+ if stdout_only:
+ return res.stdout
+ return res
+
+
+def extract_msgids(po):
+ msgids = []
+ for msgid in msgid_pattern.findall(po):
+ msgid_string = ''.join(msgid_string_pattern.findall(msgid))
+ msgid_string = msgid_string.replace(r'\"', '"')
+ if msgid_string:
+ msgids.append(msgid_string)
+ return sorted(msgids)
+
+
+def update_translation_snapshots():
+ contents = generate_po_file()
+ msgids = extract_msgids(contents)
+ snapshot_path.write_text('\n'.join(msgids))
+
+
def tearDownModule():
# Remove global references to avoid looking like we have refleaks.
RFile.seen = {}
@@ -6395,4 +6455,8 @@ def tearDownModule():
if __name__ == '__main__':
+ # To regenerate translation snapshots
+ if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update':
+ update_translation_snapshots()
+ sys.exit(0)
unittest.main()
diff --git a/Lib/test/translationdata/argparse/msgids.txt b/Lib/test/translationdata/argparse/msgids.txt
new file mode 100644
index 00000000000000..a1b4f94d8487b7
--- /dev/null
+++ b/Lib/test/translationdata/argparse/msgids.txt
@@ -0,0 +1,36 @@
+ (default: %(default)s)
+%(heading)s:
+%(prog)s: error: %(message)s\n
+%r is not callable
+'required' is an invalid argument for positionals
+.__call__() not defined
+ambiguous option: %(option)s could match %(matches)s
+argument "-" with mode %r
+argument %(argument_name)s: %(message)s
+can't open '%(filename)s': %(error)s
+cannot have multiple subparser arguments
+cannot merge actions - two groups are named %r
+conflicting subparser alias: %s
+conflicting subparser: %s
+dest= is required for options like %r
+expected at least one argument
+expected at most one argument
+expected one argument
+ignored explicit argument %r
+invalid %(type)s value: %(value)r
+invalid choice: %(value)r (choose from %(choices)s)
+invalid conflict_resolution value: %r
+invalid option string %(option)r: must start with a character %(prefix_chars)r
+mutually exclusive arguments must be optional
+not allowed with argument %s
+one of the arguments %s is required
+options
+positional arguments
+show program's version number and exit
+show this help message and exit
+subcommands
+the following arguments are required: %s
+unexpected option string: %s
+unknown parser %(parser_name)r (choices: %(choices)s)
+unrecognized arguments: %s
+usage:
\ No newline at end of file
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 0e64ccc5c21351..14eea08dcfbd33 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -2246,6 +2246,8 @@ TESTSUBDIRS= idlelib/idle_test \
test/test_zoneinfo/data \
test/tokenizedata \
test/tracedmodules \
+ test/translationdata \
+ test/translationdata/argparse \
test/typinganndata \
test/wheeldata \
test/xmltestdata \
diff --git a/Misc/NEWS.d/next/Tests/2024-09-30-22-52-44.gh-issue-124295.VZy5kx.rst b/Misc/NEWS.d/next/Tests/2024-09-30-22-52-44.gh-issue-124295.VZy5kx.rst
new file mode 100644
index 00000000000000..3c2455cfc8c530
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2024-09-30-22-52-44.gh-issue-124295.VZy5kx.rst
@@ -0,0 +1 @@
+Add translation tests to the :mod:`argparse` module.
From 9f35fbc068a1bcaf7eca5dae3ef75eb8449bd9c8 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sun, 27 Oct 2024 23:12:20 +0100
Subject: [PATCH 108/269] [3.12] gh-126035: add missing whitespace to
*Py_EnterRecursiveCall() messages (GH-126036) (#126059)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
(cherry picked from commit 19e93e2e269889ecb3c4c039091abff489f247c2)
Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
---
Modules/_bisectmodule.c | 4 ++--
Modules/_ctypes/_ctypes.c | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c
index 0773bbd191931d..d79946c84061ae 100644
--- a/Modules/_bisectmodule.c
+++ b/Modules/_bisectmodule.c
@@ -66,7 +66,7 @@ internal_bisect_right(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t
if (sq_item == NULL) {
return -1;
}
- if (Py_EnterRecursiveCall("in _bisect.bisect_right")) {
+ if (Py_EnterRecursiveCall(" in _bisect.bisect_right")) {
return -1;
}
PyTypeObject *tp = Py_TYPE(item);
@@ -250,7 +250,7 @@ internal_bisect_left(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t h
if (sq_item == NULL) {
return -1;
}
- if (Py_EnterRecursiveCall("in _bisect.bisect_left")) {
+ if (Py_EnterRecursiveCall(" in _bisect.bisect_left")) {
return -1;
}
PyTypeObject *tp = Py_TYPE(item);
diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
index d72614d2b8d972..b6d45e92647684 100644
--- a/Modules/_ctypes/_ctypes.c
+++ b/Modules/_ctypes/_ctypes.c
@@ -2269,7 +2269,7 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value)
return NULL;
}
if (as_parameter) {
- if (_Py_EnterRecursiveCall("while processing _as_parameter_")) {
+ if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) {
Py_DECREF(as_parameter);
Py_XDECREF(exc);
return NULL;
From 05214e66b0e8d8e6c9255058903c3bfd75849a42 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 28 Oct 2024 14:21:05 +0100
Subject: [PATCH 109/269] [3.12] gh-120313: amend documentation regarding
`ctypes._CFuncPtr` (GH-120989) (GH-125978)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
gh-120313: amend documentation regarding `ctypes._CFuncPtr` (GH-120989)
(cherry picked from commit 417c130ba55ca29e132808a0a500329f73b6ec41)
Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
---
Doc/library/ctypes.rst | 48 +++++++++++++++++++++++++-----------------
1 file changed, 29 insertions(+), 19 deletions(-)
diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst
index 64b2cb2703b114..9ebf9d8f751a96 100644
--- a/Doc/library/ctypes.rst
+++ b/Doc/library/ctypes.rst
@@ -383,7 +383,7 @@ as calling functions with a fixed number of parameters. On some platforms, and i
particular ARM64 for Apple Platforms, the calling convention for variadic functions
is different than that for regular functions.
-On those platforms it is required to specify the :attr:`~_FuncPtr.argtypes`
+On those platforms it is required to specify the :attr:`~_CFuncPtr.argtypes`
attribute for the regular, non-variadic, function arguments:
.. code-block:: python3
@@ -391,7 +391,7 @@ attribute for the regular, non-variadic, function arguments:
libc.printf.argtypes = [ctypes.c_char_p]
Because specifying the attribute does not inhibit portability it is advised to always
-specify :attr:`~_FuncPtr.argtypes` for all variadic functions.
+specify :attr:`~_CFuncPtr.argtypes` for all variadic functions.
.. _ctypes-calling-functions-with-own-custom-data-types:
@@ -426,9 +426,9 @@ Specifying the required argument types (function prototypes)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
It is possible to specify the required argument types of functions exported from
-DLLs by setting the :attr:`~_FuncPtr.argtypes` attribute.
+DLLs by setting the :attr:`~_CFuncPtr.argtypes` attribute.
-:attr:`~_FuncPtr.argtypes` must be a sequence of C data types (the :func:`!printf` function is
+:attr:`~_CFuncPtr.argtypes` must be a sequence of C data types (the :func:`!printf` function is
probably not a good example here, because it takes a variable number and
different types of parameters depending on the format string, on the other hand
this is quite handy to experiment with this feature)::
@@ -453,7 +453,7 @@ prototype for a C function), and tries to convert the arguments to valid types::
If you have defined your own classes which you pass to function calls, you have
to implement a :meth:`~_CData.from_param` class method for them to be able to use them
-in the :attr:`~_FuncPtr.argtypes` sequence. The :meth:`~_CData.from_param` class method receives
+in the :attr:`~_CFuncPtr.argtypes` sequence. The :meth:`~_CData.from_param` class method receives
the Python object passed to the function call, it should do a typecheck or
whatever is needed to make sure this object is acceptable, and then return the
object itself, its :attr:`!_as_parameter_` attribute, or whatever you want to
@@ -476,7 +476,7 @@ Return types
By default functions are assumed to return the C :c:expr:`int` type. Other
-return types can be specified by setting the :attr:`~_FuncPtr.restype` attribute of the
+return types can be specified by setting the :attr:`~_CFuncPtr.restype` attribute of the
function object.
The C prototype of :c:func:`time` is ``time_t time(time_t *)``. Because :c:type:`time_t`
@@ -485,7 +485,7 @@ specify the :attr:`!restype` attribute::
>>> libc.time.restype = c_time_t
-The argument types can be specified using :attr:`~_FuncPtr.argtypes`::
+The argument types can be specified using :attr:`~_CFuncPtr.argtypes`::
>>> libc.time.argtypes = (POINTER(c_time_t),)
@@ -508,7 +508,7 @@ a string pointer and a char, and returns a pointer to a string::
>>>
If you want to avoid the :func:`ord("x") ` calls above, you can set the
-:attr:`~_FuncPtr.argtypes` attribute, and the second argument will be converted from a
+:attr:`~_CFuncPtr.argtypes` attribute, and the second argument will be converted from a
single character Python bytes object into a C char:
.. doctest::
@@ -527,7 +527,7 @@ single character Python bytes object into a C char:
>>>
You can also use a callable Python object (a function or a class for example) as
-the :attr:`~_FuncPtr.restype` attribute, if the foreign function returns an integer. The
+the :attr:`~_CFuncPtr.restype` attribute, if the foreign function returns an integer. The
callable will be called with the *integer* the C function returns, and the
result of this call will be used as the result of your function call. This is
useful to check for error return values and automatically raise an exception::
@@ -555,7 +555,7 @@ get the string representation of an error code, and *returns* an exception.
:func:`GetLastError` to retrieve it.
Please note that a much more powerful error checking mechanism is available
-through the :attr:`~_FuncPtr.errcheck` attribute;
+through the :attr:`~_CFuncPtr.errcheck` attribute;
see the reference manual for details.
@@ -855,7 +855,7 @@ Type conversions
^^^^^^^^^^^^^^^^
Usually, ctypes does strict type checking. This means, if you have
-``POINTER(c_int)`` in the :attr:`~_FuncPtr.argtypes` list of a function or as the type of
+``POINTER(c_int)`` in the :attr:`~_CFuncPtr.argtypes` list of a function or as the type of
a member field in a structure definition, only instances of exactly the same
type are accepted. There are some exceptions to this rule, where ctypes accepts
other objects. For example, you can pass compatible array instances instead of
@@ -876,7 +876,7 @@ pointer types. So, for ``POINTER(c_int)``, ctypes accepts an array of c_int::
>>>
In addition, if a function argument is explicitly declared to be a pointer type
-(such as ``POINTER(c_int)``) in :attr:`~_FuncPtr.argtypes`, an object of the pointed
+(such as ``POINTER(c_int)``) in :attr:`~_CFuncPtr.argtypes`, an object of the pointed
type (``c_int`` in this case) can be passed to the function. ctypes will apply
the required :func:`byref` conversion in this case automatically.
@@ -1604,10 +1604,20 @@ As explained in the previous section, foreign functions can be accessed as
attributes of loaded shared libraries. The function objects created in this way
by default accept any number of arguments, accept any ctypes data instances as
arguments, and return the default result type specified by the library loader.
-They are instances of a private class:
+They are instances of a private local class :class:`!_FuncPtr` (not exposed
+in :mod:`!ctypes`) which inherits from the private :class:`_CFuncPtr` class:
-.. class:: _FuncPtr
+.. doctest::
+
+ >>> import ctypes
+ >>> lib = ctypes.CDLL(None)
+ >>> issubclass(lib._FuncPtr, ctypes._CFuncPtr)
+ True
+ >>> lib._FuncPtr is ctypes._CFuncPtr
+ False
+
+.. class:: _CFuncPtr
Base class for C callable foreign functions.
@@ -1773,7 +1783,7 @@ different ways, depending on the type and number of the parameters in the call:
The optional *paramflags* parameter creates foreign function wrappers with much
more functionality than the features described above.
-*paramflags* must be a tuple of the same length as :attr:`~_FuncPtr.argtypes`.
+*paramflags* must be a tuple of the same length as :attr:`~_CFuncPtr.argtypes`.
Each item in this tuple contains further information about a parameter, it must
be a tuple containing one, two, or three items.
@@ -1844,7 +1854,7 @@ value if there is a single one, or a tuple containing the output parameter
values when there are more than one, so the GetWindowRect function now returns a
RECT instance, when called.
-Output parameters can be combined with the :attr:`~_FuncPtr.errcheck` protocol to do
+Output parameters can be combined with the :attr:`~_CFuncPtr.errcheck` protocol to do
further output processing and error checking. The win32 ``GetWindowRect`` api
function returns a ``BOOL`` to signal success or failure, so this function could
do the error checking, and raises an exception when the api call failed::
@@ -1857,7 +1867,7 @@ do the error checking, and raises an exception when the api call failed::
>>> GetWindowRect.errcheck = errcheck
>>>
-If the :attr:`~_FuncPtr.errcheck` function returns the argument tuple it receives
+If the :attr:`~_CFuncPtr.errcheck` function returns the argument tuple it receives
unchanged, :mod:`ctypes` continues the normal processing it does on the output
parameters. If you want to return a tuple of window coordinates instead of a
``RECT`` instance, you can retrieve the fields in the function and return them
@@ -2157,7 +2167,7 @@ Data types
This method adapts *obj* to a ctypes type. It is called with the actual
object used in a foreign function call when the type is present in the
- foreign function's :attr:`~_FuncPtr.argtypes` tuple;
+ foreign function's :attr:`~_CFuncPtr.argtypes` tuple;
it must return an object that can be used as a function call parameter.
All ctypes data types have a default implementation of this classmethod
@@ -2223,7 +2233,7 @@ Fundamental data types
Fundamental data types, when returned as foreign function call results, or, for
example, by retrieving structure field members or array items, are transparently
converted to native Python types. In other words, if a foreign function has a
-:attr:`~_FuncPtr.restype` of :class:`c_char_p`, you will always receive a Python bytes
+:attr:`~_CFuncPtr.restype` of :class:`c_char_p`, you will always receive a Python bytes
object, *not* a :class:`c_char_p` instance.
.. XXX above is false, it actually returns a Unicode string
From 1e01dcf429e498ef592b4152b9663fbf263ea54c Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 28 Oct 2024 14:30:29 +0100
Subject: [PATCH 110/269] [3.12] gh-121277: Allow `.. versionadded:: next` in
docs (GH-121278) (GH-125980)
Make `versionchanged:: next`` expand to current (unreleased) version.
When a new CPython release is cut, the release manager will replace
all such occurences of "next" with the just-released version.
(See the issue for release-tools and devguide PRs.)
(cherry picked from commit 7d24ea9db3e8fdca52058629c9ba577aba3d8e5c)
Also backports a minor fix-up:
gh-121277: Raise nice error on `next` as second argument to deprecated-removed (GH-124623)
(cherry-picked from commit e349f73a5ad2856b0a7cbe4aef7cc081c7aed777)
Co-authored-by: Petr Viktorin
---
Doc/tools/extensions/pyspecific.py | 27 +++++++++++++++++--
...-07-19-12-22-48.gh-issue-121277.wF_zKd.rst | 2 ++
2 files changed, 27 insertions(+), 2 deletions(-)
create mode 100644 Misc/NEWS.d/next/Documentation/2024-07-19-12-22-48.gh-issue-121277.wF_zKd.rst
diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py
index 8cb5eaffebfad0..615c7f1f30fc6c 100644
--- a/Doc/tools/extensions/pyspecific.py
+++ b/Doc/tools/extensions/pyspecific.py
@@ -184,7 +184,22 @@ def run(self):
return PyMethod.run(self)
-# Support for documenting version of removal in deprecations
+# Support for documenting version of changes, additions, deprecations
+
+def expand_version_arg(argument, release):
+ """Expand "next" to the current version"""
+ if argument == 'next':
+ return sphinx_gettext('{} (unreleased)').format(release)
+ return argument
+
+
+class PyVersionChange(VersionChange):
+ def run(self):
+ # Replace the 'next' special token with the current development version
+ self.arguments[0] = expand_version_arg(self.arguments[0],
+ self.config.release)
+ return super().run()
+
class DeprecatedRemoved(VersionChange):
required_arguments = 2
@@ -195,8 +210,12 @@ class DeprecatedRemoved(VersionChange):
def run(self):
# Replace the first two arguments (deprecated version and removed version)
# with a single tuple of both versions.
- version_deprecated = self.arguments[0]
+ version_deprecated = expand_version_arg(self.arguments[0],
+ self.config.release)
version_removed = self.arguments.pop(1)
+ if version_removed == 'next':
+ raise ValueError(
+ 'deprecated-removed:: second argument cannot be `next`')
self.arguments[0] = version_deprecated, version_removed
# Set the label based on if we have reached the removal version
@@ -398,6 +417,10 @@ def setup(app):
app.add_role('issue', issue_role)
app.add_role('gh', gh_issue_role)
app.add_directive('impl-detail', ImplementationDetail)
+ app.add_directive('versionadded', PyVersionChange, override=True)
+ app.add_directive('versionchanged', PyVersionChange, override=True)
+ app.add_directive('versionremoved', PyVersionChange, override=True)
+ app.add_directive('deprecated', PyVersionChange, override=True)
app.add_directive('deprecated-removed', DeprecatedRemoved)
app.add_builder(PydocTopicsBuilder)
app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature)
diff --git a/Misc/NEWS.d/next/Documentation/2024-07-19-12-22-48.gh-issue-121277.wF_zKd.rst b/Misc/NEWS.d/next/Documentation/2024-07-19-12-22-48.gh-issue-121277.wF_zKd.rst
new file mode 100644
index 00000000000000..60f75ae0c21326
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2024-07-19-12-22-48.gh-issue-121277.wF_zKd.rst
@@ -0,0 +1,2 @@
+Writers of CPython's documentation can now use ``next`` as the version for
+the ``versionchanged``, ``versionadded``, ``deprecated`` directives.
From d89283b3e7591c0a6cac8f22efbb5862ba83c431 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 28 Oct 2024 15:25:00 +0100
Subject: [PATCH 111/269] [3.12] gh-124594: Create and reuse the same context
for the entire asyncio REPL session (GH-124595) (#124849)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* gh-124594: Create and reuse the same context for the entire asyncio REPL session (GH-124595)
(cherry picked from commit 67e01a430f4ecfcb540d6a29b347966ff4e53454)
Co-authored-by: Bartosz Sławecki
Co-authored-by: Andrew Svetlov
---------
Co-authored-by: Bartosz Sławecki
Co-authored-by: Andrew Svetlov
---
Lib/asyncio/__main__.py | 6 ++-
Lib/test/test_repl.py | 37 +++++++++++++++++++
...-09-26-13-43-39.gh-issue-124594.peYhsP.rst | 1 +
3 files changed, 42 insertions(+), 2 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-09-26-13-43-39.gh-issue-124594.peYhsP.rst
diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py
index 046558011513cb..29e528aeed754f 100644
--- a/Lib/asyncio/__main__.py
+++ b/Lib/asyncio/__main__.py
@@ -2,6 +2,7 @@
import asyncio
import code
import concurrent.futures
+import contextvars
import inspect
import sys
import threading
@@ -17,6 +18,7 @@ def __init__(self, locals, loop):
super().__init__(locals)
self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT
self.loop = loop
+ self.context = contextvars.copy_context()
def runcode(self, code):
future = concurrent.futures.Future()
@@ -46,12 +48,12 @@ def callback():
return
try:
- repl_future = self.loop.create_task(coro)
+ repl_future = self.loop.create_task(coro, context=self.context)
futures._chain_future(repl_future, future)
except BaseException as exc:
future.set_exception(exc)
- loop.call_soon_threadsafe(callback)
+ loop.call_soon_threadsafe(callback, context=self.context)
try:
return future.result()
diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py
index ddb4aa68048df1..f2f18a2581cb13 100644
--- a/Lib/test/test_repl.py
+++ b/Lib/test/test_repl.py
@@ -146,5 +146,42 @@ def f():
self.assertEqual(traceback_lines, expected_lines)
+class TestAsyncioREPLContextVars(unittest.TestCase):
+ def test_toplevel_contextvars_sync(self):
+ user_input = dedent("""\
+ from contextvars import ContextVar
+ var = ContextVar("var", default="failed")
+ var.set("ok")
+ """)
+ p = spawn_repl("-m", "asyncio")
+ p.stdin.write(user_input)
+ user_input2 = dedent("""
+ print(f"toplevel contextvar test: {var.get()}")
+ """)
+ p.stdin.write(user_input2)
+ output = kill_python(p)
+ self.assertEqual(p.returncode, 0)
+ expected = "toplevel contextvar test: ok"
+ self.assertIn(expected, output, expected)
+
+ def test_toplevel_contextvars_async(self):
+ user_input = dedent("""\
+ from contextvars import ContextVar
+ var = ContextVar('var', default='failed')
+ """)
+ p = spawn_repl("-m", "asyncio")
+ p.stdin.write(user_input+"\n")
+ user_input2 = "async def set_var(): var.set('ok')\n"
+ p.stdin.write(user_input2+"\n")
+ user_input3 = "await set_var()\n"
+ p.stdin.write(user_input3+"\n")
+ user_input4 = "print(f'toplevel contextvar test: {var.get()}')\n"
+ p.stdin.write(user_input4+"\n")
+ output = kill_python(p)
+ self.assertEqual(p.returncode, 0)
+ expected = "toplevel contextvar test: ok"
+ self.assertIn(expected, output, expected)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/Misc/NEWS.d/next/Library/2024-09-26-13-43-39.gh-issue-124594.peYhsP.rst b/Misc/NEWS.d/next/Library/2024-09-26-13-43-39.gh-issue-124594.peYhsP.rst
new file mode 100644
index 00000000000000..ac48bd84930745
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-09-26-13-43-39.gh-issue-124594.peYhsP.rst
@@ -0,0 +1 @@
+All :mod:`asyncio` REPL prompts run in the same :class:`context `. Contributed by Bartosz Sławecki.
From 8da17bb4f7a9a57264792ac9c192440112e85929 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 28 Oct 2024 23:01:18 +0100
Subject: [PATCH 112/269] [3.12] gh-89762: Document strftime %G, %V, and %u
format specifiers (GH-124572) (#126095)
(cherry picked from commit 85799f1ffd5f285ef93a608b0aaf6acbb464ff9d)
Co-authored-by: RUANG (James Roy)
---
Doc/library/time.rst | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/Doc/library/time.rst b/Doc/library/time.rst
index 188dbca8fd1691..5d6365f23fdc2b 100644
--- a/Doc/library/time.rst
+++ b/Doc/library/time.rst
@@ -460,6 +460,9 @@ Functions
| | | |
| | | |
+-----------+------------------------------------------------+-------+
+ | ``%u`` | Day of the week (Monday is 1; Sunday is 7) | |
+ | | as a decimal number [1, 7]. | |
+ +-----------+------------------------------------------------+-------+
| ``%w`` | Weekday as a decimal number [0(Sunday),6]. | |
| | | |
+-----------+------------------------------------------------+-------+
@@ -492,6 +495,16 @@ Functions
| ``%Z`` | Time zone name (no characters if no time zone | |
| | exists). Deprecated. [1]_ | |
+-----------+------------------------------------------------+-------+
+ | ``%G`` | ISO 8601 year (similar to ``%Y`` but follows | |
+ | | the rules for the ISO 8601 calendar year). | |
+ | | The year starts with the week that contains | |
+ | | the first Thursday of the calendar year. | |
+ +-----------+------------------------------------------------+-------+
+ | ``%V`` | ISO 8601 week number (as a decimal number | |
+ | | [01,53]). The first week of the year is the | |
+ | | one that contains the first Thursday of the | |
+ | | year. Weeks start on Monday. | |
+ +-----------+------------------------------------------------+-------+
| ``%%`` | A literal ``'%'`` character. | |
+-----------+------------------------------------------------+-------+
From 385fa83a4348eaa146a84ad97056d2b10534b033 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 29 Oct 2024 09:04:50 +0100
Subject: [PATCH 113/269] [3.12] gh-126014: Ignore `__pycache__`-only folders
in makefile tests (GH-126066) (#126110)
gh-126014: Ignore `__pycache__`-only folders in makefile tests (GH-126066)
(cherry picked from commit aeafaf4cda5bfce44bb054b4c530696901646abe)
Co-authored-by: sobolevn
Co-authored-by: Tomas R.
---
Lib/test/test_tools/test_makefile.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py
index e253bd0049ffa6..f1f0cd87ba7496 100644
--- a/Lib/test/test_tools/test_makefile.py
+++ b/Lib/test/test_tools/test_makefile.py
@@ -51,7 +51,10 @@ def test_makefile_test_folders(self):
if not dirs and not files:
continue
# Skip dirs with hidden-only files:
- if files and all(filename.startswith('.') for filename in files):
+ if files and all(
+ filename.startswith('.') or filename == '__pycache__'
+ for filename in files
+ ):
continue
relpath = os.path.relpath(dirpath, support.STDLIB_DIR)
From a8472fdbcd6098bbf4ae0cabc4b555b92e605181 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 29 Oct 2024 10:13:52 +0100
Subject: [PATCH 114/269] [3.12] Align functools.reduce() docstring with
PEP-257 (GH-126045) (#126114)
Yak-shave in preparation for Argument Clinic adaption in gh-125999.
(cherry picked from commit 9b14083497f50213f908c1359eeaf47c97161347)
Co-authored-by: Sergey B Kirpichev
---
Lib/functools.py | 14 ++++++++------
Modules/_functoolsmodule.c | 14 ++++++++------
2 files changed, 16 insertions(+), 12 deletions(-)
diff --git a/Lib/functools.py b/Lib/functools.py
index 318efd04fd81db..f6849899e7578e 100644
--- a/Lib/functools.py
+++ b/Lib/functools.py
@@ -238,12 +238,14 @@ def reduce(function, sequence, initial=_initial_missing):
"""
reduce(function, iterable[, initial]) -> value
- Apply a function of two arguments cumulatively to the items of a sequence
- or iterable, from left to right, so as to reduce the iterable to a single
- value. For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates
- ((((1+2)+3)+4)+5). If initial is present, it is placed before the items
- of the iterable in the calculation, and serves as a default when the
- iterable is empty.
+ Apply a function of two arguments cumulatively to the items of an iterable, from left to right.
+
+ This effectively reduces the iterable to a single value. If initial is present,
+ it is placed before the items of the iterable in the calculation, and serves as
+ a default when the iterable is empty.
+
+ For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])
+ calculates ((((1 + 2) + 3) + 4) + 5).
"""
it = iter(sequence)
diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c
index a8001d71223fdc..a6d1b83984bcfa 100644
--- a/Modules/_functoolsmodule.c
+++ b/Modules/_functoolsmodule.c
@@ -734,12 +734,14 @@ functools_reduce(PyObject *self, PyObject *args)
PyDoc_STRVAR(functools_reduce_doc,
"reduce(function, iterable[, initial]) -> value\n\
\n\
-Apply a function of two arguments cumulatively to the items of a sequence\n\
-or iterable, from left to right, so as to reduce the iterable to a single\n\
-value. For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates\n\
-((((1+2)+3)+4)+5). If initial is present, it is placed before the items\n\
-of the iterable in the calculation, and serves as a default when the\n\
-iterable is empty.");
+Apply a function of two arguments cumulatively to the items of an iterable, from left to right.\n\
+\n\
+This effectively reduces the iterable to a single value. If initial is present,\n\
+it is placed before the items of the iterable in the calculation, and serves as\n\
+a default when the iterable is empty.\n\
+\n\
+For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])\n\
+calculates ((((1 + 2) + 3) + 4) + 5).");
/* lru_cache object **********************************************************/
From bc9ae4a1caf053a264c6f6623755a5f39da76dc7 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 29 Oct 2024 10:26:28 +0100
Subject: [PATCH 115/269] [3.12] gh-126106: Fix `NULL` possible derefrence in
`Modules/_ssl.c` (GH-126111) (#126117)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
gh-126106: Fix `NULL` possible derefrence in `Modules/_ssl.c` (GH-126111)
(cherry picked from commit a64a1c920660b0c1e4dd5a9573004cd527e15184)
Co-authored-by: sobolevn
Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
---
.../Library/2024-10-29-10-58-52.gh-issue-126106.rlF798.rst | 1 +
Modules/_ssl.c | 6 +++---
2 files changed, 4 insertions(+), 3 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-29-10-58-52.gh-issue-126106.rlF798.rst
diff --git a/Misc/NEWS.d/next/Library/2024-10-29-10-58-52.gh-issue-126106.rlF798.rst b/Misc/NEWS.d/next/Library/2024-10-29-10-58-52.gh-issue-126106.rlF798.rst
new file mode 100644
index 00000000000000..de989007b4c35a
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-29-10-58-52.gh-issue-126106.rlF798.rst
@@ -0,0 +1 @@
+Fixes a possible ``NULL`` pointer dereference in :mod:`ssl`.
diff --git a/Modules/_ssl.c b/Modules/_ssl.c
index 7fcb79fec9f74e..35e4ce7af50fc6 100644
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -4951,14 +4951,14 @@ PySSLSession_dealloc(PySSLSession *self)
static PyObject *
PySSLSession_richcompare(PyObject *left, PyObject *right, int op)
{
- int result;
- PyTypeObject *sesstype = ((PySSLSession*)left)->ctx->state->PySSLSession_Type;
-
if (left == NULL || right == NULL) {
PyErr_BadInternalCall();
return NULL;
}
+ int result;
+ PyTypeObject *sesstype = ((PySSLSession*)left)->ctx->state->PySSLSession_Type;
+
if (!Py_IS_TYPE(left, sesstype) || !Py_IS_TYPE(right, sesstype)) {
Py_RETURN_NOTIMPLEMENTED;
}
From bce9df97d5349e351666055bd797c99e55a1770a Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 29 Oct 2024 14:15:09 +0100
Subject: [PATCH 116/269] [3.12] gh-125783: Add tests to prevent regressions
with the combination of `ctypes` and metaclasses. (GH-125881) (GH-125988)
cherry picked from commit 13844094609cf8265a2eed023e33c7002f3f530d
by Jun Komoda
Also: Add test_ctypes/_support.py from 3.13+
This partially backports be89ee5649031e08f191bf596fa20a09c5698079
(https://github.com/python/cpython/pull/113727)
by AN Long
Co-authored-by: Jun Komoda <45822440+junkmd@users.noreply.github.com>
Co-authored-by: AN Long
Co-authored-by: Erlend E. Aasland
---
Lib/test/test_ctypes/_support.py | 24 +++++
.../test_ctypes/test_c_simple_type_meta.py | 87 +++++++++++++++++++
2 files changed, 111 insertions(+)
create mode 100644 Lib/test/test_ctypes/_support.py
create mode 100644 Lib/test/test_ctypes/test_c_simple_type_meta.py
diff --git a/Lib/test/test_ctypes/_support.py b/Lib/test/test_ctypes/_support.py
new file mode 100644
index 00000000000000..e4c2b33825ae8f
--- /dev/null
+++ b/Lib/test/test_ctypes/_support.py
@@ -0,0 +1,24 @@
+# Some classes and types are not export to _ctypes module directly.
+
+import ctypes
+from _ctypes import Structure, Union, _Pointer, Array, _SimpleCData, CFuncPtr
+
+
+_CData = Structure.__base__
+assert _CData.__name__ == "_CData"
+
+class _X(Structure):
+ _fields_ = [("x", ctypes.c_int)]
+CField = type(_X.x)
+
+# metaclasses
+PyCStructType = type(Structure)
+UnionType = type(Union)
+PyCPointerType = type(_Pointer)
+PyCArrayType = type(Array)
+PyCSimpleType = type(_SimpleCData)
+PyCFuncPtrType = type(CFuncPtr)
+
+# type flags
+Py_TPFLAGS_DISALLOW_INSTANTIATION = 1 << 7
+Py_TPFLAGS_IMMUTABLETYPE = 1 << 8
diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py
new file mode 100644
index 00000000000000..fa5144a3ca01bb
--- /dev/null
+++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py
@@ -0,0 +1,87 @@
+import unittest
+import ctypes
+from ctypes import POINTER, c_void_p
+
+from ._support import PyCSimpleType
+
+
+class PyCSimpleTypeAsMetaclassTest(unittest.TestCase):
+ def tearDown(self):
+ # to not leak references, we must clean _pointer_type_cache
+ ctypes._reset_cache()
+
+ def test_creating_pointer_in_dunder_new_1(self):
+ # Test metaclass whose instances are C types; when the type is
+ # created it automatically creates a pointer type for itself.
+ # The pointer type is also an instance of the metaclass.
+ # Such an implementation is used in `IUnknown` of the `comtypes`
+ # project. See gh-124520.
+
+ class ct_meta(type):
+ def __new__(cls, name, bases, namespace):
+ self = super().__new__(cls, name, bases, namespace)
+
+ # Avoid recursion: don't set up a pointer to
+ # a pointer (to a pointer...)
+ if bases == (c_void_p,):
+ # When creating PtrBase itself, the name
+ # is not yet available
+ return self
+ if issubclass(self, PtrBase):
+ return self
+
+ if bases == (object,):
+ ptr_bases = (self, PtrBase)
+ else:
+ ptr_bases = (self, POINTER(bases[0]))
+ p = p_meta(f"POINTER({self.__name__})", ptr_bases, {})
+ ctypes._pointer_type_cache[self] = p
+ return self
+
+ class p_meta(PyCSimpleType, ct_meta):
+ pass
+
+ class PtrBase(c_void_p, metaclass=p_meta):
+ pass
+
+ class CtBase(object, metaclass=ct_meta):
+ pass
+
+ class Sub(CtBase):
+ pass
+
+ class Sub2(Sub):
+ pass
+
+ self.assertIsInstance(POINTER(Sub2), p_meta)
+ self.assertTrue(issubclass(POINTER(Sub2), Sub2))
+ self.assertTrue(issubclass(POINTER(Sub2), POINTER(Sub)))
+ self.assertTrue(issubclass(POINTER(Sub), POINTER(CtBase)))
+
+ def test_creating_pointer_in_dunder_new_2(self):
+ # A simpler variant of the above, used in `CoClass` of the `comtypes`
+ # project.
+
+ class ct_meta(type):
+ def __new__(cls, name, bases, namespace):
+ self = super().__new__(cls, name, bases, namespace)
+ if isinstance(self, p_meta):
+ return self
+ p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {})
+ ctypes._pointer_type_cache[self] = p
+ return self
+
+ class p_meta(PyCSimpleType, ct_meta):
+ pass
+
+ class Core(object):
+ pass
+
+ class CtBase(Core, metaclass=ct_meta):
+ pass
+
+ class Sub(CtBase):
+ pass
+
+ self.assertIsInstance(POINTER(Sub), p_meta)
+ self.assertTrue(issubclass(POINTER(Sub), Sub))
From 515a5d3498b572057056f0eef143a29838978705 Mon Sep 17 00:00:00 2001
From: Kirill Podoprigora
Date: Tue, 29 Oct 2024 20:20:40 +0200
Subject: [PATCH 117/269] =?UTF-8?q?[3.12]=20gh-126105:=20Fix=20crash=20in?=
=?UTF-8?q?=20`ast`=20module,=20when=20`.=5Ffields`=20is=20delet=E2=80=A6?=
=?UTF-8?q?=20(#126132)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
[3.12] gh-126105: Fix crash in `ast` module, when `._fields` is deleted (GH-126115)
Previously, if the `ast.AST._fields` attribute was deleted, attempts to create a new `as`t node would crash due to the assumption that `_fields` always had a non-NULL value. Now it has been fixed by adding an extra check to ensure that `_fields` does not have a NULL value (this can happen when you manually remove `_fields` attribute).
(cherry picked from commit b2eaa75b176e07730215d76d8dce4d63fb493391)
Co-authored-by: sobolevn
---
Lib/test/test_ast/test_ast.py | 17 +++++++++++++++++
...24-10-29-11-45-44.gh-issue-126105.cOL-R6.rst | 1 +
Parser/asdl_c.py | 13 +++++++------
Python/Python-ast.c | 13 +++++++------
4 files changed, 32 insertions(+), 12 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-29-11-45-44.gh-issue-126105.cOL-R6.rst
diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py
index 7510995b20e9d4..f07e2d027b1624 100644
--- a/Lib/test/test_ast/test_ast.py
+++ b/Lib/test/test_ast/test_ast.py
@@ -66,6 +66,23 @@ def test_AST_objects(self):
# "ast.AST constructor takes 0 positional arguments"
ast.AST(2)
+ def test_AST_fields_NULL_check(self):
+ # See: https://github.com/python/cpython/issues/126105
+ old_value = ast.AST._fields
+
+ def cleanup():
+ ast.AST._fields = old_value
+ self.addCleanup(cleanup)
+
+ del ast.AST._fields
+
+ msg = "type object 'ast.AST' has no attribute '_fields'"
+ # Both examples used to crash:
+ with self.assertRaisesRegex(AttributeError, msg):
+ ast.AST(arg1=123)
+ with self.assertRaisesRegex(AttributeError, msg):
+ ast.AST()
+
def test_AST_garbage_collection(self):
class X:
pass
diff --git a/Misc/NEWS.d/next/Library/2024-10-29-11-45-44.gh-issue-126105.cOL-R6.rst b/Misc/NEWS.d/next/Library/2024-10-29-11-45-44.gh-issue-126105.cOL-R6.rst
new file mode 100644
index 00000000000000..547eb3af1ca064
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-29-11-45-44.gh-issue-126105.cOL-R6.rst
@@ -0,0 +1 @@
+Fix a crash in :mod:`ast` when the :attr:`ast.AST._fields` attribute is deleted.
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 2c34f5c1bc2675..6f5a772dccbd3f 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -813,14 +813,15 @@ def visitModule(self, mod):
Py_ssize_t i, numfields = 0;
int res = -1;
PyObject *key, *value, *fields;
- if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) {
+
+ fields = PyObject_GetAttr((PyObject*)Py_TYPE(self), state->_fields);
+ if (fields == NULL) {
goto cleanup;
}
- if (fields) {
- numfields = PySequence_Size(fields);
- if (numfields == -1) {
- goto cleanup;
- }
+
+ numfields = PySequence_Size(fields);
+ if (numfields == -1) {
+ goto cleanup;
}
res = 0; /* if no error occurs, this stays 0 to the end */
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index ecaff2041f9565..3d5aeff837953b 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -863,14 +863,15 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
Py_ssize_t i, numfields = 0;
int res = -1;
PyObject *key, *value, *fields;
- if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) {
+
+ fields = PyObject_GetAttr((PyObject*)Py_TYPE(self), state->_fields);
+ if (fields == NULL) {
goto cleanup;
}
- if (fields) {
- numfields = PySequence_Size(fields);
- if (numfields == -1) {
- goto cleanup;
- }
+
+ numfields = PySequence_Size(fields);
+ if (numfields == -1) {
+ goto cleanup;
}
res = 0; /* if no error occurs, this stays 0 to the end */
From db0a1b8c1291bf1aa9e016e43bc2f7ed0acf83bd Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 29 Oct 2024 20:22:38 +0100
Subject: [PATCH 118/269] [3.12] Add lightweight comments to conf.py and update
docs readme (GH-126100) (#126135)
Add lightweight comments to conf.py and update docs readme (GH-126100)
* Update contributing contact info in readme
* Add lightweight comments to improve docs workflow understanding
* Apply code review suggestions from @hugovk
* Add code review suggestion from @AA-Turner
* Update Doc/conf.py
* Update Doc/conf.py
* Update Doc/conf.py
---------
(cherry picked from commit 9effa0ff06047f3d957bf37267742a98106581ff)
Co-authored-by: Carol Willing
Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
---
Doc/README.rst | 7 ++-----
Doc/conf.py | 23 ++++++++++++++---------
2 files changed, 16 insertions(+), 14 deletions(-)
diff --git a/Doc/README.rst b/Doc/README.rst
index efcee0db428908..2d1148753e0c6b 100644
--- a/Doc/README.rst
+++ b/Doc/README.rst
@@ -133,8 +133,5 @@ Bugs in the content should be reported to the
Bugs in the toolset should be reported to the tools themselves.
-You can also send a mail to the Python Documentation Team at docs@python.org,
-and we will process your request as soon as possible.
-
-If you want to help the Documentation Team, you are always welcome. Just send
-a mail to docs@python.org.
+To help with the documentation, or report any problems, please leave a message
+on `discuss.python.org `_.
diff --git a/Doc/conf.py b/Doc/conf.py
index f4cf4ee9547e19..938a61347bac3f 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -13,14 +13,17 @@
import sphinx
+# Make our custom extensions available to Sphinx
sys.path.append(os.path.abspath('tools/extensions'))
sys.path.append(os.path.abspath('includes'))
+# Python specific content from Doc/Tools/extensions/pyspecific.py
from pyspecific import SOURCE_URI
# General configuration
# ---------------------
+# Our custom Sphinx extensions are found in Doc/Tools/extensions/
extensions = [
'audit_events',
'availability',
@@ -48,7 +51,7 @@
except ImportError:
_tkinter = None
# Treat warnings as errors, done here to prevent warnings in Sphinx code from
-# causing spurious test failures.
+# causing spurious CPython test failures.
import warnings
warnings.simplefilter('error')
del warnings
@@ -72,10 +75,10 @@
.. |python_version_literal| replace:: ``Python {version}``
"""
-# There are two options for replacing |today|: either, you set today to some
-# non-false value, then it is used:
+# There are two options for replacing |today|. Either, you set today to some
+# non-false value and use it.
today = ''
-# Else, today_fmt is used as the format for a strftime call.
+# Or else, today_fmt is used as the format for a strftime call.
today_fmt = '%B %d, %Y'
# By default, highlight as Python 3.
@@ -87,10 +90,11 @@
# Create table of contents entries for domain objects (e.g. functions, classes,
# attributes, etc.). Default is True.
toc_object_entries = True
+# Hide parents to tidy up long entries in sidebar
toc_object_entries_show_parents = 'hide'
# Ignore any .rst files in the includes/ directory;
-# they're embedded in pages but not rendered individually.
+# they're embedded in pages but not rendered as individual pages.
# Ignore any .rst files in the venv/ directory.
exclude_patterns = ['includes/*.rst', 'venv/*', 'README.rst']
venvdir = os.getenv('VENVDIR')
@@ -307,8 +311,9 @@
# Options for HTML output
# -----------------------
-# Use our custom theme.
+# Use our custom theme: https://github.com/python/python-docs-theme
html_theme = 'python_docs_theme'
+# Location of overrides for theme templates and static files
html_theme_path = ['tools']
html_theme_options = {
'collapsiblesidebar': True,
@@ -354,7 +359,7 @@
html_last_updated_fmt, time.gmtime(html_time)
)
-# Path to find HTML templates.
+# Path to find HTML templates to override theme
templates_path = ['tools/templates']
# Custom sidebar templates, filenames relative to this file.
@@ -600,8 +605,8 @@
"cwe": ("https://cwe.mitre.org/data/definitions/%s.html", "CWE-%s"),
}
-# Options for c_annotations
-# -------------------------
+# Options for c_annotations extension
+# -----------------------------------
# Relative filename of the data files
refcount_file = 'data/refcounts.dat'
From b267b6241b8ac9eb6ceb4c65aaf4341d7e54510e Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 29 Oct 2024 22:02:12 +0100
Subject: [PATCH 119/269] [3.12] GH-125866: Improve tests for `pathname2url()`
and `url2pathname()` (GH-125993) (#126145)
GH-125866: Improve tests for `pathname2url()` and `url2pathname()` (GH-125993)
Merge `URL2PathNameTests` and `PathName2URLTests` test cases (which test
only the Windows-specific implementations from `nturl2path`) into the main
`Pathname_Tests` test case for these functions.
Copy/port some test cases for `pathlib.Path.as_uri()` and `from_uri()`.
(cherry picked from commit 6742f14dfd3fa8ba8a245efa21a4f723160d93d4)
Co-authored-by: Barney Gale
---
Lib/test/test_urllib.py | 163 +++++++++++++++++++---------------------
1 file changed, 77 insertions(+), 86 deletions(-)
diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py
index 2df74f5e6f99b2..5a6d877f6a3094 100644
--- a/Lib/test/test_urllib.py
+++ b/Lib/test/test_urllib.py
@@ -19,7 +19,6 @@
ssl = None
import sys
import tempfile
-from nturl2path import url2pathname, pathname2url
from base64 import b64encode
import collections
@@ -1529,39 +1528,86 @@ def test_quoting(self):
(expect, result))
@unittest.skipUnless(sys.platform == 'win32',
- 'test specific to the nturl2path functions.')
- def test_prefixes(self):
+ 'test specific to Windows pathnames.')
+ def test_pathname2url_win(self):
# Test special prefixes are correctly handled in pathname2url()
- given = '\\\\?\\C:\\dir'
- expect = '///C:/dir'
- result = urllib.request.pathname2url(given)
- self.assertEqual(expect, result,
- "pathname2url() failed; %s != %s" %
- (expect, result))
- given = '\\\\?\\unc\\server\\share\\dir'
- expect = '/server/share/dir'
- result = urllib.request.pathname2url(given)
- self.assertEqual(expect, result,
- "pathname2url() failed; %s != %s" %
- (expect, result))
-
+ fn = urllib.request.pathname2url
+ self.assertEqual(fn('\\\\?\\C:\\dir'), '///C:/dir')
+ self.assertEqual(fn('\\\\?\\unc\\server\\share\\dir'), '/server/share/dir')
+ self.assertEqual(fn("C:"), '///C:')
+ self.assertEqual(fn("C:\\"), '///C:')
+ self.assertEqual(fn('C:\\a\\b.c'), '///C:/a/b.c')
+ self.assertEqual(fn('C:\\a\\b%#c'), '///C:/a/b%25%23c')
+ self.assertEqual(fn('C:\\a\\b\xe9'), '///C:/a/b%C3%A9')
+ self.assertEqual(fn('C:\\foo\\bar\\spam.foo'), "///C:/foo/bar/spam.foo")
+ # Long drive letter
+ self.assertRaises(IOError, fn, "XX:\\")
+ # No drive letter
+ self.assertEqual(fn("\\folder\\test\\"), '/folder/test/')
+ self.assertEqual(fn("\\\\folder\\test\\"), '////folder/test/')
+ self.assertEqual(fn("\\\\\\folder\\test\\"), '/////folder/test/')
+ self.assertEqual(fn('\\\\some\\share\\'), '////some/share/')
+ self.assertEqual(fn('\\\\some\\share\\a\\b.c'), '////some/share/a/b.c')
+ self.assertEqual(fn('\\\\some\\share\\a\\b%#c\xe9'), '////some/share/a/b%25%23c%C3%A9')
+ # Round-tripping
+ urls = ['///C:',
+ '/////folder/test/',
+ '///C:/foo/bar/spam.foo']
+ for url in urls:
+ self.assertEqual(fn(urllib.request.url2pathname(url)), url)
+
+ @unittest.skipIf(sys.platform == 'win32',
+ 'test specific to POSIX pathnames')
+ def test_pathname2url_posix(self):
+ fn = urllib.request.pathname2url
+ self.assertEqual(fn('/'), '/')
+ self.assertEqual(fn('/a/b.c'), '/a/b.c')
+ self.assertEqual(fn('/a/b%#c'), '/a/b%25%23c')
@unittest.skipUnless(sys.platform == 'win32',
- 'test specific to the urllib.url2path function.')
- def test_ntpath(self):
- given = ('/C:/', '///C:/', '/C|//')
- expect = 'C:\\'
- for url in given:
- result = urllib.request.url2pathname(url)
- self.assertEqual(expect, result,
- 'urllib.request..url2pathname() failed; %s != %s' %
- (expect, result))
- given = '///C|/path'
- expect = 'C:\\path'
- result = urllib.request.url2pathname(given)
- self.assertEqual(expect, result,
- 'urllib.request.url2pathname() failed; %s != %s' %
- (expect, result))
+ 'test specific to Windows pathnames.')
+ def test_url2pathname_win(self):
+ fn = urllib.request.url2pathname
+ self.assertEqual(fn('/C:/'), 'C:\\')
+ self.assertEqual(fn("///C|"), 'C:')
+ self.assertEqual(fn("///C:"), 'C:')
+ self.assertEqual(fn('///C:/'), 'C:\\')
+ self.assertEqual(fn('/C|//'), 'C:\\')
+ self.assertEqual(fn('///C|/path'), 'C:\\path')
+ # No DOS drive
+ self.assertEqual(fn("///C/test/"), '\\\\\\C\\test\\')
+ self.assertEqual(fn("////C/test/"), '\\\\C\\test\\')
+ # DOS drive paths
+ self.assertEqual(fn('C:/path/to/file'), 'C:\\path\\to\\file')
+ self.assertEqual(fn('C|/path/to/file'), 'C:\\path\\to\\file')
+ self.assertEqual(fn('/C|/path/to/file'), 'C:\\path\\to\\file')
+ self.assertEqual(fn('///C|/path/to/file'), 'C:\\path\\to\\file')
+ self.assertEqual(fn("///C|/foo/bar/spam.foo"), 'C:\\foo\\bar\\spam.foo')
+ # Non-ASCII drive letter
+ self.assertRaises(IOError, fn, "///\u00e8|/")
+ # UNC paths
+ self.assertEqual(fn('//server/path/to/file'), '\\\\server\\path\\to\\file')
+ self.assertEqual(fn('////server/path/to/file'), '\\\\server\\path\\to\\file')
+ self.assertEqual(fn('/////server/path/to/file'), '\\\\\\server\\path\\to\\file')
+ # Localhost paths
+ self.assertEqual(fn('//localhost/C:/path/to/file'), 'C:\\path\\to\\file')
+ self.assertEqual(fn('//localhost/C|/path/to/file'), 'C:\\path\\to\\file')
+ # Round-tripping
+ paths = ['C:',
+ r'\\\C\test\\',
+ r'C:\foo\bar\spam.foo']
+ for path in paths:
+ self.assertEqual(fn(urllib.request.pathname2url(path)), path)
+
+ @unittest.skipIf(sys.platform == 'win32',
+ 'test specific to POSIX pathnames')
+ def test_url2pathname_posix(self):
+ fn = urllib.request.url2pathname
+ self.assertEqual(fn('/foo/bar'), '/foo/bar')
+ self.assertEqual(fn('//foo/bar'), '//foo/bar')
+ self.assertEqual(fn('///foo/bar'), '///foo/bar')
+ self.assertEqual(fn('////foo/bar'), '////foo/bar')
+ self.assertEqual(fn('//localhost/foo/bar'), '//localhost/foo/bar')
class Utility_Tests(unittest.TestCase):
"""Testcase to test the various utility functions in the urllib."""
@@ -1645,60 +1691,5 @@ def test_with_method_arg(self):
self.assertEqual(request.get_method(), 'HEAD')
-class URL2PathNameTests(unittest.TestCase):
-
- def test_converting_drive_letter(self):
- self.assertEqual(url2pathname("///C|"), 'C:')
- self.assertEqual(url2pathname("///C:"), 'C:')
- self.assertEqual(url2pathname("///C|/"), 'C:\\')
-
- def test_converting_when_no_drive_letter(self):
- # cannot end a raw string in \
- self.assertEqual(url2pathname("///C/test/"), r'\\\C\test' '\\')
- self.assertEqual(url2pathname("////C/test/"), r'\\C\test' '\\')
-
- def test_simple_compare(self):
- self.assertEqual(url2pathname("///C|/foo/bar/spam.foo"),
- r'C:\foo\bar\spam.foo')
-
- def test_non_ascii_drive_letter(self):
- self.assertRaises(IOError, url2pathname, "///\u00e8|/")
-
- def test_roundtrip_url2pathname(self):
- list_of_paths = ['C:',
- r'\\\C\test\\',
- r'C:\foo\bar\spam.foo'
- ]
- for path in list_of_paths:
- self.assertEqual(url2pathname(pathname2url(path)), path)
-
-class PathName2URLTests(unittest.TestCase):
-
- def test_converting_drive_letter(self):
- self.assertEqual(pathname2url("C:"), '///C:')
- self.assertEqual(pathname2url("C:\\"), '///C:')
-
- def test_converting_when_no_drive_letter(self):
- self.assertEqual(pathname2url(r"\\\folder\test" "\\"),
- '/////folder/test/')
- self.assertEqual(pathname2url(r"\\folder\test" "\\"),
- '////folder/test/')
- self.assertEqual(pathname2url(r"\folder\test" "\\"),
- '/folder/test/')
-
- def test_simple_compare(self):
- self.assertEqual(pathname2url(r'C:\foo\bar\spam.foo'),
- "///C:/foo/bar/spam.foo" )
-
- def test_long_drive_letter(self):
- self.assertRaises(IOError, pathname2url, "XX:\\")
-
- def test_roundtrip_pathname2url(self):
- list_of_paths = ['///C:',
- '/////folder/test/',
- '///C:/foo/bar/spam.foo']
- for path in list_of_paths:
- self.assertEqual(pathname2url(url2pathname(path)), path)
-
if __name__ == '__main__':
unittest.main()
From 1fc1a185edee02ea04b6859706d178146aa0f77f Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 30 Oct 2024 00:22:20 +0100
Subject: [PATCH 120/269] [3.12] gh-116938: Fix `dict.update` docstring and
remove erraneous full stop from `dict` documentation (GH-125421) (#126151)
gh-116938: Fix `dict.update` docstring and remove erraneous full stop from `dict` documentation (GH-125421)
(cherry picked from commit 5527c4051c0b58218ce69044f92b45f1d66ed43f)
Co-authored-by: Prometheus3375 <35541026+Prometheus3375@users.noreply.github.com>
Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
---
Doc/library/stdtypes.rst | 2 +-
Objects/dictobject.c | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index 0c1f29d8b69d97..cc67a378d5906f 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -4626,7 +4626,7 @@ can be used interchangeably to index the same dictionary entry.
:meth:`update` accepts either another object with a ``keys()`` method (in
which case :meth:`~object.__getitem__` is called with every key returned from
- the method). or an iterable of key/value pairs (as tuples or other iterables
+ the method) or an iterable of key/value pairs (as tuples or other iterables
of length two). If keyword arguments are specified, the dictionary is then
updated with those key/value pairs: ``d.update(red=1, blue=2)``.
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index 0712bedc838eee..4e965314945aca 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -3634,8 +3634,8 @@ PyDoc_STRVAR(sizeof__doc__,
"D.__sizeof__() -> size of D in memory, in bytes");
PyDoc_STRVAR(update__doc__,
-"D.update([E, ]**F) -> None. Update D from dict/iterable E and F.\n\
-If E is present and has a .keys() method, then does: for k in E: D[k] = E[k]\n\
+"D.update([E, ]**F) -> None. Update D from mapping/iterable E and F.\n\
+If E is present and has a .keys() method, then does: for k in E.keys(): D[k] = E[k]\n\
If E is present and lacks a .keys() method, then does: for k, v in E: D[k] = v\n\
In either case, this is followed by: for k in F: D[k] = F[k]");
From 7812dc37ab2c1ec79c93656f951252aa84652593 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 30 Oct 2024 00:26:29 +0100
Subject: [PATCH 121/269] [3.12] Doc: Note that pydoc uses and prefers
``MANPAGER`` (GH-125362) (#126153)
Doc: Note that pydoc uses and prefers ``MANPAGER`` (GH-125362)
(cherry picked from commit 0e45b1fd0ffbb165f580ecdfd5234c1d54389501)
Co-authored-by: Matthieu Ancellin <31126826+mancellin@users.noreply.github.com>
Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
---
Doc/conf.py | 1 +
Doc/library/pydoc.rst | 5 +++--
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/Doc/conf.py b/Doc/conf.py
index 938a61347bac3f..4e76c21fe0c670 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -178,6 +178,7 @@
('envvar', 'LC_TIME'),
('envvar', 'LINES'),
('envvar', 'LOGNAME'),
+ ('envvar', 'MANPAGER'),
('envvar', 'PAGER'),
('envvar', 'PATH'),
('envvar', 'PATHEXT'),
diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst
index 70e9c604ebac4f..e8f153ee1b35ce 100644
--- a/Doc/library/pydoc.rst
+++ b/Doc/library/pydoc.rst
@@ -52,8 +52,9 @@ produced for that file.
only execute code when a file is invoked as a script and not just imported.
When printing output to the console, :program:`pydoc` attempts to paginate the
-output for easier reading. If the :envvar:`PAGER` environment variable is set,
-:program:`pydoc` will use its value as a pagination program.
+output for easier reading. If either the :envvar:`MANPAGER` or the
+:envvar:`PAGER` environment variable is set, :program:`pydoc` will use its
+value as a pagination program. When both are set, :envvar:`MANPAGER` is used.
Specifying a ``-w`` flag before the argument will cause HTML documentation
to be written out to a file in the current directory, instead of displaying text
From e5c7543f121ad970da46fab3e8e0b4f93e253aa2 Mon Sep 17 00:00:00 2001
From: Brian Schubert
Date: Tue, 29 Oct 2024 20:54:32 -0400
Subject: [PATCH 122/269] [3.12] gh-126139: Improve error message location for
future statement with unknown feature (GH-126140) (#126160)
(cherry picked from commit 224c370a3680132997f1e43d20a3b4ca95a060ab)
---
Lib/test/test_exceptions.py | 4 ++--
Lib/test/test_future_stmt/test_future.py | 6 +++---
.../2024-10-29-15-17-31.gh-issue-126139.B4OQ8a.rst | 2 ++
Python/future.c | 12 ++++++++++--
4 files changed, 17 insertions(+), 7 deletions(-)
create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-15-17-31.gh-issue-126139.B4OQ8a.rst
diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index c5f4b892efb50f..72c86eecae2539 100644
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -316,8 +316,8 @@ def baz():
check('def f():\n global x\n nonlocal x', 2, 3)
# Errors thrown by future.c
- check('from __future__ import doesnt_exist', 1, 1)
- check('from __future__ import braces', 1, 1)
+ check('from __future__ import doesnt_exist', 1, 24)
+ check('from __future__ import braces', 1, 24)
check('x=1\nfrom __future__ import division', 2, 1)
check('foo(1=2)', 1, 5)
check('def f():\n x, y: int', 2, 3)
diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py
index 5a85e08e19b7b6..a06e0510f12dde 100644
--- a/Lib/test/test_future_stmt/test_future.py
+++ b/Lib/test/test_future_stmt/test_future.py
@@ -55,7 +55,7 @@ def test_future_multiple_features(self):
def test_badfuture3(self):
with self.assertRaises(SyntaxError) as cm:
from test.test_future_stmt import badsyntax_future3
- self.check_syntax_error(cm.exception, "badsyntax_future3", 3)
+ self.check_syntax_error(cm.exception, "badsyntax_future3", 3, 24)
def test_badfuture4(self):
with self.assertRaises(SyntaxError) as cm:
@@ -80,12 +80,12 @@ def test_badfuture7(self):
def test_badfuture8(self):
with self.assertRaises(SyntaxError) as cm:
from test.test_future_stmt import badsyntax_future8
- self.check_syntax_error(cm.exception, "badsyntax_future8", 3)
+ self.check_syntax_error(cm.exception, "badsyntax_future8", 3, 24)
def test_badfuture9(self):
with self.assertRaises(SyntaxError) as cm:
from test.test_future_stmt import badsyntax_future9
- self.check_syntax_error(cm.exception, "badsyntax_future9", 3)
+ self.check_syntax_error(cm.exception, "badsyntax_future9", 3, 39)
def test_badfuture10(self):
with self.assertRaises(SyntaxError) as cm:
diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-15-17-31.gh-issue-126139.B4OQ8a.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-15-17-31.gh-issue-126139.B4OQ8a.rst
new file mode 100644
index 00000000000000..278971b46d18ab
--- /dev/null
+++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-15-17-31.gh-issue-126139.B4OQ8a.rst
@@ -0,0 +1,2 @@
+Provide better error location when attempting to use a :term:`future
+statement <__future__>` with an unknown future feature.
diff --git a/Python/future.c b/Python/future.c
index d56f7330964684..998a7f740eac01 100644
--- a/Python/future.c
+++ b/Python/future.c
@@ -39,12 +39,20 @@ future_check_features(PyFutureFeatures *ff, stmt_ty s, PyObject *filename)
} else if (strcmp(feature, "braces") == 0) {
PyErr_SetString(PyExc_SyntaxError,
"not a chance");
- PyErr_SyntaxLocationObject(filename, s->lineno, s->col_offset + 1);
+ PyErr_RangedSyntaxLocationObject(filename,
+ name->lineno,
+ name->col_offset + 1,
+ name->end_lineno,
+ name->end_col_offset + 1);
return 0;
} else {
PyErr_Format(PyExc_SyntaxError,
UNDEFINED_FUTURE_FEATURE, feature);
- PyErr_SyntaxLocationObject(filename, s->lineno, s->col_offset + 1);
+ PyErr_RangedSyntaxLocationObject(filename,
+ name->lineno,
+ name->col_offset + 1,
+ name->end_lineno,
+ name->end_col_offset + 1);
return 0;
}
}
From b69bb1e3feca562dd8df1548674b8a73fa6153f5 Mon Sep 17 00:00:00 2001
From: Brian Schubert
Date: Wed, 30 Oct 2024 02:22:31 -0400
Subject: [PATCH 123/269] [3.12] gh-118633: Add warning regarding the unsafe
usage of eval and exec (GH-118437) (#126162)
(cherry picked from commit 00e5ec0d35193c1665e5c0cfe5ef82eed270d0f4)
Co-authored-by: Daniel Ruf
Co-authored-by: Kirill Podoprigora
Co-authored-by: Jelle Zijlstra
---
Doc/library/functions.rst | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 23f1fdb0dd3bad..3b7c9645d1496c 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -588,6 +588,11 @@ are always available. They are listed here in alphabetical order.
:returns: The result of the evaluated expression.
:raises: Syntax errors are reported as exceptions.
+ .. warning::
+
+ This function executes arbitrary code. Calling it with
+ user-supplied input may lead to security vulnerabilities.
+
The *expression* argument is parsed and evaluated as a Python expression
(technically speaking, a condition list) using the *globals* and *locals*
dictionaries as global and local namespace. If the *globals* dictionary is
@@ -634,6 +639,11 @@ are always available. They are listed here in alphabetical order.
.. function:: exec(object, globals=None, locals=None, /, *, closure=None)
+ .. warning::
+
+ This function executes arbitrary code. Calling it with
+ user-supplied input may lead to security vulnerabilities.
+
This function supports dynamic execution of Python code. *object* must be
either a string or a code object. If it is a string, the string is parsed as
a suite of Python statements which is then executed (unless a syntax error
From 5df3c88432f9179850948ed2ca9c0fbf4a2176d6 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 30 Oct 2024 09:17:16 +0100
Subject: [PATCH 124/269] [3.12] docs: restore an anchor to for/else
(GH-126154) (#126158)
docs: restore an anchor to for/else (GH-126154)
(cherry picked from commit 2d9d10179ff3f13029bf4430e62c455a839987ca)
Co-authored-by: Ned Batchelder
---
Doc/tutorial/controlflow.rst | 1 +
1 file changed, 1 insertion(+)
diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst
index b830ce94ba4f47..8261bbdbfb7a01 100644
--- a/Doc/tutorial/controlflow.rst
+++ b/Doc/tutorial/controlflow.rst
@@ -196,6 +196,7 @@ iteration of the loop::
Found an odd number 9
.. _tut-for-else:
+.. _break-and-continue-statements-and-else-clauses-on-loops:
:keyword:`!else` Clauses on Loops
=================================
From b4040c41ac943edf1f242c0945e2fe50ce5dab4f Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka
Date: Wed, 30 Oct 2024 11:10:10 +0200
Subject: [PATCH 125/269] [3.12] gh-126071: Improve formatting of the argparse
documentation (GH-126073) (GH-126174)
* Use appropriate roles for ArgumentParser, Action, etc.
* Remove superfluous repeated links.
* Explicitly document signatures and add index entries for some methods
and classes.
* Make it more clear that some parameters are keyword-only.
* Fix some minor errors.
(cherry picked from commit 2ab377a47c8290f8bf52c8ffb5d7fc4c45452611)
---
Doc/library/argparse.rst | 131 +++++++++++++++++++++------------------
1 file changed, 69 insertions(+), 62 deletions(-)
diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index f72bd14d56fe0d..b142135f6e635b 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -50,8 +50,8 @@ the extracted data in a :class:`argparse.Namespace` object::
print(args.filename, args.count, args.verbose)
.. note::
- If you're looking a guide about how to upgrade optparse code
- to argparse, see :ref:`Upgrading Optparse Code `.
+ If you're looking for a guide about how to upgrade :mod:`optparse` code
+ to :mod:`!argparse`, see :ref:`Upgrading Optparse Code `.
ArgumentParser objects
----------------------
@@ -100,7 +100,7 @@ ArgumentParser objects
* allow_abbrev_ - Allows long options to be abbreviated if the
abbreviation is unambiguous. (default: ``True``)
- * exit_on_error_ - Determines whether or not ArgumentParser exits with
+ * exit_on_error_ - Determines whether or not :class:`!ArgumentParser` exits with
error info when an error occurs. (default: ``True``)
.. versionchanged:: 3.5
@@ -372,7 +372,7 @@ Most command-line options will use ``-`` as the prefix, e.g. ``-f/--foo``.
Parsers that need to support different or additional prefix
characters, e.g. for options
like ``+f`` or ``/foo``, may specify them using the ``prefix_chars=`` argument
-to the ArgumentParser constructor::
+to the :class:`ArgumentParser` constructor::
>>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='-+')
>>> parser.add_argument('+f')
@@ -503,9 +503,9 @@ string was overridden.
add_help
^^^^^^^^
-By default, ArgumentParser objects add an option which simply displays
+By default, :class:`ArgumentParser` objects add an option which simply displays
the parser's help message. If ``-h`` or ``--help`` is supplied at the command
-line, the ArgumentParser help will be printed.
+line, the :class:`!ArgumentParser` help will be printed.
Occasionally, it may be useful to disable the addition of this help option.
This can be achieved by passing ``False`` as the ``add_help=`` argument to
@@ -559,15 +559,15 @@ If the user would like to catch errors manually, the feature can be enabled by s
The add_argument() method
-------------------------
-.. method:: ArgumentParser.add_argument(name or flags..., [action], [nargs], \
+.. method:: ArgumentParser.add_argument(name or flags..., *, [action], [nargs], \
[const], [default], [type], [choices], [required], \
[help], [metavar], [dest])
Define how a single command-line argument should be parsed. Each parameter
has its own more detailed description below, but in short they are:
- * `name or flags`_ - Either a name or a list of option strings, e.g. ``foo``
- or ``-f, --foo``.
+ * `name or flags`_ - Either a name or a list of option strings, e.g. ``'foo'``
+ or ``'-f', '--foo'``.
* action_ - The basic type of action to be taken when this argument is
encountered at the command line.
@@ -733,22 +733,24 @@ how the command-line arguments should be handled. The supplied actions are:
Only actions that consume command-line arguments (e.g. ``'store'``,
``'append'`` or ``'extend'``) can be used with positional arguments.
-You may also specify an arbitrary action by passing an Action subclass or
-other object that implements the same interface. The ``BooleanOptionalAction``
-is available in ``argparse`` and adds support for boolean actions such as
-``--foo`` and ``--no-foo``::
+.. class:: BooleanOptionalAction
- >>> import argparse
- >>> parser = argparse.ArgumentParser()
- >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction)
- >>> parser.parse_args(['--no-foo'])
- Namespace(foo=False)
+ You may also specify an arbitrary action by passing an :class:`Action` subclass or
+ other object that implements the same interface. The :class:`!BooleanOptionalAction`
+ is available in :mod:`!argparse` and adds support for boolean actions such as
+ ``--foo`` and ``--no-foo``::
-.. versionadded:: 3.9
+ >>> import argparse
+ >>> parser = argparse.ArgumentParser()
+ >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction)
+ >>> parser.parse_args(['--no-foo'])
+ Namespace(foo=False)
+
+ .. versionadded:: 3.9
The recommended way to create a custom action is to extend :class:`Action`,
-overriding the ``__call__`` method and optionally the ``__init__`` and
-``format_usage`` methods.
+overriding the :meth:`!__call__` method and optionally the :meth:`!__init__` and
+:meth:`!format_usage` methods.
An example of a custom action::
@@ -778,7 +780,7 @@ For more details, see :class:`Action`.
nargs
^^^^^
-ArgumentParser objects usually associate a single command-line argument with a
+:class:`ArgumentParser` objects usually associate a single command-line argument with a
single action to be taken. The ``nargs`` keyword argument associates a
different number of command-line arguments with a single action.
See also :ref:`specifying-ambiguous-arguments`. The supported values are:
@@ -1067,7 +1069,7 @@ many choices), just specify an explicit metavar_.
required
^^^^^^^^
-In general, the :mod:`argparse` module assumes that flags like ``-f`` and ``--bar``
+In general, the :mod:`!argparse` module assumes that flags like ``-f`` and ``--bar``
indicate *optional* arguments, which can always be omitted at the command line.
To make an option *required*, ``True`` can be specified for the ``required=``
keyword argument to :meth:`~ArgumentParser.add_argument`::
@@ -1120,7 +1122,7 @@ specifiers include the program name, ``%(prog)s`` and most keyword arguments to
As the help string supports %-formatting, if you want a literal ``%`` to appear
in the help string, you must escape it as ``%%``.
-:mod:`argparse` supports silencing the help entry for certain options, by
+:mod:`!argparse` supports silencing the help entry for certain options, by
setting the ``help`` value to ``argparse.SUPPRESS``::
>>> parser = argparse.ArgumentParser(prog='frobble')
@@ -1138,7 +1140,7 @@ metavar
^^^^^^^
When :class:`ArgumentParser` generates help messages, it needs some way to refer
-to each expected argument. By default, ArgumentParser objects use the dest_
+to each expected argument. By default, :class:`!ArgumentParser` objects use the dest_
value as the "name" of each object. By default, for positional argument
actions, the dest_ value is used directly, and for optional argument actions,
the dest_ value is uppercased. So, a single positional argument with
@@ -1242,7 +1244,7 @@ behavior::
Action classes
^^^^^^^^^^^^^^
-Action classes implement the Action API, a callable which returns a callable
+:class:`!Action` classes implement the Action API, a callable which returns a callable
which processes arguments from the command-line. Any object which follows
this API may be passed as the ``action`` parameter to
:meth:`~ArgumentParser.add_argument`.
@@ -1251,40 +1253,45 @@ this API may be passed as the ``action`` parameter to
type=None, choices=None, required=False, help=None, \
metavar=None)
- Action objects are used by an ArgumentParser to represent the information
+ :class:`!Action` objects are used by an :class:`ArgumentParser` to represent the information
needed to parse a single argument from one or more strings from the
- command line. The Action class must accept the two positional arguments
+ command line. The :class:`!Action` class must accept the two positional arguments
plus any keyword arguments passed to :meth:`ArgumentParser.add_argument`
except for the ``action`` itself.
- Instances of Action (or return value of any callable to the ``action``
- parameter) should have attributes "dest", "option_strings", "default", "type",
- "required", "help", etc. defined. The easiest way to ensure these attributes
- are defined is to call ``Action.__init__``.
+ Instances of :class:`!Action` (or return value of any callable to the
+ ``action`` parameter) should have attributes :attr:`!dest`,
+ :attr:`!option_strings`, :attr:`!default`, :attr:`!type`, :attr:`!required`,
+ :attr:`!help`, etc. defined. The easiest way to ensure these attributes
+ are defined is to call :meth:`!Action.__init__`.
+
+ .. method:: __call__(parser, namespace, values, option_string=None)
+
+ :class:`!Action` instances should be callable, so subclasses must override the
+ :meth:`!__call__` method, which should accept four parameters:
- Action instances should be callable, so subclasses must override the
- ``__call__`` method, which should accept four parameters:
+ * *parser* - The :class:`ArgumentParser` object which contains this action.
- * *parser* - The ArgumentParser object which contains this action.
+ * *namespace* - The :class:`Namespace` object that will be returned by
+ :meth:`~ArgumentParser.parse_args`. Most actions add an attribute to this
+ object using :func:`setattr`.
- * *namespace* - The :class:`Namespace` object that will be returned by
- :meth:`~ArgumentParser.parse_args`. Most actions add an attribute to this
- object using :func:`setattr`.
+ * *values* - The associated command-line arguments, with any type conversions
+ applied. Type conversions are specified with the type_ keyword argument to
+ :meth:`~ArgumentParser.add_argument`.
- * *values* - The associated command-line arguments, with any type conversions
- applied. Type conversions are specified with the type_ keyword argument to
- :meth:`~ArgumentParser.add_argument`.
+ * *option_string* - The option string that was used to invoke this action.
+ The ``option_string`` argument is optional, and will be absent if the action
+ is associated with a positional argument.
- * *option_string* - The option string that was used to invoke this action.
- The ``option_string`` argument is optional, and will be absent if the action
- is associated with a positional argument.
+ The :meth:`!__call__` method may perform arbitrary actions, but will typically set
+ attributes on the ``namespace`` based on ``dest`` and ``values``.
- The ``__call__`` method may perform arbitrary actions, but will typically set
- attributes on the ``namespace`` based on ``dest`` and ``values``.
+ .. method:: format_usage()
- Action subclasses can define a ``format_usage`` method that takes no argument
- and return a string which will be used when printing the usage of the program.
- If such method is not provided, a sensible default will be used.
+ :class:`!Action` subclasses can define a :meth:`!format_usage` method that takes no argument
+ and return a string which will be used when printing the usage of the program.
+ If such method is not provided, a sensible default will be used.
The parse_args() method
@@ -1297,7 +1304,7 @@ The parse_args() method
Previous calls to :meth:`add_argument` determine exactly what objects are
created and how they are assigned. See the documentation for
- :meth:`add_argument` for details.
+ :meth:`!add_argument` for details.
* args_ - List of strings to parse. The default is taken from
:data:`sys.argv`.
@@ -1453,7 +1460,7 @@ This feature can be disabled by setting :ref:`allow_abbrev` to ``False``.
Beyond ``sys.argv``
^^^^^^^^^^^^^^^^^^^
-Sometimes it may be useful to have an ArgumentParser parse arguments other than those
+Sometimes it may be useful to have an :class:`ArgumentParser` parse arguments other than those
of :data:`sys.argv`. This can be accomplished by passing a list of strings to
:meth:`~ArgumentParser.parse_args`. This is useful for testing at the
interactive prompt::
@@ -1511,9 +1518,9 @@ Other utilities
Sub-commands
^^^^^^^^^^^^
-.. method:: ArgumentParser.add_subparsers([title], [description], [prog], \
+.. method:: ArgumentParser.add_subparsers(*, [title], [description], [prog], \
[parser_class], [action], \
- [option_strings], [dest], [required], \
+ [dest], [required], \
[help], [metavar])
Many programs split up their functionality into a number of subcommands,
@@ -1522,11 +1529,11 @@ Sub-commands
this way can be a particularly good idea when a program performs several
different functions which require different kinds of command-line arguments.
:class:`ArgumentParser` supports the creation of such subcommands with the
- :meth:`add_subparsers` method. The :meth:`add_subparsers` method is normally
+ :meth:`!add_subparsers` method. The :meth:`!add_subparsers` method is normally
called with no arguments and returns a special action object. This object
has a single method, :meth:`~_SubParsersAction.add_parser`, which takes a
- command name and any :class:`ArgumentParser` constructor arguments, and
- returns an :class:`ArgumentParser` object that can be modified as usual.
+ command name and any :class:`!ArgumentParser` constructor arguments, and
+ returns an :class:`!ArgumentParser` object that can be modified as usual.
Description of parameters:
@@ -1542,7 +1549,7 @@ Sub-commands
subparser argument
* *parser_class* - class which will be used to create sub-parser instances, by
- default the class of the current parser (e.g. ArgumentParser)
+ default the class of the current parser (e.g. :class:`ArgumentParser`)
* action_ - the basic type of action to be taken when this argument is
encountered at the command line
@@ -1708,7 +1715,7 @@ Sub-commands
Namespace(subparser_name='2', y='frobble')
.. versionchanged:: 3.7
- New *required* keyword argument.
+ New *required* keyword-only parameter.
FileType objects
@@ -1751,7 +1758,7 @@ Argument groups
"positional arguments" and "options" when displaying help
messages. When there is a better conceptual grouping of arguments than this
default one, appropriate groups can be created using the
- :meth:`add_argument_group` method::
+ :meth:`!add_argument_group` method::
>>> parser = argparse.ArgumentParser(prog='PROG', add_help=False)
>>> group = parser.add_argument_group('group')
@@ -1768,7 +1775,7 @@ Argument groups
has an :meth:`~ArgumentParser.add_argument` method just like a regular
:class:`ArgumentParser`. When an argument is added to the group, the parser
treats it just like a normal argument, but displays the argument in a
- separate group for help messages. The :meth:`add_argument_group` method
+ separate group for help messages. The :meth:`!add_argument_group` method
accepts *title* and *description* arguments which can be used to
customize this display::
@@ -1810,7 +1817,7 @@ Mutual exclusion
.. method:: ArgumentParser.add_mutually_exclusive_group(required=False)
- Create a mutually exclusive group. :mod:`argparse` will make sure that only
+ Create a mutually exclusive group. :mod:`!argparse` will make sure that only
one of the arguments in the mutually exclusive group was present on the
command line::
@@ -2023,7 +2030,7 @@ Intermixed parsing
and :meth:`~ArgumentParser.parse_known_intermixed_args` methods
support this parsing style.
- These parsers do not support all the argparse features, and will raise
+ These parsers do not support all the :mod:`!argparse` features, and will raise
exceptions if unsupported features are used. In particular, subparsers,
and mutually exclusive groups that include both
optionals and positionals are not supported.
From 679dfaeb4c78a138441a2073530be105c6752766 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 30 Oct 2024 15:22:50 +0100
Subject: [PATCH 126/269] [3.12] gh-124612: Use ghcr.io/python/autoconf instead
of public image (GH-124657) (#126184)
(cherry picked from commit b502573f7f800dbb2e401fa2a7a05eceac692c7a)
Co-authored-by: Donghee Na
---
Tools/build/regen-configure.sh | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/Tools/build/regen-configure.sh b/Tools/build/regen-configure.sh
index e34a36c1a573e5..efc80c8527885c 100755
--- a/Tools/build/regen-configure.sh
+++ b/Tools/build/regen-configure.sh
@@ -5,12 +5,10 @@ set -e -x
# The check_generated_files job of .github/workflows/build.yml must kept in
# sync with this script. Use the same container image than the job so the job
# doesn't need to run autoreconf in a container.
-IMAGE="ubuntu:22.04"
-DEPENDENCIES="autotools-dev autoconf autoconf-archive pkg-config"
+IMAGE="ghcr.io/python/autoconf:2024.10.06.11200919239"
AUTORECONF="autoreconf -ivf -Werror"
WORK_DIR="/src"
-SHELL_CMD="apt-get update && apt-get -yq install $DEPENDENCIES && cd $WORK_DIR && $AUTORECONF"
abs_srcdir=$(cd $(dirname $0)/../..; pwd)
@@ -28,4 +26,4 @@ if command -v selinuxenabled >/dev/null && selinuxenabled; then
PATH_OPT=":Z"
fi
-"$RUNTIME" run --rm -v "$abs_srcdir:$WORK_DIR$PATH_OPT" "$IMAGE" /usr/bin/bash -c "$SHELL_CMD"
+"$RUNTIME" run --rm -v "$abs_srcdir:$WORK_DIR$PATH_OPT" "$IMAGE"
From 317350ec1bf340f2b871ecd215c951ea7adf975f Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 30 Oct 2024 18:36:11 +0100
Subject: [PATCH 127/269] [3.12] gh-85583: Add f-string index in
tutorial/inputoutput.rst (GH-21681) (GH-126192)
gh-85583: Add f-string index in tutorial/inputoutput.rst (GH-21681)
* bpo-41411 fstring index in tutorial/inputoutput
To assist in searching for fstrings I have added an index
* Add newline
---------
(cherry picked from commit a3443c0e22a8623afe4c0518433b28afbc3a6df6)
Co-authored-by: amaajemyfren <32741226+amaajemyfren@users.noreply.github.com>
Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
Co-authored-by: Carol Willing
---
Doc/tutorial/inputoutput.rst | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst
index 2e6fd419b21106..35b8c7cd8eb049 100644
--- a/Doc/tutorial/inputoutput.rst
+++ b/Doc/tutorial/inputoutput.rst
@@ -100,6 +100,13 @@ yet another way to substitute values into strings, using placeholders like
``$x`` and replacing them with values from a dictionary, but offers much less
control of the formatting.
+.. index::
+ single: formatted string literal
+ single: interpolated string literal
+ single: string; formatted literal
+ single: string; interpolated literal
+ single: f-string
+ single: fstring
.. _tut-f-strings:
From 76c800f751166557c91b7180555840e57310095b Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 30 Oct 2024 19:01:39 +0100
Subject: [PATCH 128/269] [3.12] Fix incorrect indentation in
importlib.metadata.rst (GH-126189) (GH-126194)
Fix incorrect indentation in importlib.metadata.rst (GH-126189)
(cherry picked from commit 6f512c603465ff3e51e07cb7e1d36b4e5e88905d)
Co-authored-by: Rafael Fontenelle
---
Doc/library/importlib.metadata.rst | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst
index 14414d973f20c1..fe898cf4af8bd5 100644
--- a/Doc/library/importlib.metadata.rst
+++ b/Doc/library/importlib.metadata.rst
@@ -227,10 +227,10 @@ Distribution metadata
.. class:: PackageMetadata
A concrete implementation of the
- `PackageMetadata protocol `_.
+ `PackageMetadata protocol `_.
- In addition to providing the defined protocol methods and attributes, subscripting
- the instance is equivalent to calling the :meth:`!get` method.
+ In addition to providing the defined protocol methods and attributes, subscripting
+ the instance is equivalent to calling the :meth:`!get` method.
Every `Distribution Package `_
includes some metadata, which you can extract using the :func:`!metadata` function::
From f2315ee175222bf986de31e654d265bcbb176435 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 30 Oct 2024 21:14:27 +0100
Subject: [PATCH 129/269] [3.12] gh-126055: Add omitted command (in docs
[os.walk]) for code to fulfill `shutil.rmtree` algorithm (GH-126067)
(GH-126200)
gh-126055: Add omitted command (in docs [os.walk]) for code to fulfill `shutil.rmtree` algorithm (GH-126067)
* gh-126055: Add omitted command (in docs [os.walk]) for code to fulfill `shutil.rmtree` algorithm.
Resolves GH-126055
* gh-126055: Fix omitted code highlighting
(cherry picked from commit 597d814334742dde386a4d2979b9418aee6fcaba)
Co-authored-by: Victor Wheeler
---
Doc/library/os.rst | 1 +
1 file changed, 1 insertion(+)
diff --git a/Doc/library/os.rst b/Doc/library/os.rst
index 3a5deaa1d692bd..423480e927c900 100644
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -3571,6 +3571,7 @@ features:
os.remove(os.path.join(root, name))
for name in dirs:
os.rmdir(os.path.join(root, name))
+ os.rmdir(top)
.. audit-event:: os.walk top,topdown,onerror,followlinks os.walk
From bda8190b891574e66378a14723597ecfecdec5d8 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 30 Oct 2024 21:15:24 +0100
Subject: [PATCH 130/269] [3.12] gh-60712: Include the "object" type in the
lists of documented types (GH-103036) (GH-126198)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
gh-60712: Include the "object" type in the lists of documented types (GH-103036)
* add test for the predefined object's attributes
* Include the "object" type in the lists of documented types
* remove 'or' from augment tuple
* 📜🤖 Added by blurb_it.
* Add cross-reference to news
* Fix format for the function parameter
* Add space
* add reference for the 'object'
* add reference for NotImplemented
* Change ref:`string ` as class:`str`
* remove hyphen from `newly-created`
* Update Doc/reference/datamodel.rst
'dictionaries' to 'dict'
* Update predefined attribute types in testPredefinedAttrs
* Change `universal type` as `top type`
* Don't mention about the top type
* Update the description of richcmpfuncs
* Update Doc/library/stdtypes.rst
* Revert: Hierarchy Section in Data Model Documentation
* Revert to original explanations of __new__ and __init__ methods in datamodel.rst for improved clarity.
* Update Doc/reference/datamodel.rst
* Remove blank line
* Use ref:`str ` instead of :class:`str
* Revert changes the description of Other Built-in Types in stdtypes.rst
* Update Doc/reference/datamodel.rst
---------
(cherry picked from commit 4f826214b30ad50365f45bf07e4a02f34f897ab5)
Co-authored-by: Furkan Onder
Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>
Co-authored-by: C.A.M. Gerlach
Co-authored-by: Terry Jan Reedy
Co-authored-by: Éric
Co-authored-by: Carol Willing
---
Doc/library/functions.rst | 7 +--
Doc/reference/datamodel.rst | 52 +++++++++++++------
Lib/test/test_class.py | 50 ++++++++++++++++++
...3-03-28-22-24-45.gh-issue-60712.So5uad.rst | 2 +
4 files changed, 93 insertions(+), 18 deletions(-)
create mode 100644 Misc/NEWS.d/next/Documentation/2023-03-28-22-24-45.gh-issue-60712.So5uad.rst
diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 3b7c9645d1496c..d26e79d12fc159 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -1223,9 +1223,10 @@ are always available. They are listed here in alphabetical order.
.. class:: object()
- Return a new featureless object. :class:`object` is a base for all classes.
- It has methods that are common to all instances of Python classes. This
- function does not accept any arguments.
+ This is the ultimate base class of all other classes. It has methods
+ that are common to all instances of Python classes. When the constructor
+ is called, it returns a new featureless object. The constructor does not
+ accept any arguments.
.. note::
diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index 7c0719bd93d5da..bc287e1e7cdcf9 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -1942,7 +1942,8 @@ Basic customization
"informal" string representation of instances of that class is required.
This is typically used for debugging, so it is important that the representation
- is information-rich and unambiguous.
+ is information-rich and unambiguous. A default implementation is provided by the
+ :class:`object` class itself.
.. index::
single: string; __str__() (object method)
@@ -1952,10 +1953,10 @@ Basic customization
.. method:: object.__str__(self)
- Called by :func:`str(object) ` and the built-in functions
- :func:`format` and :func:`print` to compute the "informal" or nicely
+ Called by :func:`str(object) `, the default :meth:`__format__` implementation,
+ and the built-in function :func:`print`, to compute the "informal" or nicely
printable string representation of an object. The return value must be a
- :ref:`string ` object.
+ :ref:`str ` object.
This method differs from :meth:`object.__repr__` in that there is no
expectation that :meth:`__str__` return a valid Python expression: a more
@@ -1972,7 +1973,8 @@ Basic customization
.. index:: pair: built-in function; bytes
Called by :ref:`bytes ` to compute a byte-string representation
- of an object. This should return a :class:`bytes` object.
+ of an object. This should return a :class:`bytes` object. The :class:`object`
+ class itself does not provide this method.
.. index::
single: string; __format__() (object method)
@@ -1996,6 +1998,9 @@ Basic customization
The return value must be a string object.
+ The default implementation by the :class:`object` class should be given
+ an empty *format_spec* string. It delegates to :meth:`__str__`.
+
.. versionchanged:: 3.4
The __format__ method of ``object`` itself raises a :exc:`TypeError`
if passed any non-empty string.
@@ -2038,6 +2043,12 @@ Basic customization
``(x` (such as :class:`lists ` or
+The following methods can be defined to implement container objects. None of them
+are provided by the :class:`object` class itself. Containers usually are
+:term:`sequences ` (such as :class:`lists ` or
:class:`tuples `) or :term:`mappings ` (like
-:class:`dictionaries `),
+:term:`dictionaries `),
but can represent other containers as well. The first set of methods is used
either to emulate a sequence or to emulate a mapping; the difference is that for
a sequence, the allowable keys should be the integers *k* for which ``0 <= k <
@@ -3372,6 +3386,7 @@ Typical uses of context managers include saving and restoring various kinds of
global state, locking and unlocking resources, closing opened files, etc.
For more information on context managers, see :ref:`typecontextmanager`.
+The :class:`object` class itself does not provide the context manager methods.
.. method:: object.__enter__(self)
@@ -3576,6 +3591,8 @@ are awaitable.
Must return an :term:`iterator`. Should be used to implement
:term:`awaitable` objects. For instance, :class:`asyncio.Future` implements
this method to be compatible with the :keyword:`await` expression.
+ The :class:`object` class itself is not awaitable and does not provide
+ this method.
.. note::
@@ -3661,6 +3678,9 @@ its ``__anext__`` method.
Asynchronous iterators can be used in an :keyword:`async for` statement.
+The :class:`object` class itself does not provide these methods.
+
+
.. method:: object.__aiter__(self)
Must return an *asynchronous iterator* object.
@@ -3707,6 +3727,8 @@ suspend execution in its ``__aenter__`` and ``__aexit__`` methods.
Asynchronous context managers can be used in an :keyword:`async with` statement.
+The :class:`object` class itself does not provide these methods.
+
.. method:: object.__aenter__(self)
Semantically similar to :meth:`~object.__enter__`, the only
diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py
index 7a159760e98e25..28d3c0db12c19e 100644
--- a/Lib/test/test_class.py
+++ b/Lib/test/test_class.py
@@ -503,6 +503,56 @@ def __eq__(self, other): return 1
self.assertRaises(TypeError, hash, C2())
+ def testPredefinedAttrs(self):
+ o = object()
+
+ class Custom:
+ pass
+
+ c = Custom()
+
+ methods = (
+ '__class__', '__delattr__', '__dir__', '__eq__', '__format__',
+ '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__',
+ '__init__', '__init_subclass__', '__le__', '__lt__', '__ne__',
+ '__new__', '__reduce__', '__reduce_ex__', '__repr__',
+ '__setattr__', '__sizeof__', '__str__', '__subclasshook__'
+ )
+ for name in methods:
+ with self.subTest(name):
+ self.assertTrue(callable(getattr(object, name, None)))
+ self.assertTrue(callable(getattr(o, name, None)))
+ self.assertTrue(callable(getattr(Custom, name, None)))
+ self.assertTrue(callable(getattr(c, name, None)))
+
+ not_defined = [
+ '__abs__', '__aenter__', '__aexit__', '__aiter__', '__anext__',
+ '__await__', '__bool__', '__bytes__', '__ceil__',
+ '__complex__', '__contains__', '__del__', '__delete__',
+ '__delitem__', '__divmod__', '__enter__', '__exit__',
+ '__float__', '__floor__', '__get__', '__getattr__', '__getitem__',
+ '__index__', '__int__', '__invert__', '__iter__', '__len__',
+ '__length_hint__', '__missing__', '__neg__', '__next__',
+ '__objclass__', '__pos__', '__rdivmod__', '__reversed__',
+ '__round__', '__set__', '__setitem__', '__trunc__'
+ ]
+ augment = (
+ 'add', 'and', 'floordiv', 'lshift', 'matmul', 'mod', 'mul', 'pow',
+ 'rshift', 'sub', 'truediv', 'xor'
+ )
+ not_defined.extend(map("__{}__".format, augment))
+ not_defined.extend(map("__r{}__".format, augment))
+ not_defined.extend(map("__i{}__".format, augment))
+ for name in not_defined:
+ with self.subTest(name):
+ self.assertFalse(hasattr(object, name))
+ self.assertFalse(hasattr(o, name))
+ self.assertFalse(hasattr(Custom, name))
+ self.assertFalse(hasattr(c, name))
+
+ # __call__() is defined on the metaclass but not the class
+ self.assertFalse(hasattr(o, "__call__"))
+ self.assertFalse(hasattr(c, "__call__"))
def testSFBug532646(self):
# Test for SF bug 532646
diff --git a/Misc/NEWS.d/next/Documentation/2023-03-28-22-24-45.gh-issue-60712.So5uad.rst b/Misc/NEWS.d/next/Documentation/2023-03-28-22-24-45.gh-issue-60712.So5uad.rst
new file mode 100644
index 00000000000000..e401cc2535e389
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2023-03-28-22-24-45.gh-issue-60712.So5uad.rst
@@ -0,0 +1,2 @@
+Include the :class:`object` type in the lists of documented types.
+Change by Furkan Onder and Martin Panter.
From 4a846f291e0b56a06b505a4ff20ee830e12e0541 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 30 Oct 2024 21:17:35 +0100
Subject: [PATCH 131/269] gh-125315: Avoid crashing in _wmimodule due to slow
WMI calls on some Windows machines (GH-126141)
(cherry picked from commit 60c415bd531392a239c23c754154a7944695ac99)
Co-authored-by: Steve Dower
---
...-10-29-19-48-03.gh-issue-125315.jdB9qN.rst | 2 ++
PC/_wmimodule.cpp | 22 +++++++++++++------
2 files changed, 17 insertions(+), 7 deletions(-)
create mode 100644 Misc/NEWS.d/next/Windows/2024-10-29-19-48-03.gh-issue-125315.jdB9qN.rst
diff --git a/Misc/NEWS.d/next/Windows/2024-10-29-19-48-03.gh-issue-125315.jdB9qN.rst b/Misc/NEWS.d/next/Windows/2024-10-29-19-48-03.gh-issue-125315.jdB9qN.rst
new file mode 100644
index 00000000000000..3d813248766a5b
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2024-10-29-19-48-03.gh-issue-125315.jdB9qN.rst
@@ -0,0 +1,2 @@
+Avoid crashing in :mod:`platform` due to slow WMI calls on some Windows
+machines.
diff --git a/PC/_wmimodule.cpp b/PC/_wmimodule.cpp
index 22ed05276e6f07..2c24761be6400d 100644
--- a/PC/_wmimodule.cpp
+++ b/PC/_wmimodule.cpp
@@ -55,12 +55,26 @@ _query_thread(LPVOID param)
IWbemLocator *locator = NULL;
IWbemServices *services = NULL;
IEnumWbemClassObject* enumerator = NULL;
+ HRESULT hr = S_OK;
BSTR bstrQuery = NULL;
struct _query_data *data = (struct _query_data*)param;
- HRESULT hr = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED);
+ // gh-125315: Copy the query string first, so that if the main thread gives
+ // up on waiting we aren't left with a dangling pointer (and a likely crash)
+ bstrQuery = SysAllocString(data->query);
+ if (!bstrQuery) {
+ hr = HRESULT_FROM_WIN32(ERROR_NOT_ENOUGH_MEMORY);
+ }
+
+ if (SUCCEEDED(hr)) {
+ hr = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED);
+ }
+
if (FAILED(hr)) {
CloseHandle(data->writePipe);
+ if (bstrQuery) {
+ SysFreeString(bstrQuery);
+ }
return (DWORD)hr;
}
@@ -101,12 +115,6 @@ _query_thread(LPVOID param)
NULL, EOAC_NONE
);
}
- if (SUCCEEDED(hr)) {
- bstrQuery = SysAllocString(data->query);
- if (!bstrQuery) {
- hr = HRESULT_FROM_WIN32(ERROR_NOT_ENOUGH_MEMORY);
- }
- }
if (SUCCEEDED(hr)) {
hr = services->ExecQuery(
bstr_t("WQL"),
From a089adfb539ebd83545fea0397d90d32e4cc8bc9 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 30 Oct 2024 21:34:13 +0100
Subject: [PATCH 132/269] [3.12] Prefer "similar" over "equivalent" in tutorial
(GH-125343) (GH-125373)
Prefer "similar" over "equivalent" in tutorial (GH-125343)
In the datastructures tutorial doc, some operations are described as
"equivalent to" others. This has led to some user-confusion -- at
least in the Discourse forums -- about cases in which the operations
differ.
This change doesn't systematically eliminate the word "equivalent"
from the tutorial. It just substitutes "similar to" in several cases
in which "equivalent to" could mislead users into expecting exact
equivalence.
(cherry picked from commit 4a2282b0679bbf7b7fbd36aae1b1565145238961)
Co-authored-by: Stephen Rosen
---
Doc/tutorial/datastructures.rst | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst
index 73f17adeea72de..31941bc112a135 100644
--- a/Doc/tutorial/datastructures.rst
+++ b/Doc/tutorial/datastructures.rst
@@ -19,13 +19,13 @@ objects:
.. method:: list.append(x)
:noindex:
- Add an item to the end of the list. Equivalent to ``a[len(a):] = [x]``.
+ Add an item to the end of the list. Similar to ``a[len(a):] = [x]``.
.. method:: list.extend(iterable)
:noindex:
- Extend the list by appending all the items from the iterable. Equivalent to
+ Extend the list by appending all the items from the iterable. Similar to
``a[len(a):] = iterable``.
@@ -56,7 +56,7 @@ objects:
.. method:: list.clear()
:noindex:
- Remove all items from the list. Equivalent to ``del a[:]``.
+ Remove all items from the list. Similar to ``del a[:]``.
.. method:: list.index(x[, start[, end]])
@@ -93,7 +93,7 @@ objects:
.. method:: list.copy()
:noindex:
- Return a shallow copy of the list. Equivalent to ``a[:]``.
+ Return a shallow copy of the list. Similar to ``a[:]``.
An example that uses most of the list methods::
From 3ef8a3a9004ff7ed2bf6f2f67f13ff5b37c42204 Mon Sep 17 00:00:00 2001
From: Rafael Fontenelle
Date: Wed, 30 Oct 2024 17:46:13 -0300
Subject: [PATCH 133/269] [3.12]: gh-103484: Relink _xxsubinterpretersmodule.c
in whatsnew/3.12.rst (GH-124183)
Relink _xxsubinterpretersmodule.c in whatsnew/3.12.rst
---
Doc/whatsnew/3.12.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst
index 7ec1935ce956d3..c99c4559a79f91 100644
--- a/Doc/whatsnew/3.12.rst
+++ b/Doc/whatsnew/3.12.rst
@@ -359,7 +359,7 @@ create an interpreter with its own GIL:
/* The new interpreter is now active in the current thread. */
For further examples how to use the C-API for sub-interpreters with a
-per-interpreter GIL, see ``Modules/_xxsubinterpretersmodule.c``.
+per-interpreter GIL, see :source:`Modules/_xxsubinterpretersmodule.c`.
(Contributed by Eric Snow in :gh:`104210`, etc.)
From 6bfe5fbc19b96d6dc4c5f5f45bbb1acde45ade8f Mon Sep 17 00:00:00 2001
From: Prometheus3375 <35541026+Prometheus3375@users.noreply.github.com>
Date: Thu, 31 Oct 2024 03:08:28 +0300
Subject: [PATCH 134/269] [3.12] gh-116633: Add a note about buggy behavior of
csv.QUOTE_NOTNULL and csv.QUOTE_STRINGS (GH-117235)
* Add a note about bug
* Properly link constants
---
Doc/library/csv.rst | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst
index 533cdf13974be6..8e432078e1949a 100644
--- a/Doc/library/csv.rst
+++ b/Doc/library/csv.rst
@@ -365,6 +365,12 @@ The :mod:`csv` module defines the following constants:
.. versionadded:: 3.12
+.. note::
+
+ Due to a bug, constants :data:`QUOTE_NOTNULL` and :data:`QUOTE_STRINGS`
+ do not affect behaviour of :class:`reader` objects.
+ This bug is fixed in Python 3.13.
+
The :mod:`csv` module defines the following exception:
From 975911887dd767babfe06aa45e835b32c4112b51 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 31 Oct 2024 02:31:35 +0100
Subject: [PATCH 135/269] [3.12] gh-125818: Fix incorrect signature of argument
`skip_file_prefixes` in warnings docs (GH-125823) (GH-126217)
gh-125818: Fix incorrect signature of argument `skip_file_prefixes` in warnings docs (GH-125823)
Change documentation
(cherry picked from commit d467d9246cbe0ce5dc149c4c74223bb8374ece73)
Co-authored-by: RUANG (James Roy)
---
Doc/library/warnings.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst
index 511b94a386d5b9..0cd3ca3c9de73f 100644
--- a/Doc/library/warnings.rst
+++ b/Doc/library/warnings.rst
@@ -396,7 +396,7 @@ Available Functions
-------------------
-.. function:: warn(message, category=None, stacklevel=1, source=None, *, skip_file_prefixes=None)
+.. function:: warn(message, category=None, stacklevel=1, source=None, *, skip_file_prefixes=())
Issue a warning, or maybe ignore it or raise an exception. The *category*
argument, if given, must be a :ref:`warning category class `; it
From 90de322b32a72c69afafa2fb879676923357cf3a Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 31 Oct 2024 09:19:56 +0100
Subject: [PATCH 136/269] [3.12] gh-126083: Fix a reference leak in
`asyncio.Task` when reinitializing with new non-`None` context (GH-126103)
(#126230)
gh-126083: Fix a reference leak in `asyncio.Task` when reinitializing with new non-`None` context (GH-126103)
(cherry picked from commit d07dcce6935364cab807e0df931ed09b088ade69)
Co-authored-by: Nico-Posada <102486290+Nico-Posada@users.noreply.github.com>
---
Lib/test/test_asyncio/test_tasks.py | 22 +++++++++++++++++++
...-10-28-22-35-22.gh-issue-126083.TuI--n.rst | 1 +
Modules/_asynciomodule.c | 2 +-
3 files changed, 24 insertions(+), 1 deletion(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-28-22-35-22.gh-issue-126083.TuI--n.rst
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
index 6e8a51ce2555d5..792ab4fbede0a2 100644
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -2489,6 +2489,28 @@ def test_get_context(self):
finally:
loop.close()
+ def test_proper_refcounts(self):
+ # see: https://github.com/python/cpython/issues/126083
+ class Break:
+ def __str__(self):
+ raise RuntimeError("break")
+
+ obj = object()
+ initial_refcount = sys.getrefcount(obj)
+
+ coro = coroutine_function()
+ loop = asyncio.new_event_loop()
+ task = asyncio.Task.__new__(asyncio.Task)
+
+ for _ in range(5):
+ with self.assertRaisesRegex(RuntimeError, 'break'):
+ task.__init__(coro, loop=loop, context=obj, name=Break())
+
+ coro.close()
+ del task
+
+ self.assertEqual(sys.getrefcount(obj), initial_refcount)
+
def add_subclass_tests(cls):
BaseTask = cls.Task
diff --git a/Misc/NEWS.d/next/Library/2024-10-28-22-35-22.gh-issue-126083.TuI--n.rst b/Misc/NEWS.d/next/Library/2024-10-28-22-35-22.gh-issue-126083.TuI--n.rst
new file mode 100644
index 00000000000000..d64b7dd2fedbd6
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-28-22-35-22.gh-issue-126083.TuI--n.rst
@@ -0,0 +1 @@
+Fixed a reference leak in :class:`asyncio.Task` objects when reinitializing the same object with a non-``None`` context. Patch by Nico Posada.
diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c
index 9016c077e0d657..96821762c5ae68 100644
--- a/Modules/_asynciomodule.c
+++ b/Modules/_asynciomodule.c
@@ -2143,7 +2143,7 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop,
return -1;
}
} else {
- self->task_context = Py_NewRef(context);
+ Py_XSETREF(self->task_context, Py_NewRef(context));
}
Py_CLEAR(self->task_fut_waiter);
From 3416ffaa0844477e006eb1245992262d4cf2e69a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?=
<10796600+picnixz@users.noreply.github.com>
Date: Thu, 31 Oct 2024 16:04:17 +0100
Subject: [PATCH 137/269] [3.12] gh-126240: handle `NULL` returned by
`_Py_asdl_expr_seq_new` (GH-126241) (#126245)
gh-126240: handle `NULL` returned by `_Py_asdl_expr_seq_new` (#126241)
check return value of `_Py_asdl_expr_seq_new`
---
Parser/action_helpers.c | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c
index 085622e6551203..be52d89495c7dc 100644
--- a/Parser/action_helpers.c
+++ b/Parser/action_helpers.c
@@ -1043,6 +1043,9 @@ expr_ty _PyPegen_collect_call_seqs(Parser *p, asdl_expr_seq *a, asdl_seq *b,
}
asdl_expr_seq *args = _Py_asdl_expr_seq_new(total_len, arena);
+ if (args == NULL) {
+ return NULL;
+ }
Py_ssize_t i = 0;
for (i = 0; i < args_len; i++) {
@@ -1210,6 +1213,9 @@ unpack_top_level_joined_strs(Parser *p, asdl_expr_seq *raw_expressions) {
}
asdl_expr_seq *expressions = _Py_asdl_expr_seq_new(req_size, p->arena);
+ if (expressions == NULL) {
+ return NULL;
+ }
Py_ssize_t raw_index, req_index = 0;
for (raw_index = 0; raw_index < raw_size; raw_index++) {
@@ -1400,6 +1406,9 @@ expr_ty _PyPegen_formatted_value(Parser *p, expr_ty expression, Token *debug,
}
asdl_expr_seq *values = _Py_asdl_expr_seq_new(2, arena);
+ if (values == NULL) {
+ return NULL;
+ }
asdl_seq_SET(values, 0, debug_text);
asdl_seq_SET(values, 1, formatted_value);
return _PyAST_JoinedStr(values, lineno, col_offset, debug_end_line,
From 78f307f39e6d3f4ebb172140945ad2ec1ee10811 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 31 Oct 2024 17:59:15 +0100
Subject: [PATCH 138/269] [3.12] GH-126205: Fix conversion of UNC paths to file
URIs (GH-126208) (#126248)
GH-126205: Fix conversion of UNC paths to file URIs (GH-126208)
File URIs for Windows UNC paths should begin with two slashes, not four.
(cherry picked from commit 951cb2c369e8b8fb9f93662658391a53fd727787)
Co-authored-by: Barney Gale
---
Lib/nturl2path.py | 7 +------
Lib/test/test_urllib.py | 14 +++++++-------
.../2024-10-30-20-45-17.gh-issue-126205.CHEmtx.rst | 2 ++
3 files changed, 10 insertions(+), 13 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-30-20-45-17.gh-issue-126205.CHEmtx.rst
diff --git a/Lib/nturl2path.py b/Lib/nturl2path.py
index 61852aff58912d..6453f202c26d14 100644
--- a/Lib/nturl2path.py
+++ b/Lib/nturl2path.py
@@ -55,16 +55,11 @@ def pathname2url(p):
if p[:4] == '\\\\?\\':
p = p[4:]
if p[:4].upper() == 'UNC\\':
- p = '\\' + p[4:]
+ p = '\\\\' + p[4:]
elif p[1:2] != ':':
raise OSError('Bad path: ' + p)
if not ':' in p:
# No drive specifier, just convert slashes and quote the name
- if p[:2] == '\\\\':
- # path is something like \\host\path\on\remote\host
- # convert this to ////host/path/on/remote/host
- # (notice doubling of slashes at the start of the path)
- p = '\\\\' + p
components = p.split('\\')
return urllib.parse.quote('/'.join(components))
comp = p.split(':', maxsplit=2)
diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py
index 5a6d877f6a3094..0e1b13d31c9273 100644
--- a/Lib/test/test_urllib.py
+++ b/Lib/test/test_urllib.py
@@ -1533,7 +1533,7 @@ def test_pathname2url_win(self):
# Test special prefixes are correctly handled in pathname2url()
fn = urllib.request.pathname2url
self.assertEqual(fn('\\\\?\\C:\\dir'), '///C:/dir')
- self.assertEqual(fn('\\\\?\\unc\\server\\share\\dir'), '/server/share/dir')
+ self.assertEqual(fn('\\\\?\\unc\\server\\share\\dir'), '//server/share/dir')
self.assertEqual(fn("C:"), '///C:')
self.assertEqual(fn("C:\\"), '///C:')
self.assertEqual(fn('C:\\a\\b.c'), '///C:/a/b.c')
@@ -1544,14 +1544,14 @@ def test_pathname2url_win(self):
self.assertRaises(IOError, fn, "XX:\\")
# No drive letter
self.assertEqual(fn("\\folder\\test\\"), '/folder/test/')
- self.assertEqual(fn("\\\\folder\\test\\"), '////folder/test/')
- self.assertEqual(fn("\\\\\\folder\\test\\"), '/////folder/test/')
- self.assertEqual(fn('\\\\some\\share\\'), '////some/share/')
- self.assertEqual(fn('\\\\some\\share\\a\\b.c'), '////some/share/a/b.c')
- self.assertEqual(fn('\\\\some\\share\\a\\b%#c\xe9'), '////some/share/a/b%25%23c%C3%A9')
+ self.assertEqual(fn("\\\\folder\\test\\"), '//folder/test/')
+ self.assertEqual(fn("\\\\\\folder\\test\\"), '///folder/test/')
+ self.assertEqual(fn('\\\\some\\share\\'), '//some/share/')
+ self.assertEqual(fn('\\\\some\\share\\a\\b.c'), '//some/share/a/b.c')
+ self.assertEqual(fn('\\\\some\\share\\a\\b%#c\xe9'), '//some/share/a/b%25%23c%C3%A9')
# Round-tripping
urls = ['///C:',
- '/////folder/test/',
+ '///folder/test/',
'///C:/foo/bar/spam.foo']
for url in urls:
self.assertEqual(fn(urllib.request.url2pathname(url)), url)
diff --git a/Misc/NEWS.d/next/Library/2024-10-30-20-45-17.gh-issue-126205.CHEmtx.rst b/Misc/NEWS.d/next/Library/2024-10-30-20-45-17.gh-issue-126205.CHEmtx.rst
new file mode 100644
index 00000000000000..c92ffb75056606
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-30-20-45-17.gh-issue-126205.CHEmtx.rst
@@ -0,0 +1,2 @@
+Fix issue where :func:`urllib.request.pathname2url` generated URLs beginning
+with four slashes (rather than two) when given a Windows UNC path.
From 8450b2482586857d689b6658f08de9c8179af7db Mon Sep 17 00:00:00 2001
From: Victor Stinner
Date: Thu, 31 Oct 2024 19:09:20 +0100
Subject: [PATCH 139/269] [3.12] gh-124651: Quote template strings in `venv`
activation scripts (GH-124712) (GH-126185)
(cherry picked from commit d48cc82ed25e26b02eb97c6263d95dcaa1e9111b)
---
Lib/test/test_venv.py | 81 +++++++++++++++++++
Lib/venv/__init__.py | 42 ++++++++--
Lib/venv/scripts/common/activate | 10 +--
Lib/venv/scripts/nt/activate.bat | 6 +-
Lib/venv/scripts/posix/activate.csh | 8 +-
Lib/venv/scripts/posix/activate.fish | 8 +-
...-09-28-02-03-04.gh-issue-124651.bLBGtH.rst | 1 +
7 files changed, 135 insertions(+), 21 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-09-28-02-03-04.gh-issue-124651.bLBGtH.rst
diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py
index 83d03aa3bf0d9c..8254a701e09a10 100644
--- a/Lib/test/test_venv.py
+++ b/Lib/test/test_venv.py
@@ -17,6 +17,7 @@
import sys
import sysconfig
import tempfile
+import shlex
from test.support import (captured_stdout, captured_stderr,
skip_if_broken_multiprocessing_synchronize, verbose,
requires_subprocess, is_emscripten, is_wasi,
@@ -97,6 +98,10 @@ def get_text_file_contents(self, *args, encoding='utf-8'):
result = f.read()
return result
+ def assertEndsWith(self, string, tail):
+ if not string.endswith(tail):
+ self.fail(f"String {string!r} does not end with {tail!r}")
+
class BasicTest(BaseTest):
"""Test venv module functionality."""
@@ -446,6 +451,82 @@ def test_executable_symlinks(self):
'import sys; print(sys.executable)'])
self.assertEqual(out.strip(), envpy.encode())
+ # gh-124651: test quoted strings
+ @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows')
+ def test_special_chars_bash(self):
+ """
+ Test that the template strings are quoted properly (bash)
+ """
+ rmtree(self.env_dir)
+ bash = shutil.which('bash')
+ if bash is None:
+ self.skipTest('bash required for this test')
+ env_name = '"\';&&$e|\'"'
+ env_dir = os.path.join(os.path.realpath(self.env_dir), env_name)
+ builder = venv.EnvBuilder(clear=True)
+ builder.create(env_dir)
+ activate = os.path.join(env_dir, self.bindir, 'activate')
+ test_script = os.path.join(self.env_dir, 'test_special_chars.sh')
+ with open(test_script, "w") as f:
+ f.write(f'source {shlex.quote(activate)}\n'
+ 'python -c \'import sys; print(sys.executable)\'\n'
+ 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n'
+ 'deactivate\n')
+ out, err = check_output([bash, test_script])
+ lines = out.splitlines()
+ self.assertTrue(env_name.encode() in lines[0])
+ self.assertEndsWith(lines[1], env_name.encode())
+
+ # gh-124651: test quoted strings
+ @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows')
+ def test_special_chars_csh(self):
+ """
+ Test that the template strings are quoted properly (csh)
+ """
+ rmtree(self.env_dir)
+ csh = shutil.which('tcsh') or shutil.which('csh')
+ if csh is None:
+ self.skipTest('csh required for this test')
+ env_name = '"\';&&$e|\'"'
+ env_dir = os.path.join(os.path.realpath(self.env_dir), env_name)
+ builder = venv.EnvBuilder(clear=True)
+ builder.create(env_dir)
+ activate = os.path.join(env_dir, self.bindir, 'activate.csh')
+ test_script = os.path.join(self.env_dir, 'test_special_chars.csh')
+ with open(test_script, "w") as f:
+ f.write(f'source {shlex.quote(activate)}\n'
+ 'python -c \'import sys; print(sys.executable)\'\n'
+ 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n'
+ 'deactivate\n')
+ out, err = check_output([csh, test_script])
+ lines = out.splitlines()
+ self.assertTrue(env_name.encode() in lines[0])
+ self.assertEndsWith(lines[1], env_name.encode())
+
+ # gh-124651: test quoted strings on Windows
+ @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows')
+ def test_special_chars_windows(self):
+ """
+ Test that the template strings are quoted properly on Windows
+ """
+ rmtree(self.env_dir)
+ env_name = "'&&^$e"
+ env_dir = os.path.join(os.path.realpath(self.env_dir), env_name)
+ builder = venv.EnvBuilder(clear=True)
+ builder.create(env_dir)
+ activate = os.path.join(env_dir, self.bindir, 'activate.bat')
+ test_batch = os.path.join(self.env_dir, 'test_special_chars.bat')
+ with open(test_batch, "w") as f:
+ f.write('@echo off\n'
+ f'"{activate}" & '
+ f'{self.exe} -c "import sys; print(sys.executable)" & '
+ f'{self.exe} -c "import os; print(os.environ[\'VIRTUAL_ENV\'])" & '
+ 'deactivate')
+ out, err = check_output([test_batch])
+ lines = out.splitlines()
+ self.assertTrue(env_name.encode() in lines[0])
+ self.assertEndsWith(lines[1], env_name.encode())
+
@unittest.skipUnless(os.name == 'nt', 'only relevant on Windows')
def test_unicode_in_batch_file(self):
"""
diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py
index d5dec4ab44b8b5..aeb522c3199ca2 100644
--- a/Lib/venv/__init__.py
+++ b/Lib/venv/__init__.py
@@ -11,6 +11,7 @@
import sys
import sysconfig
import types
+import shlex
CORE_VENV_DEPS = ('pip',)
@@ -422,11 +423,41 @@ def replace_variables(self, text, context):
:param context: The information for the environment creation request
being processed.
"""
- text = text.replace('__VENV_DIR__', context.env_dir)
- text = text.replace('__VENV_NAME__', context.env_name)
- text = text.replace('__VENV_PROMPT__', context.prompt)
- text = text.replace('__VENV_BIN_NAME__', context.bin_name)
- text = text.replace('__VENV_PYTHON__', context.env_exe)
+ replacements = {
+ '__VENV_DIR__': context.env_dir,
+ '__VENV_NAME__': context.env_name,
+ '__VENV_PROMPT__': context.prompt,
+ '__VENV_BIN_NAME__': context.bin_name,
+ '__VENV_PYTHON__': context.env_exe,
+ }
+
+ def quote_ps1(s):
+ """
+ This should satisfy PowerShell quoting rules [1], unless the quoted
+ string is passed directly to Windows native commands [2].
+ [1]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_quoting_rules
+ [2]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_parsing#passing-arguments-that-contain-quote-characters
+ """
+ s = s.replace("'", "''")
+ return f"'{s}'"
+
+ def quote_bat(s):
+ return s
+
+ # gh-124651: need to quote the template strings properly
+ quote = shlex.quote
+ script_path = context.script_path
+ if script_path.endswith('.ps1'):
+ quote = quote_ps1
+ elif script_path.endswith('.bat'):
+ quote = quote_bat
+ else:
+ # fallbacks to POSIX shell compliant quote
+ quote = shlex.quote
+
+ replacements = {key: quote(s) for key, s in replacements.items()}
+ for key, quoted in replacements.items():
+ text = text.replace(key, quoted)
return text
def install_scripts(self, context, path):
@@ -466,6 +497,7 @@ def install_scripts(self, context, path):
with open(srcfile, 'rb') as f:
data = f.read()
if not srcfile.endswith(('.exe', '.pdb')):
+ context.script_path = srcfile
try:
data = data.decode('utf-8')
data = self.replace_variables(data, context)
diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate
index 44df44a7404b43..74825877c38313 100644
--- a/Lib/venv/scripts/common/activate
+++ b/Lib/venv/scripts/common/activate
@@ -40,14 +40,14 @@ deactivate nondestructive
if [ "${OSTYPE:-}" = "cygwin" ] || [ "${OSTYPE:-}" = "msys" ] ; then
# transform D:\path\to\venv to /d/path/to/venv on MSYS
# and to /cygdrive/d/path/to/venv on Cygwin
- export VIRTUAL_ENV=$(cygpath "__VENV_DIR__")
+ export VIRTUAL_ENV=$(cygpath __VENV_DIR__)
else
# use the path as-is
- export VIRTUAL_ENV="__VENV_DIR__"
+ export VIRTUAL_ENV=__VENV_DIR__
fi
_OLD_VIRTUAL_PATH="$PATH"
-PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH"
+PATH="$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH"
export PATH
# unset PYTHONHOME if set
@@ -60,9 +60,9 @@ fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1:-}"
- PS1="__VENV_PROMPT__${PS1:-}"
+ PS1=__VENV_PROMPT__"${PS1:-}"
export PS1
- VIRTUAL_ENV_PROMPT="__VENV_PROMPT__"
+ VIRTUAL_ENV_PROMPT=__VENV_PROMPT__
export VIRTUAL_ENV_PROMPT
fi
diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat
index c1c3c82ee37f10..715b21b13fbe35 100644
--- a/Lib/venv/scripts/nt/activate.bat
+++ b/Lib/venv/scripts/nt/activate.bat
@@ -8,7 +8,7 @@ if defined _OLD_CODEPAGE (
"%SystemRoot%\System32\chcp.com" 65001 > nul
)
-set VIRTUAL_ENV=__VENV_DIR__
+set "VIRTUAL_ENV=__VENV_DIR__"
if not defined PROMPT set PROMPT=$P$G
@@ -24,8 +24,8 @@ set PYTHONHOME=
if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH%
if not defined _OLD_VIRTUAL_PATH set _OLD_VIRTUAL_PATH=%PATH%
-set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%
-set VIRTUAL_ENV_PROMPT=__VENV_PROMPT__
+set "PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%"
+set "VIRTUAL_ENV_PROMPT=__VENV_PROMPT__"
:END
if defined _OLD_CODEPAGE (
diff --git a/Lib/venv/scripts/posix/activate.csh b/Lib/venv/scripts/posix/activate.csh
index 5e8d66fa9e5061..08f79296f59352 100644
--- a/Lib/venv/scripts/posix/activate.csh
+++ b/Lib/venv/scripts/posix/activate.csh
@@ -9,17 +9,17 @@ alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PA
# Unset irrelevant variables.
deactivate nondestructive
-setenv VIRTUAL_ENV "__VENV_DIR__"
+setenv VIRTUAL_ENV __VENV_DIR__
set _OLD_VIRTUAL_PATH="$PATH"
-setenv PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH"
+setenv PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH"
set _OLD_VIRTUAL_PROMPT="$prompt"
if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then
- set prompt = "__VENV_PROMPT__$prompt"
- setenv VIRTUAL_ENV_PROMPT "__VENV_PROMPT__"
+ set prompt = __VENV_PROMPT__"$prompt"
+ setenv VIRTUAL_ENV_PROMPT __VENV_PROMPT__
endif
alias pydoc python -m pydoc
diff --git a/Lib/venv/scripts/posix/activate.fish b/Lib/venv/scripts/posix/activate.fish
index 91ad6442e05692..508cab0db468dd 100644
--- a/Lib/venv/scripts/posix/activate.fish
+++ b/Lib/venv/scripts/posix/activate.fish
@@ -33,10 +33,10 @@ end
# Unset irrelevant variables.
deactivate nondestructive
-set -gx VIRTUAL_ENV "__VENV_DIR__"
+set -gx VIRTUAL_ENV __VENV_DIR__
set -gx _OLD_VIRTUAL_PATH $PATH
-set -gx PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__" $PATH
+set -gx PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__ $PATH
# Unset PYTHONHOME if set.
if set -q PYTHONHOME
@@ -56,7 +56,7 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
set -l old_status $status
# Output the venv prompt; color taken from the blue of the Python logo.
- printf "%s%s%s" (set_color 4B8BBE) "__VENV_PROMPT__" (set_color normal)
+ printf "%s%s%s" (set_color 4B8BBE) __VENV_PROMPT__ (set_color normal)
# Restore the return status of the previous command.
echo "exit $old_status" | .
@@ -65,5 +65,5 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
end
set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
- set -gx VIRTUAL_ENV_PROMPT "__VENV_PROMPT__"
+ set -gx VIRTUAL_ENV_PROMPT __VENV_PROMPT__
end
diff --git a/Misc/NEWS.d/next/Library/2024-09-28-02-03-04.gh-issue-124651.bLBGtH.rst b/Misc/NEWS.d/next/Library/2024-09-28-02-03-04.gh-issue-124651.bLBGtH.rst
new file mode 100644
index 00000000000000..17fc9171390dd9
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-09-28-02-03-04.gh-issue-124651.bLBGtH.rst
@@ -0,0 +1 @@
+Properly quote template strings in :mod:`venv` activation scripts.
From dc3c075d9eebc82c63ec54bb3f217d67b2aea914 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 31 Oct 2024 20:33:48 +0100
Subject: [PATCH 140/269] [3.12] gh-126080: fix UAF on `task->task_context` in
`task_call_step_soon` due to an evil `loop.__getattribute__` (GH-126120)
(#126251)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
gh-126080: fix UAF on `task->task_context` in `task_call_step_soon` due to an evil `loop.__getattribute__` (GH-126120)
(cherry picked from commit 0e8665554b2f1334e530fd6de5b3a4e908405419)
Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
---
.../Library/2024-10-29-10-38-28.gh-issue-126080.qKRBuo.rst | 3 +++
Modules/_asynciomodule.c | 6 +++++-
2 files changed, 8 insertions(+), 1 deletion(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-29-10-38-28.gh-issue-126080.qKRBuo.rst
diff --git a/Misc/NEWS.d/next/Library/2024-10-29-10-38-28.gh-issue-126080.qKRBuo.rst b/Misc/NEWS.d/next/Library/2024-10-29-10-38-28.gh-issue-126080.qKRBuo.rst
new file mode 100644
index 00000000000000..e54ac17b217c92
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-29-10-38-28.gh-issue-126080.qKRBuo.rst
@@ -0,0 +1,3 @@
+Fix a use-after-free crash on :class:`asyncio.Task` objects for which the
+underlying event loop implements an evil :meth:`~object.__getattribute__`.
+Reported by Nico-Posada. Patch by Bénédikt Tran.
diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c
index 96821762c5ae68..9bb71623ba6c7e 100644
--- a/Modules/_asynciomodule.c
+++ b/Modules/_asynciomodule.c
@@ -2751,7 +2751,11 @@ task_call_step_soon(asyncio_state *state, TaskObj *task, PyObject *arg)
return -1;
}
- int ret = call_soon(state, task->task_loop, cb, NULL, task->task_context);
+ // Beware: An evil call_soon could alter task_context.
+ // See: https://github.com/python/cpython/issues/126080.
+ PyObject *task_context = Py_NewRef(task->task_context);
+ int ret = call_soon(state, task->task_loop, cb, NULL, task_context);
+ Py_DECREF(task_context);
Py_DECREF(cb);
return ret;
}
From e8ce87219192b9e486198c11b57fea0f573ea9ea Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Fri, 1 Nov 2024 08:25:51 +0100
Subject: [PATCH 141/269] [3.12] gh-126259: Fix "unclosed database" warning in
sqlite3 doctest (GH-126260) (#126266)
(cherry picked from commit 295262c8ecb085b4fea552bc6229af3551bbaf7a)
Co-authored-by: sobolevn
---
Doc/library/sqlite3.rst | 1 +
1 file changed, 1 insertion(+)
diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst
index e85a63531a36aa..3078ec29afe1f0 100644
--- a/Doc/library/sqlite3.rst
+++ b/Doc/library/sqlite3.rst
@@ -2420,6 +2420,7 @@ Some useful URI tricks include:
>>> con.execute("CREATE TABLE readonly(data)")
Traceback (most recent call last):
OperationalError: attempt to write a readonly database
+ >>> con.close()
* Do not implicitly create a new database file if it does not already exist;
will raise :exc:`~sqlite3.OperationalError` if unable to create a new file:
From d268bd04e96d0bd0d0de9cc2660ea9c9f6d7895c Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Fri, 1 Nov 2024 20:29:53 +0100
Subject: [PATCH 142/269] [3.12] gh-122767: document "new style" formatting for
complexes (GH-122848) (#126129)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
gh-122767: document "new style" formatting for complexes (GH-122848)
(cherry picked from commit 0bbbe15f5688552236c48f2b6e320c5312720b8e)
Co-authored-by: Sergey B Kirpichev
Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
---
Doc/library/string.rst | 20 +++++++++++++++-----
1 file changed, 15 insertions(+), 5 deletions(-)
diff --git a/Doc/library/string.rst b/Doc/library/string.rst
index 49aeb28d57c8d1..9e8e44a8abe770 100644
--- a/Doc/library/string.rst
+++ b/Doc/library/string.rst
@@ -350,8 +350,9 @@ The meaning of the various alignment options is as follows:
| ``'='`` | Forces the padding to be placed after the sign (if any) |
| | but before the digits. This is used for printing fields |
| | in the form '+000000120'. This alignment option is only |
-| | valid for numeric types. It becomes the default for |
-| | numbers when '0' immediately precedes the field width. |
+| | valid for numeric types, excluding :class:`complex`. |
+| | It becomes the default for numbers when '0' immediately |
+| | precedes the field width. |
+---------+----------------------------------------------------------+
| ``'^'`` | Forces the field to be centered within the available |
| | space. |
@@ -432,9 +433,9 @@ including any prefixes, separators, and other formatting characters.
If not specified, then the field width will be determined by the content.
When no explicit alignment is given, preceding the *width* field by a zero
-(``'0'``) character enables
-sign-aware zero-padding for numeric types. This is equivalent to a *fill*
-character of ``'0'`` with an *alignment* type of ``'='``.
+(``'0'``) character enables sign-aware zero-padding for numeric types,
+excluding :class:`complex`. This is equivalent to a *fill* character of
+``'0'`` with an *alignment* type of ``'='``.
.. versionchanged:: 3.10
Preceding the *width* field by ``'0'`` no longer affects the default
@@ -588,6 +589,15 @@ The available presentation types for :class:`float` and
| | as altered by the other format modifiers. |
+---------+----------------------------------------------------------+
+The available presentation types for :class:`complex` are the same as those for
+:class:`float` (``'%'`` is not allowed). Both the real and imaginary components
+of a complex number are formatted as floating-point numbers, according to the
+specified presentation type. They are separated by the mandatory sign of the
+imaginary part, the latter being terminated by a ``j`` suffix. If the presentation
+type is missing, the result will match the output of :func:`str` (complex numbers with
+a non-zero real part are also surrounded by parentheses), possibly altered by
+other format modifiers.
+
.. _formatexamples:
From 479e366ce3b61a33a13e1b94b70cd5395710569b Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Fri, 1 Nov 2024 20:36:51 +0100
Subject: [PATCH 143/269] [3.12] Docs: make a tutorial example more precise
(GH-125066) (GH-125079)
Docs: make a tutorial example more precise (GH-125066)
Based on discussion here:
(cherry picked from commit 6e3c70c61bf961e55e9912a31ca11f61c8e2cd0c)
https: //discuss.python.org/t/omission-in-the-documentation/66816
Co-authored-by: Ned Batchelder
---
Doc/tutorial/introduction.rst | 20 +++++++++-----------
1 file changed, 9 insertions(+), 11 deletions(-)
diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst
index 054bac59c955d5..65e3b1938bca9c 100644
--- a/Doc/tutorial/introduction.rst
+++ b/Doc/tutorial/introduction.rst
@@ -197,21 +197,19 @@ and workarounds.
String literals can span multiple lines. One way is using triple-quotes:
``"""..."""`` or ``'''...'''``. End of lines are automatically
included in the string, but it's possible to prevent this by adding a ``\`` at
-the end of the line. The following example::
-
- print("""\
+the end of the line. In the following example, the initial newline is not
+included::
+
+ >>> print("""\
+ ... Usage: thingy [OPTIONS]
+ ... -h Display this usage message
+ ... -H hostname Hostname to connect to
+ ... """)
Usage: thingy [OPTIONS]
-h Display this usage message
-H hostname Hostname to connect to
- """)
-
-produces the following output (note that the initial newline is not included):
-.. code-block:: text
-
- Usage: thingy [OPTIONS]
- -h Display this usage message
- -H hostname Hostname to connect to
+ >>>
Strings can be concatenated (glued together) with the ``+`` operator, and
repeated with ``*``::
From ce5bb3ca850392227ed9d5790c60789eb30daf5f Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Fri, 1 Nov 2024 20:38:23 +0100
Subject: [PATCH 144/269] [3.12] gh-123609: Clarify usage of standalone
`PyBUF_FORMAT` (GH-123778) (GH-123904)
gh-123609: Clarify usage of standalone `PyBUF_FORMAT` (GH-123778)
(cherry picked from commit 962304a54ca79da0838cf46dd4fb744045167cdd)
Co-authored-by: Peter Bierma
---
Doc/c-api/buffer.rst | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst
index 9500fe465c7d94..dc43a3d5fcb094 100644
--- a/Doc/c-api/buffer.rst
+++ b/Doc/c-api/buffer.rst
@@ -244,7 +244,6 @@ The following fields are not influenced by *flags* and must always be filled in
with the correct values: :c:member:`~Py_buffer.obj`, :c:member:`~Py_buffer.buf`,
:c:member:`~Py_buffer.len`, :c:member:`~Py_buffer.itemsize`, :c:member:`~Py_buffer.ndim`.
-
readonly, format
~~~~~~~~~~~~~~~~
@@ -253,7 +252,8 @@ readonly, format
Controls the :c:member:`~Py_buffer.readonly` field. If set, the exporter
MUST provide a writable buffer or else report failure. Otherwise, the
exporter MAY provide either a read-only or writable buffer, but the choice
- MUST be consistent for all consumers.
+ MUST be consistent for all consumers. For example, :c:expr:`PyBUF_SIMPLE | PyBUF_WRITABLE`
+ can be used to request a simple writable buffer.
.. c:macro:: PyBUF_FORMAT
@@ -265,8 +265,9 @@ readonly, format
Since :c:macro:`PyBUF_SIMPLE` is defined as 0, :c:macro:`PyBUF_WRITABLE`
can be used as a stand-alone flag to request a simple writable buffer.
-:c:macro:`PyBUF_FORMAT` can be \|'d to any of the flags except :c:macro:`PyBUF_SIMPLE`.
-The latter already implies format ``B`` (unsigned bytes).
+:c:macro:`PyBUF_FORMAT` must be \|'d to any of the flags except :c:macro:`PyBUF_SIMPLE`, because
+the latter already implies format ``B`` (unsigned bytes). :c:macro:`!PyBUF_FORMAT` cannot be
+used on its own.
shape, strides, suboffsets
From b5c19bdba8f58cba80a1101ce8aad90be3486571 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Fri, 1 Nov 2024 23:18:50 +0100
Subject: [PATCH 145/269] [3.12] gh-126220: Fix crash on calls to
`_lsprof.Profiler` methods with 0 args (backportable) (GH-126271) (#126311)
gh-126220: Fix crash on calls to `_lsprof.Profiler` methods with 0 args (backportable) (GH-126271)
(cherry picked from commit 28b148fb32e4548b461137d18d1ab6d366395d36)
Co-authored-by: sobolevn
Co-authored-by: Erlend E. Aasland
---
Lib/test/test_cprofile.py | 16 +++++++++++++
...-10-31-14-06-28.gh-issue-126220.uJAJCU.rst | 2 ++
Modules/_lsprof.c | 24 +++++++++++++++++++
3 files changed, 42 insertions(+)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-31-14-06-28.gh-issue-126220.uJAJCU.rst
diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py
index 14d69b6f5f3f15..3b7fd034276a22 100644
--- a/Lib/test/test_cprofile.py
+++ b/Lib/test/test_cprofile.py
@@ -30,6 +30,22 @@ def test_bad_counter_during_dealloc(self):
self.assertEqual(cm.unraisable.exc_type, TypeError)
+ def test_crash_with_not_enough_args(self):
+ # gh-126220
+ import _lsprof
+
+ for profile in [_lsprof.Profiler(), cProfile.Profile()]:
+ for method in [
+ "_pystart_callback",
+ "_pyreturn_callback",
+ "_ccall_callback",
+ "_creturn_callback",
+ ]:
+ with self.subTest(profile=profile, method=method):
+ method_obj = getattr(profile, method)
+ with self.assertRaises(TypeError):
+ method_obj() # should not crash
+
def test_evil_external_timer(self):
# gh-120289
# Disabling profiler in external timer should not crash
diff --git a/Misc/NEWS.d/next/Library/2024-10-31-14-06-28.gh-issue-126220.uJAJCU.rst b/Misc/NEWS.d/next/Library/2024-10-31-14-06-28.gh-issue-126220.uJAJCU.rst
new file mode 100644
index 00000000000000..555f2f3bafbf33
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-31-14-06-28.gh-issue-126220.uJAJCU.rst
@@ -0,0 +1,2 @@
+Fix crash in :class:`!cProfile.Profile` and :class:`!_lsprof.Profiler` when their
+callbacks were directly called with 0 arguments.
diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c
index 2c82b18c0e18d7..d11689906e574d 100644
--- a/Modules/_lsprof.c
+++ b/Modules/_lsprof.c
@@ -604,6 +604,12 @@ setBuiltins(ProfilerObject *pObj, int nvalue)
PyObject* pystart_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size)
{
+ if (size < 2) {
+ PyErr_Format(PyExc_TypeError,
+ "_pystart_callback expected 2 arguments, got %zd",
+ size);
+ return NULL;
+ }
PyObject* code = args[0];
ptrace_enter_call((PyObject*)self, (void *)code, (PyObject *)code);
@@ -612,6 +618,12 @@ PyObject* pystart_callback(ProfilerObject* self, PyObject *const *args, Py_ssize
PyObject* pyreturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size)
{
+ if (size < 3) {
+ PyErr_Format(PyExc_TypeError,
+ "_pyreturn_callback expected 3 arguments, got %zd",
+ size);
+ return NULL;
+ }
PyObject* code = args[0];
ptrace_leave_call((PyObject*)self, (void *)code);
@@ -647,6 +659,12 @@ PyObject* get_cfunc_from_callable(PyObject* callable, PyObject* self_arg, PyObje
PyObject* ccall_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size)
{
+ if (size < 4) {
+ PyErr_Format(PyExc_TypeError,
+ "_ccall_callback expected 4 arguments, got %zd",
+ size);
+ return NULL;
+ }
if (self->flags & POF_BUILTINS) {
PyObject* callable = args[2];
PyObject* self_arg = args[3];
@@ -665,6 +683,12 @@ PyObject* ccall_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t
PyObject* creturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size)
{
+ if (size < 4) {
+ PyErr_Format(PyExc_TypeError,
+ "_creturn_callback expected 4 arguments, got %zd",
+ size);
+ return NULL;
+ }
if (self->flags & POF_BUILTINS) {
PyObject* callable = args[2];
PyObject* self_arg = args[3];
From 8d4ef5280215207ee112ac6594d252c3f38b44bc Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sat, 2 Nov 2024 00:18:00 +0100
Subject: [PATCH 146/269] [3.12] docs: add a more precise example in enum doc
(GH-121015) (#126307)
docs: add a more precise example in enum doc (GH-121015)
* docs: add a more precise example
Previous example used manual integer value assignment in class based declaration but in functional syntax has been used auto value assignment what could be confusing for the new users. Additionally documentation doesn't show how to declare new enum via functional syntax with usage of the manual value assignment.
* docs: remove whitespace characters
* refactor: change example
---------
(cherry picked from commit ff257c7843d8ed0dffb6624f2f14996a46e74801)
Co-authored-by: Filip "Ret2Me" Poplewski <37419029+Ret2Me@users.noreply.github.com>
Co-authored-by: Ethan Furman
---
Doc/library/enum.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst
index 6e2872b9c70731..a4b6a53d29ee35 100644
--- a/Doc/library/enum.rst
+++ b/Doc/library/enum.rst
@@ -44,7 +44,7 @@ using function-call syntax::
... BLUE = 3
>>> # functional syntax
- >>> Color = Enum('Color', ['RED', 'GREEN', 'BLUE'])
+ >>> Color = Enum('Color', [('RED', 1), ('GREEN', 2), ('BLUE', 3)])
Even though we can use :keyword:`class` syntax to create Enums, Enums
are not normal Python classes. See
From b59269961f73cecd5d14ef4fa04689170aa0b2f3 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sat, 2 Nov 2024 09:03:51 +0100
Subject: [PATCH 147/269] [3.12] gh-126138: Fix use-after-free in
`_asyncio.Task` by evil `__getattribute__` (GH-126305) (#126325)
gh-126138: Fix use-after-free in `_asyncio.Task` by evil `__getattribute__` (GH-126305)
(cherry picked from commit f032f6ba8fec6fab35edeec0eb40cd73e9d58928)
Co-authored-by: Nico-Posada <102486290+Nico-Posada@users.noreply.github.com>
Co-authored-by: Carol Willing
---
...-11-01-14-31-41.gh-issue-126138.yTniOG.rst | 3 +++
Modules/_asynciomodule.c | 22 +++++++++++++++++--
2 files changed, 23 insertions(+), 2 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-11-01-14-31-41.gh-issue-126138.yTniOG.rst
diff --git a/Misc/NEWS.d/next/Library/2024-11-01-14-31-41.gh-issue-126138.yTniOG.rst b/Misc/NEWS.d/next/Library/2024-11-01-14-31-41.gh-issue-126138.yTniOG.rst
new file mode 100644
index 00000000000000..459eebc82bd42a
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-11-01-14-31-41.gh-issue-126138.yTniOG.rst
@@ -0,0 +1,3 @@
+Fix a use-after-free crash on :class:`asyncio.Task` objects
+whose underlying coroutine yields an object that implements
+an evil :meth:`~object.__getattribute__`. Patch by Nico Posada.
diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c
index 9bb71623ba6c7e..20a9d992834b0e 100644
--- a/Modules/_asynciomodule.c
+++ b/Modules/_asynciomodule.c
@@ -2994,8 +2994,17 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu
if (task->task_must_cancel) {
PyObject *r;
int is_true;
+
+ // Beware: An evil `__getattribute__` could
+ // prematurely delete task->task_cancel_msg before the
+ // task is cancelled, thereby causing a UAF crash.
+ //
+ // See https://github.com/python/cpython/issues/126138
+ PyObject *task_cancel_msg = Py_NewRef(task->task_cancel_msg);
r = PyObject_CallMethodOneArg(result, &_Py_ID(cancel),
- task->task_cancel_msg);
+ task_cancel_msg);
+ Py_DECREF(task_cancel_msg);
+
if (r == NULL) {
return NULL;
}
@@ -3087,8 +3096,17 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu
if (task->task_must_cancel) {
PyObject *r;
int is_true;
+
+ // Beware: An evil `__getattribute__` could
+ // prematurely delete task->task_cancel_msg before the
+ // task is cancelled, thereby causing a UAF crash.
+ //
+ // See https://github.com/python/cpython/issues/126138
+ PyObject *task_cancel_msg = Py_NewRef(task->task_cancel_msg);
r = PyObject_CallMethodOneArg(result, &_Py_ID(cancel),
- task->task_cancel_msg);
+ task_cancel_msg);
+ Py_DECREF(task_cancel_msg);
+
if (r == NULL) {
return NULL;
}
From eecea8f64a8873c1466d4b81f4bd398b835b6cbf Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sat, 2 Nov 2024 09:49:32 +0100
Subject: [PATCH 148/269] [3.12] gh-125522: Remove bare except in
test_zlib.test_flushes (gh-126321) (gh-126328)
gh-125522: Remove bare except in test_zlib.test_flushes (gh-126321)
(cherry picked from commit cfb1b2f0cb999558a30e61a9e1a62fdb7f55f6a4)
Co-authored-by: simple-is-great <103080930+simple-is-great@users.noreply.github.com>
---
Lib/test/test_zlib.py | 14 +++++---------
1 file changed, 5 insertions(+), 9 deletions(-)
diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py
index 8654b93ec64ac8..9611a0d2d02e46 100644
--- a/Lib/test/test_zlib.py
+++ b/Lib/test/test_zlib.py
@@ -506,20 +506,16 @@ def test_flushes(self):
for sync in sync_opt:
for level in range(10):
- try:
+ with self.subTest(sync=sync, level=level):
obj = zlib.compressobj( level )
a = obj.compress( data[:3000] )
b = obj.flush( sync )
c = obj.compress( data[3000:] )
d = obj.flush()
- except:
- print("Error for flush mode={}, level={}"
- .format(sync, level))
- raise
- self.assertEqual(zlib.decompress(b''.join([a,b,c,d])),
- data, ("Decompress failed: flush "
- "mode=%i, level=%i") % (sync, level))
- del obj
+ self.assertEqual(zlib.decompress(b''.join([a,b,c,d])),
+ data, ("Decompress failed: flush "
+ "mode=%i, level=%i") % (sync, level))
+ del obj
@unittest.skipUnless(hasattr(zlib, 'Z_SYNC_FLUSH'),
'requires zlib.Z_SYNC_FLUSH')
From f4bc64d204dce541c20e1425a3b70cab6e811607 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sat, 2 Nov 2024 12:45:36 +0100
Subject: [PATCH 149/269] [3.12] gh-125761: Clarify repeated warning
suppression criteria in warnings module (gh-126331)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
(cherry picked from commit 10eeec2d4ffb6b09a6d925877b6d9ef6aa6bb59d)
Co-authored-by: 고병찬 <70642609+byungchanKo99@users.noreply.github.com>
---
Doc/library/warnings.rst | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst
index 0cd3ca3c9de73f..53a402e3fcddf0 100644
--- a/Doc/library/warnings.rst
+++ b/Doc/library/warnings.rst
@@ -178,6 +178,19 @@ If a warning is reported and doesn't match any registered filter then the
"default" action is applied (hence its name).
+
+.. _repeated-warning-suppression-criteria:
+
+Repeated Warning Suppression Criteria
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The filters that suppress repeated warnings apply the following criteria to determine if a warning is considered a repeat:
+
+- ``"default"``: A warning is considered a repeat only if the (*message*, *category*, *module*, *lineno*) are all the same.
+- ``"module"``: A warning is considered a repeat if the (*message*, *category*, *module*) are the same, ignoring the line number.
+- ``"once"``: A warning is considered a repeat if the (*message*, *category*) are the same, ignoring the module and line number.
+
+
.. _describing-warning-filters:
Describing Warning Filters
From 4afa129ed08780ef84fcee4dd1168a33034f2ea5 Mon Sep 17 00:00:00 2001
From: Donghee Na
Date: Sun, 3 Nov 2024 13:00:52 +0900
Subject: [PATCH 150/269] =?UTF-8?q?[3.12]=20gh-125832:=20Clarify=20comment?=
=?UTF-8?q?=20for=20inlined=20comprehensions=20as=20per=20P=E2=80=A6=20(gh?=
=?UTF-8?q?-126345)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* [3.12] gh-125832: Clarify comment for inlined comprehensions as per PEP-709 (gh-126322)
* Fix comprehensions comment to inlined by pep 709
* Update spacing
Co-authored-by: RUANG (James Roy)
* Add reference to PEP 709
---------
Co-authored-by: Carol Willing
Co-authored-by: RUANG (James Roy)
* Add space
---------
Co-authored-by: rimchoi
Co-authored-by: Carol Willing
Co-authored-by: RUANG (James Roy)
---
Python/compile.c | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/Python/compile.c b/Python/compile.c
index 49b2ebf003bad4..6050b7281fa051 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -5191,9 +5191,12 @@ compiler_call_helper(struct compiler *c, location loc,
}
-/* List and set comprehensions and generator expressions work by creating a
- nested function to perform the actual iteration. This means that the
- iteration variables don't leak into the current scope.
+/* List and set comprehensions work by being inlined at the location where
+ they are defined. The isolation of iteration variables is provided by
+ pushing/popping clashing locals on the stack. Generator expressions work
+ by creating a nested function to perform the actual iteration.
+ This means that the iteration variables don't leak into the current scope.
+ See https://peps.python.org/pep-0709/ for additional information.
The defined function is called immediately following its definition, with the
result of that call being the result of the expression.
The LC/SC version returns the populated container, while the GE version is
From 6e3e91fa76bd330b13eb65745911b3ef87190a1f Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sun, 3 Nov 2024 06:48:19 +0100
Subject: [PATCH 151/269] [3.12] gh-113977, gh-120754: Remove unbounded reads
from zipfile (GH-122101) (#126347)
gh-113977, gh-120754: Remove unbounded reads from zipfile (GH-122101)
GH-113977, GH-120754: Remove unbounded reads from zipfile
Read without a size may read an unbounded amount of data + allocate
unbounded size buffers. Move to capped size reads to prevent potential
issues.
(cherry picked from commit 556dc9b8a78bad296513221f3f414a3f8fd0ae70)
Co-authored-by: Cody Maloney
Co-authored-by: Daniel Hillier
Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>
---
Lib/zipfile/__init__.py | 6 +++---
.../Library/2024-07-23-02-24-50.gh-issue-120754.nHb5mG.rst | 1 +
2 files changed, 4 insertions(+), 3 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-07-23-02-24-50.gh-issue-120754.nHb5mG.rst
diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py
index 91358156bc1e49..cf71c6dba2b836 100644
--- a/Lib/zipfile/__init__.py
+++ b/Lib/zipfile/__init__.py
@@ -295,7 +295,7 @@ def _EndRecData(fpin):
fpin.seek(-sizeEndCentDir, 2)
except OSError:
return None
- data = fpin.read()
+ data = fpin.read(sizeEndCentDir)
if (len(data) == sizeEndCentDir and
data[0:4] == stringEndArchive and
data[-2:] == b"\000\000"):
@@ -315,9 +315,9 @@ def _EndRecData(fpin):
# record signature. The comment is the last item in the ZIP file and may be
# up to 64K long. It is assumed that the "end of central directory" magic
# number does not appear in the comment.
- maxCommentStart = max(filesize - (1 << 16) - sizeEndCentDir, 0)
+ maxCommentStart = max(filesize - ZIP_MAX_COMMENT - sizeEndCentDir, 0)
fpin.seek(maxCommentStart, 0)
- data = fpin.read()
+ data = fpin.read(ZIP_MAX_COMMENT + sizeEndCentDir)
start = data.rfind(stringEndArchive)
if start >= 0:
# found the magic number; attempt to unpack and interpret
diff --git a/Misc/NEWS.d/next/Library/2024-07-23-02-24-50.gh-issue-120754.nHb5mG.rst b/Misc/NEWS.d/next/Library/2024-07-23-02-24-50.gh-issue-120754.nHb5mG.rst
new file mode 100644
index 00000000000000..6c33e7b7ec7716
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-07-23-02-24-50.gh-issue-120754.nHb5mG.rst
@@ -0,0 +1 @@
+Update unbounded ``read`` calls in :mod:`zipfile` to specify an explicit ``size`` putting a limit on how much data they may read. This also updates handling around ZIP max comment size to match the standard instead of reading comments that are one byte too long.
From b0e08f513b116986f53053811d946df4e3f8ceab Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sun, 3 Nov 2024 15:24:41 +0100
Subject: [PATCH 152/269] [3.12] gh-104400: Add more tests to pygettext
(GH-108173) (GH-126362)
(cherry picked from commit dcae5cd6abaae4f73e656ebc054f30d3f15ca7b8)
Co-authored-by: Tomas R
---
Lib/test/test_tools/i18n_data/docstrings.pot | 40 +++++++
Lib/test/test_tools/i18n_data/docstrings.py | 41 +++++++
Lib/test/test_tools/i18n_data/fileloc.pot | 35 ++++++
Lib/test/test_tools/i18n_data/fileloc.py | 26 +++++
Lib/test/test_tools/i18n_data/messages.pot | 67 +++++++++++
Lib/test/test_tools/i18n_data/messages.py | 64 +++++++++++
Lib/test/test_tools/test_i18n.py | 110 +++++++++++++++----
Makefile.pre.in | 1 +
8 files changed, 363 insertions(+), 21 deletions(-)
create mode 100644 Lib/test/test_tools/i18n_data/docstrings.pot
create mode 100644 Lib/test/test_tools/i18n_data/docstrings.py
create mode 100644 Lib/test/test_tools/i18n_data/fileloc.pot
create mode 100644 Lib/test/test_tools/i18n_data/fileloc.py
create mode 100644 Lib/test/test_tools/i18n_data/messages.pot
create mode 100644 Lib/test/test_tools/i18n_data/messages.py
diff --git a/Lib/test/test_tools/i18n_data/docstrings.pot b/Lib/test/test_tools/i18n_data/docstrings.pot
new file mode 100644
index 00000000000000..5af1d41422ff62
--- /dev/null
+++ b/Lib/test/test_tools/i18n_data/docstrings.pot
@@ -0,0 +1,40 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR , YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2000-01-01 00:00+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME \n"
+"Language-Team: LANGUAGE \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: docstrings.py:7
+#, docstring
+msgid ""
+msgstr ""
+
+#: docstrings.py:18
+#, docstring
+msgid ""
+"multiline\n"
+" docstring\n"
+" "
+msgstr ""
+
+#: docstrings.py:25
+#, docstring
+msgid "docstring1"
+msgstr ""
+
+#: docstrings.py:30
+#, docstring
+msgid "Hello, {}!"
+msgstr ""
+
diff --git a/Lib/test/test_tools/i18n_data/docstrings.py b/Lib/test/test_tools/i18n_data/docstrings.py
new file mode 100644
index 00000000000000..85d7f159d37775
--- /dev/null
+++ b/Lib/test/test_tools/i18n_data/docstrings.py
@@ -0,0 +1,41 @@
+# Test docstring extraction
+from gettext import gettext as _
+
+
+# Empty docstring
+def test(x):
+ """"""
+
+
+# Leading empty line
+def test2(x):
+
+ """docstring""" # XXX This should be extracted but isn't.
+
+
+# XXX Multiline docstrings should be cleaned with `inspect.cleandoc`.
+def test3(x):
+ """multiline
+ docstring
+ """
+
+
+# Multiple docstrings - only the first should be extracted
+def test4(x):
+ """docstring1"""
+ """docstring2"""
+
+
+def test5(x):
+ """Hello, {}!""".format("world!") # XXX This should not be extracted.
+
+
+# Nested docstrings
+def test6(x):
+ def inner(y):
+ """nested docstring""" # XXX This should be extracted but isn't.
+
+
+class Outer:
+ class Inner:
+ "nested class docstring" # XXX This should be extracted but isn't.
diff --git a/Lib/test/test_tools/i18n_data/fileloc.pot b/Lib/test/test_tools/i18n_data/fileloc.pot
new file mode 100644
index 00000000000000..dbd28687a73556
--- /dev/null
+++ b/Lib/test/test_tools/i18n_data/fileloc.pot
@@ -0,0 +1,35 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR , YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2000-01-01 00:00+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME \n"
+"Language-Team: LANGUAGE \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: fileloc.py:5 fileloc.py:6
+msgid "foo"
+msgstr ""
+
+#: fileloc.py:9
+msgid "bar"
+msgstr ""
+
+#: fileloc.py:14 fileloc.py:18
+#, docstring
+msgid "docstring"
+msgstr ""
+
+#: fileloc.py:22 fileloc.py:26
+#, docstring
+msgid "baz"
+msgstr ""
+
diff --git a/Lib/test/test_tools/i18n_data/fileloc.py b/Lib/test/test_tools/i18n_data/fileloc.py
new file mode 100644
index 00000000000000..c5d4d0595fea52
--- /dev/null
+++ b/Lib/test/test_tools/i18n_data/fileloc.py
@@ -0,0 +1,26 @@
+# Test file locations
+from gettext import gettext as _
+
+# Duplicate strings
+_('foo')
+_('foo')
+
+# Duplicate strings on the same line should only add one location to the output
+_('bar'), _('bar')
+
+
+# Duplicate docstrings
+class A:
+ """docstring"""
+
+
+def f():
+ """docstring"""
+
+
+# Duplicate message and docstring
+_('baz')
+
+
+def g():
+ """baz"""
diff --git a/Lib/test/test_tools/i18n_data/messages.pot b/Lib/test/test_tools/i18n_data/messages.pot
new file mode 100644
index 00000000000000..ddfbd18349ef4f
--- /dev/null
+++ b/Lib/test/test_tools/i18n_data/messages.pot
@@ -0,0 +1,67 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR , YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2000-01-01 00:00+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME \n"
+"Language-Team: LANGUAGE \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: messages.py:5
+msgid ""
+msgstr ""
+
+#: messages.py:8 messages.py:9
+msgid "parentheses"
+msgstr ""
+
+#: messages.py:12
+msgid "Hello, world!"
+msgstr ""
+
+#: messages.py:15
+msgid ""
+"Hello,\n"
+" multiline!\n"
+msgstr ""
+
+#: messages.py:29
+msgid "Hello, {}!"
+msgstr ""
+
+#: messages.py:33
+msgid "1"
+msgstr ""
+
+#: messages.py:33
+msgid "2"
+msgstr ""
+
+#: messages.py:34 messages.py:35
+msgid "A"
+msgstr ""
+
+#: messages.py:34 messages.py:35
+msgid "B"
+msgstr ""
+
+#: messages.py:36
+msgid "set"
+msgstr ""
+
+#: messages.py:42
+msgid "nested string"
+msgstr ""
+
+#: messages.py:47
+msgid "baz"
+msgstr ""
+
diff --git a/Lib/test/test_tools/i18n_data/messages.py b/Lib/test/test_tools/i18n_data/messages.py
new file mode 100644
index 00000000000000..f220294b8d5c67
--- /dev/null
+++ b/Lib/test/test_tools/i18n_data/messages.py
@@ -0,0 +1,64 @@
+# Test message extraction
+from gettext import gettext as _
+
+# Empty string
+_("")
+
+# Extra parentheses
+(_("parentheses"))
+((_("parentheses")))
+
+# Multiline strings
+_("Hello, "
+ "world!")
+
+_("""Hello,
+ multiline!
+""")
+
+# Invalid arguments
+_()
+_(None)
+_(1)
+_(False)
+_(x="kwargs are not allowed")
+_("foo", "bar")
+_("something", x="something else")
+
+# .format()
+_("Hello, {}!").format("world") # valid
+_("Hello, {}!".format("world")) # invalid
+
+# Nested structures
+_("1"), _("2")
+arr = [_("A"), _("B")]
+obj = {'a': _("A"), 'b': _("B")}
+{{{_('set')}}}
+
+
+# Nested functions and classes
+def test():
+ _("nested string") # XXX This should be extracted but isn't.
+ [_("nested string")]
+
+
+class Foo:
+ def bar(self):
+ return _("baz")
+
+
+def bar(x=_('default value')): # XXX This should be extracted but isn't.
+ pass
+
+
+def baz(x=[_('default value')]): # XXX This should be extracted but isn't.
+ pass
+
+
+# Shadowing _()
+def _(x):
+ pass
+
+
+def _(x="don't extract me"):
+ pass
diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py
index c083a04475e726..21dead8f943bb7 100644
--- a/Lib/test/test_tools/test_i18n.py
+++ b/Lib/test/test_tools/test_i18n.py
@@ -1,9 +1,11 @@
"""Tests to cover the Tools/i18n package"""
import os
+import re
import sys
import unittest
from textwrap import dedent
+from pathlib import Path
from test.support.script_helper import assert_python_ok
from test.test_tools import skip_if_missing, toolsdir
@@ -12,20 +14,47 @@
skip_if_missing()
+DATA_DIR = Path(__file__).resolve().parent / 'i18n_data'
+
+
+def normalize_POT_file(pot):
+ """Normalize the POT creation timestamp, charset and
+ file locations to make the POT file easier to compare.
+
+ """
+ # Normalize the creation date.
+ date_pattern = re.compile(r'"POT-Creation-Date: .+?\\n"')
+ header = r'"POT-Creation-Date: 2000-01-01 00:00+0000\\n"'
+ pot = re.sub(date_pattern, header, pot)
+
+ # Normalize charset to UTF-8 (currently there's no way to specify the output charset).
+ charset_pattern = re.compile(r'"Content-Type: text/plain; charset=.+?\\n"')
+ charset = r'"Content-Type: text/plain; charset=UTF-8\\n"'
+ pot = re.sub(charset_pattern, charset, pot)
+
+ # Normalize file location path separators in case this test is
+ # running on Windows (which uses '\').
+ fileloc_pattern = re.compile(r'#:.+')
+
+ def replace(match):
+ return match[0].replace(os.sep, "/")
+ pot = re.sub(fileloc_pattern, replace, pot)
+ return pot
+
class Test_pygettext(unittest.TestCase):
"""Tests for the pygettext.py tool"""
- script = os.path.join(toolsdir,'i18n', 'pygettext.py')
+ script = Path(toolsdir, 'i18n', 'pygettext.py')
def get_header(self, data):
""" utility: return the header of a .po file as a dictionary """
headers = {}
for line in data.split('\n'):
- if not line or line.startswith(('#', 'msgid','msgstr')):
+ if not line or line.startswith(('#', 'msgid', 'msgstr')):
continue
line = line.strip('"')
- key, val = line.split(':',1)
+ key, val = line.split(':', 1)
headers[key] = val.strip()
return headers
@@ -53,13 +82,18 @@ def get_msgids(self, data):
return msgids
+ def assert_POT_equal(self, expected, actual):
+ """Check if two POT files are equal"""
+ self.maxDiff = None
+ self.assertEqual(normalize_POT_file(expected), normalize_POT_file(actual))
+
def extract_docstrings_from_str(self, module_content):
""" utility: return all msgids extracted from module_content """
filename = 'test_docstrings.py'
with temp_cwd(None) as cwd:
with open(filename, 'w', encoding='utf-8') as fp:
fp.write(module_content)
- assert_python_ok(self.script, '-D', filename)
+ assert_python_ok('-Xutf8', self.script, '-D', filename)
with open('messages.pot', encoding='utf-8') as fp:
data = fp.read()
return self.get_msgids(data)
@@ -69,7 +103,7 @@ def test_header(self):
http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry
"""
with temp_cwd(None) as cwd:
- assert_python_ok(self.script)
+ assert_python_ok('-Xutf8', self.script)
with open('messages.pot', encoding='utf-8') as fp:
data = fp.read()
header = self.get_header(data)
@@ -96,7 +130,7 @@ def test_POT_Creation_Date(self):
""" Match the date format from xgettext for POT-Creation-Date """
from datetime import datetime
with temp_cwd(None) as cwd:
- assert_python_ok(self.script)
+ assert_python_ok('-Xutf8', self.script)
with open('messages.pot', encoding='utf-8') as fp:
data = fp.read()
header = self.get_header(data)
@@ -310,6 +344,20 @@ def test_calls_in_fstring_with_partially_wrong_expression(self):
self.assertNotIn('foo', msgids)
self.assertIn('bar', msgids)
+ def test_pygettext_output(self):
+ """Test that the pygettext output exactly matches snapshots."""
+ for input_file in DATA_DIR.glob('*.py'):
+ output_file = input_file.with_suffix('.pot')
+ with self.subTest(input_file=f'i18n_data/{input_file}'):
+ contents = input_file.read_text(encoding='utf-8')
+ with temp_cwd(None):
+ Path(input_file.name).write_text(contents)
+ assert_python_ok('-Xutf8', self.script, '--docstrings', input_file.name)
+ output = Path('messages.pot').read_text(encoding='utf-8')
+
+ expected = output_file.read_text(encoding='utf-8')
+ self.assert_POT_equal(expected, output)
+
def test_files_list(self):
"""Make sure the directories are inspected for source files
bpo-31920
@@ -318,21 +366,41 @@ def test_files_list(self):
text2 = 'Text to translate2'
text3 = 'Text to ignore'
with temp_cwd(None), temp_dir(None) as sdir:
- os.mkdir(os.path.join(sdir, 'pypkg'))
- with open(os.path.join(sdir, 'pypkg', 'pymod.py'), 'w',
- encoding='utf-8') as sfile:
- sfile.write(f'_({text1!r})')
- os.mkdir(os.path.join(sdir, 'pkg.py'))
- with open(os.path.join(sdir, 'pkg.py', 'pymod2.py'), 'w',
- encoding='utf-8') as sfile:
- sfile.write(f'_({text2!r})')
- os.mkdir(os.path.join(sdir, 'CVS'))
- with open(os.path.join(sdir, 'CVS', 'pymod3.py'), 'w',
- encoding='utf-8') as sfile:
- sfile.write(f'_({text3!r})')
- assert_python_ok(self.script, sdir)
- with open('messages.pot', encoding='utf-8') as fp:
- data = fp.read()
+ pymod = Path(sdir, 'pypkg', 'pymod.py')
+ pymod.parent.mkdir()
+ pymod.write_text(f'_({text1!r})', encoding='utf-8')
+
+ pymod2 = Path(sdir, 'pkg.py', 'pymod2.py')
+ pymod2.parent.mkdir()
+ pymod2.write_text(f'_({text2!r})', encoding='utf-8')
+
+ pymod3 = Path(sdir, 'CVS', 'pymod3.py')
+ pymod3.parent.mkdir()
+ pymod3.write_text(f'_({text3!r})', encoding='utf-8')
+
+ assert_python_ok('-Xutf8', self.script, sdir)
+ data = Path('messages.pot').read_text(encoding='utf-8')
self.assertIn(f'msgid "{text1}"', data)
self.assertIn(f'msgid "{text2}"', data)
self.assertNotIn(text3, data)
+
+
+def update_POT_snapshots():
+ for input_file in DATA_DIR.glob('*.py'):
+ output_file = input_file.with_suffix('.pot')
+ contents = input_file.read_bytes()
+ with temp_cwd(None):
+ Path(input_file.name).write_bytes(contents)
+ assert_python_ok('-Xutf8', Test_pygettext.script, '--docstrings', input_file.name)
+ output = Path('messages.pot').read_text(encoding='utf-8')
+
+ output = normalize_POT_file(output)
+ output_file.write_text(output, encoding='utf-8')
+
+
+if __name__ == '__main__':
+ # To regenerate POT files
+ if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update':
+ update_POT_snapshots()
+ sys.exit(0)
+ unittest.main()
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 14eea08dcfbd33..f87de823974e96 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -2235,6 +2235,7 @@ TESTSUBDIRS= idlelib/idle_test \
test/test_tomllib/data/valid/dates-and-times \
test/test_tomllib/data/valid/multiline-basic-str \
test/test_tools \
+ test/test_tools/i18n_data \
test/test_ttk \
test/test_unittest \
test/test_unittest/testmock \
From 37dc2b2bbb2b5768f2c8bf84f5fec4198a3e092d Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sun, 3 Nov 2024 22:31:59 +0100
Subject: [PATCH 153/269] [3.12] Docs: Delist sqlite3 deprecation from "Pending
removal in 3.14" (GH-126370) (#126373)
Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
---
Doc/deprecations/pending-removal-in-3.14.rst | 3 ---
1 file changed, 3 deletions(-)
diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst
index 5f01e093ce1b8b..7eaa80e96138af 100644
--- a/Doc/deprecations/pending-removal-in-3.14.rst
+++ b/Doc/deprecations/pending-removal-in-3.14.rst
@@ -106,9 +106,6 @@ Pending Removal in Python 3.14
if :ref:`named placeholders ` are used and
*parameters* is a sequence instead of a :class:`dict`.
- * date and datetime adapter, date and timestamp converter:
- see the :mod:`sqlite3` documentation for suggested replacement recipes.
-
* :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was
deprecated in :pep:`626`
since 3.10 and was planned to be removed in 3.12,
From d1b87189348eb4554fe7b468d1bd331f4f4345ea Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 4 Nov 2024 05:16:35 +0100
Subject: [PATCH 154/269] [3.12] gh-126165: Improve docs of function
`math.isclose` (GH-126215) (#126381)
gh-126165: Improve docs of function `math.isclose` (GH-126215)
(cherry picked from commit 081706f873b7d1a10b27016a9ed350b20c719709)
Co-authored-by: Zhikang Yan <2951256653@qq.com>
Co-authored-by: Sergey B Kirpichev
Co-authored-by: Carol Willing
Co-authored-by: Terry Jan Reedy
---
Doc/library/cmath.rst | 18 ++++++++++--------
Doc/library/math.rst | 18 ++++++++++--------
2 files changed, 20 insertions(+), 16 deletions(-)
diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst
index 381a8332f4b187..f122e3644ece56 100644
--- a/Doc/library/cmath.rst
+++ b/Doc/library/cmath.rst
@@ -221,19 +221,21 @@ Classification functions
``False`` otherwise.
Whether or not two values are considered close is determined according to
- given absolute and relative tolerances.
+ given absolute and relative tolerances. If no errors occur, the result will
+ be: ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``.
*rel_tol* is the relative tolerance -- it is the maximum allowed difference
between *a* and *b*, relative to the larger absolute value of *a* or *b*.
For example, to set a tolerance of 5%, pass ``rel_tol=0.05``. The default
tolerance is ``1e-09``, which assures that the two values are the same
- within about 9 decimal digits. *rel_tol* must be greater than zero.
-
- *abs_tol* is the minimum absolute tolerance -- useful for comparisons near
- zero. *abs_tol* must be at least zero.
-
- If no errors occur, the result will be:
- ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``.
+ within about 9 decimal digits. *rel_tol* must be nonnegative and less
+ than ``1.0``.
+
+ *abs_tol* is the absolute tolerance; it defaults to ``0.0`` and it must be
+ nonnegative. When comparing ``x`` to ``0.0``, ``isclose(x, 0)`` is computed
+ as ``abs(x) <= rel_tol * abs(x)``, which is ``False`` for any ``x`` and
+ rel_tol less than ``1.0``. So add an appropriate positive abs_tol argument
+ to the call.
The IEEE 754 special values of ``NaN``, ``inf``, and ``-inf`` will be
handled according to IEEE rules. Specifically, ``NaN`` is not considered
diff --git a/Doc/library/math.rst b/Doc/library/math.rst
index 40742fdafeac03..fb8527128a2555 100644
--- a/Doc/library/math.rst
+++ b/Doc/library/math.rst
@@ -142,19 +142,21 @@ Number-theoretic and representation functions
``False`` otherwise.
Whether or not two values are considered close is determined according to
- given absolute and relative tolerances.
+ given absolute and relative tolerances. If no errors occur, the result will
+ be: ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``.
*rel_tol* is the relative tolerance -- it is the maximum allowed difference
between *a* and *b*, relative to the larger absolute value of *a* or *b*.
For example, to set a tolerance of 5%, pass ``rel_tol=0.05``. The default
tolerance is ``1e-09``, which assures that the two values are the same
- within about 9 decimal digits. *rel_tol* must be greater than zero.
-
- *abs_tol* is the minimum absolute tolerance -- useful for comparisons near
- zero. *abs_tol* must be at least zero.
-
- If no errors occur, the result will be:
- ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``.
+ within about 9 decimal digits. *rel_tol* must be nonnegative and less
+ than ``1.0``.
+
+ *abs_tol* is the absolute tolerance; it defaults to ``0.0`` and it must be
+ nonnegative. When comparing ``x`` to ``0.0``, ``isclose(x, 0)`` is computed
+ as ``abs(x) <= rel_tol * abs(x)``, which is ``False`` for any ``x`` and
+ rel_tol less than ``1.0``. So add an appropriate positive abs_tol argument
+ to the call.
The IEEE 754 special values of ``NaN``, ``inf``, and ``-inf`` will be
handled according to IEEE rules. Specifically, ``NaN`` is not considered
From 70f777daeaa899764002ebdad5c5821b660314f9 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 4 Nov 2024 09:37:23 +0100
Subject: [PATCH 155/269] [3.12] Docs: turn getopt examples into doctests
(GH-126377) (#126386)
(cherry picked from commit 0d80777981f95bbc79b146fc78b2189c82521ab9)
Co-authored-by: Erlend E. Aasland
---
Doc/library/getopt.rst | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/Doc/library/getopt.rst b/Doc/library/getopt.rst
index ef07ce9dd2c866..13f1d9645964fd 100644
--- a/Doc/library/getopt.rst
+++ b/Doc/library/getopt.rst
@@ -92,6 +92,8 @@ exception:
An example using only Unix style options:
+.. doctest::
+
>>> import getopt
>>> args = '-a -b -cfoo -d bar a1 a2'.split()
>>> args
@@ -104,6 +106,8 @@ An example using only Unix style options:
Using long option names is equally easy:
+.. doctest::
+
>>> s = '--condition=foo --testing --output-file abc.def -x a1 a2'
>>> args = s.split()
>>> args
@@ -115,7 +119,9 @@ Using long option names is equally easy:
>>> args
['a1', 'a2']
-In a script, typical usage is something like this::
+In a script, typical usage is something like this:
+
+.. testcode::
import getopt, sys
@@ -145,7 +151,9 @@ In a script, typical usage is something like this::
main()
Note that an equivalent command line interface could be produced with less code
-and more informative help and error messages by using the :mod:`argparse` module::
+and more informative help and error messages by using the :mod:`argparse` module:
+
+.. testcode::
import argparse
From a2356194918bc18f30a4a55c2e730dc97d09d4fe Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 4 Nov 2024 11:58:30 +0100
Subject: [PATCH 156/269] [3.12] gh-125783: Add more tests to prevent
regressions with the combination of ctypes and metaclasses. (GH-126126)
(GH-126276)
gh-125783: Add more tests to prevent regressions with the combination of ctypes and metaclasses. (GH-126126)
(cherry picked from commit 6c67446a6e73ab0e9a26e4360412cbd2f5550e66)
Co-authored-by: Jun Komoda <45822440+junkmd@users.noreply.github.com>
---
.../test_ctypes/test_c_simple_type_meta.py | 65 +++++++++++++++++++
1 file changed, 65 insertions(+)
diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py
index fa5144a3ca01bb..eb77d6d7782478 100644
--- a/Lib/test/test_ctypes/test_c_simple_type_meta.py
+++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py
@@ -85,3 +85,68 @@ class Sub(CtBase):
self.assertIsInstance(POINTER(Sub), p_meta)
self.assertTrue(issubclass(POINTER(Sub), Sub))
+
+ def test_creating_pointer_in_dunder_init_1(self):
+ class ct_meta(type):
+ def __init__(self, name, bases, namespace):
+ super().__init__(name, bases, namespace)
+
+ # Avoid recursion.
+ # (See test_creating_pointer_in_dunder_new_1)
+ if bases == (c_void_p,):
+ return
+ if issubclass(self, PtrBase):
+ return
+ if bases == (object,):
+ ptr_bases = (self, PtrBase)
+ else:
+ ptr_bases = (self, POINTER(bases[0]))
+ p = p_meta(f"POINTER({self.__name__})", ptr_bases, {})
+ ctypes._pointer_type_cache[self] = p
+
+ class p_meta(PyCSimpleType, ct_meta):
+ pass
+
+ class PtrBase(c_void_p, metaclass=p_meta):
+ pass
+
+ class CtBase(object, metaclass=ct_meta):
+ pass
+
+ class Sub(CtBase):
+ pass
+
+ class Sub2(Sub):
+ pass
+
+ self.assertIsInstance(POINTER(Sub2), p_meta)
+ self.assertTrue(issubclass(POINTER(Sub2), Sub2))
+ self.assertTrue(issubclass(POINTER(Sub2), POINTER(Sub)))
+ self.assertTrue(issubclass(POINTER(Sub), POINTER(CtBase)))
+
+ def test_creating_pointer_in_dunder_init_2(self):
+ class ct_meta(type):
+ def __init__(self, name, bases, namespace):
+ super().__init__(name, bases, namespace)
+
+ # Avoid recursion.
+ # (See test_creating_pointer_in_dunder_new_2)
+ if isinstance(self, p_meta):
+ return
+ p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {})
+ ctypes._pointer_type_cache[self] = p
+
+ class p_meta(PyCSimpleType, ct_meta):
+ pass
+
+ class Core(object):
+ pass
+
+ class CtBase(Core, metaclass=ct_meta):
+ pass
+
+ class Sub(CtBase):
+ pass
+
+ self.assertIsInstance(POINTER(Sub), p_meta)
+ self.assertTrue(issubclass(POINTER(Sub), Sub))
From 94423b6be1edbee886eae11ea2accea39bcf55d3 Mon Sep 17 00:00:00 2001
From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
Date: Mon, 4 Nov 2024 19:47:40 +0200
Subject: [PATCH 157/269] [3.12] gh-101865: Docs: Keep co_lnotab deprecation
for at least 3.14 (GH-126392) (#126404)
(cherry picked from commit eac41c5ddfadf52fbd84ee898ad56aedd5d90a41)
---
Doc/deprecations/pending-removal-in-3.14.rst | 7 -------
Doc/deprecations/pending-removal-in-3.15.rst | 11 +++++++++++
Doc/reference/datamodel.rst | 2 +-
Doc/whatsnew/3.12.rst | 4 ++--
4 files changed, 14 insertions(+), 10 deletions(-)
diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst
index 7eaa80e96138af..0e6a100574c3be 100644
--- a/Doc/deprecations/pending-removal-in-3.14.rst
+++ b/Doc/deprecations/pending-removal-in-3.14.rst
@@ -106,13 +106,6 @@ Pending Removal in Python 3.14
if :ref:`named placeholders ` are used and
*parameters* is a sequence instead of a :class:`dict`.
-* :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was
- deprecated in :pep:`626`
- since 3.10 and was planned to be removed in 3.12,
- but it only got a proper :exc:`DeprecationWarning` in 3.12.
- May be removed in 3.14.
- (Contributed by Nikita Sobolev in :gh:`101866`.)
-
* :mod:`typing`: :class:`~typing.ByteString`, deprecated since Python 3.9,
now causes a :exc:`DeprecationWarning` to be emitted when it is used.
diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst
index f9d1d4564b984c..182a72b2e1a5b2 100644
--- a/Doc/deprecations/pending-removal-in-3.15.rst
+++ b/Doc/deprecations/pending-removal-in-3.15.rst
@@ -37,6 +37,17 @@ Pending Removal in Python 3.15
(``NT = NamedTuple("NT", x=int)``) is deprecated, and will be disallowed in
3.15. Use the class-based syntax or the functional syntax instead.
+* :mod:`types`:
+
+ * :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was
+ deprecated in :pep:`626`
+ since 3.10 and was planned to be removed in 3.12,
+ but it only got a proper :exc:`DeprecationWarning` in 3.12.
+ May be removed in 3.15.
+ (Contributed by Nikita Sobolev in :gh:`101866`.)
+
+* :mod:`typing`:
+
* When using the functional syntax to create a :class:`!NamedTuple` class, failing to
pass a value to the *fields* parameter (``NT = NamedTuple("NT")``) is
deprecated. Passing ``None`` to the *fields* parameter
diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index bc287e1e7cdcf9..bdb83790653661 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -1428,7 +1428,7 @@ Special read-only attributes
.. deprecated:: 3.12
This attribute of code objects is deprecated, and may be removed in
- Python 3.14.
+ Python 3.15.
* - .. attribute:: codeobject.co_stacksize
- The required stack size of the code object
diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst
index c99c4559a79f91..e9b5499155dd1b 100644
--- a/Doc/whatsnew/3.12.rst
+++ b/Doc/whatsnew/3.12.rst
@@ -1338,8 +1338,8 @@ Deprecated
* Accessing :attr:`~codeobject.co_lnotab` on code objects was deprecated in
Python 3.10 via :pep:`626`,
- but it only got a proper :exc:`DeprecationWarning` in 3.12,
- therefore it will be removed in 3.14.
+ but it only got a proper :exc:`DeprecationWarning` in 3.12.
+ May be removed in 3.15.
(Contributed by Nikita Sobolev in :gh:`101866`.)
.. include:: ../deprecations/pending-removal-in-3.13.rst
From 844d908adb0cd5cfe07e229c614b98f7c2e7a810 Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka
Date: Tue, 5 Nov 2024 08:52:51 +0200
Subject: [PATCH 158/269] [3.12] gh-126303: Fix pickling and copying of
os.sched_param objects (GH-126336) (GH-126424)
(cherry picked from commit d3840503b0f590ee574fbdf3c96626ff8b3c45f6)
---
Include/internal/pycore_typeobject.h | 2 ++
Lib/test/test_posix.py | 15 +++++++++++++++
...24-11-02-19-20-44.gh-issue-126303.yVvyWB.rst | 1 +
Modules/posixmodule.c | 17 +++++++++++++++++
Objects/typeobject.c | 6 ++++++
5 files changed, 41 insertions(+)
create mode 100644 Misc/NEWS.d/next/Library/2024-11-02-19-20-44.gh-issue-126303.yVvyWB.rst
diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h
index 63f76fc55c9b27..4fdbc91a1ba4c6 100644
--- a/Include/internal/pycore_typeobject.h
+++ b/Include/internal/pycore_typeobject.h
@@ -143,6 +143,8 @@ PyAPI_DATA(PyTypeObject) _PyBufferWrapper_Type;
PyObject *
_PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found);
+extern int _PyType_AddMethod(PyTypeObject *, PyMethodDef *);
+
#ifdef __cplusplus
}
#endif
diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py
index aa811326e4ccf0..e225b8919d1395 100644
--- a/Lib/test/test_posix.py
+++ b/Lib/test/test_posix.py
@@ -6,12 +6,14 @@
from test.support import warnings_helper
from test.support.script_helper import assert_python_ok
+import copy
import errno
import sys
import signal
import time
import os
import platform
+import pickle
import stat
import tempfile
import unittest
@@ -1308,6 +1310,19 @@ def test_get_and_set_scheduler_and_param(self):
param = posix.sched_param(sched_priority=-large)
self.assertRaises(OverflowError, posix.sched_setparam, 0, param)
+ @requires_sched
+ def test_sched_param(self):
+ param = posix.sched_param(1)
+ for proto in range(pickle.HIGHEST_PROTOCOL+1):
+ newparam = pickle.loads(pickle.dumps(param, proto))
+ self.assertEqual(newparam, param)
+ newparam = copy.copy(param)
+ self.assertIsNot(newparam, param)
+ self.assertEqual(newparam, param)
+ newparam = copy.deepcopy(param)
+ self.assertIsNot(newparam, param)
+ self.assertEqual(newparam, param)
+
@unittest.skipUnless(hasattr(posix, "sched_rr_get_interval"), "no function")
def test_sched_rr_get_interval(self):
try:
diff --git a/Misc/NEWS.d/next/Library/2024-11-02-19-20-44.gh-issue-126303.yVvyWB.rst b/Misc/NEWS.d/next/Library/2024-11-02-19-20-44.gh-issue-126303.yVvyWB.rst
new file mode 100644
index 00000000000000..0072c97338c251
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-11-02-19-20-44.gh-issue-126303.yVvyWB.rst
@@ -0,0 +1 @@
+Fix pickling and copying of :class:`os.sched_param` objects.
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index b8558cc2265a58..d5298519b58846 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -24,6 +24,7 @@
#include "pycore_object.h" // _PyObject_LookupSpecial()
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_signal.h" // Py_NSIG
+#include "pycore_typeobject.h" // _PyType_AddMethod()
#ifdef MS_WINDOWS
# include
@@ -7866,6 +7867,16 @@ os_sched_param_impl(PyTypeObject *type, PyObject *sched_priority)
return res;
}
+static PyObject *
+os_sched_param_reduce(PyObject *self, PyObject *Py_UNUSED(ignored))
+{
+ return Py_BuildValue("(O(N))", Py_TYPE(self), PyStructSequence_GetItem(self, 0));
+}
+
+static PyMethodDef os_sched_param_reduce_method = {
+ "__reduce__", (PyCFunction)os_sched_param_reduce, METH_NOARGS|METH_COEXIST, NULL,
+};
+
PyDoc_VAR(os_sched_param__doc__);
static PyStructSequence_Field sched_param_fields[] = {
@@ -17001,6 +17012,12 @@ posixmodule_exec(PyObject *m)
return -1;
}
((PyTypeObject *)state->SchedParamType)->tp_new = os_sched_param;
+ if (_PyType_AddMethod((PyTypeObject *)state->SchedParamType,
+ &os_sched_param_reduce_method) < 0)
+ {
+ return -1;
+ }
+ PyType_Modified((PyTypeObject *)state->SchedParamType);
#endif
/* initialize TerminalSize_info */
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index a262ce07335825..7c678907ed5602 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -6645,6 +6645,12 @@ type_add_method(PyTypeObject *type, PyMethodDef *meth)
return 0;
}
+int
+_PyType_AddMethod(PyTypeObject *type, PyMethodDef *meth)
+{
+ return type_add_method(type, meth);
+}
+
/* Add the methods from tp_methods to the __dict__ in a type object */
static int
From aa5498d466957215704d85b99e7cc01103d783da Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 5 Nov 2024 11:01:37 +0100
Subject: [PATCH 159/269] [3.12] Doc: C API: Delete claim that `PyObject_Init`
is GC-aware (GH-126418) (#126432)
Doc: C API: Delete claim that `PyObject_Init` is GC-aware (GH-126418)
(cherry picked from commit 407c0366d9ccd2a36c6cc8bf92324856b16fd604)
Co-authored-by: Richard Hansen
---
Doc/c-api/allocation.rst | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst
index b3609c233156b6..e6ff40ab46e7c2 100644
--- a/Doc/c-api/allocation.rst
+++ b/Doc/c-api/allocation.rst
@@ -15,10 +15,8 @@ Allocating Objects on the Heap
.. c:function:: PyObject* PyObject_Init(PyObject *op, PyTypeObject *type)
Initialize a newly allocated object *op* with its type and initial
- reference. Returns the initialized object. If *type* indicates that the
- object participates in the cyclic garbage detector, it is added to the
- detector's set of observed objects. Other fields of the object are not
- affected.
+ reference. Returns the initialized object. Other fields of the object are
+ not affected.
.. c:function:: PyVarObject* PyObject_InitVar(PyVarObject *op, PyTypeObject *type, Py_ssize_t size)
From 47d48b62ddfc96d69bd618401132460cbcde6681 Mon Sep 17 00:00:00 2001
From: Alex Waygood
Date: Tue, 5 Nov 2024 11:19:45 +0000
Subject: [PATCH 160/269] [3.12] gh-126417: Register multiprocessing proxy
types to an appropriate collections.abc class (#126419) (#126436)
Co-authored-by: Stephen Morton
---
Lib/multiprocessing/managers.py | 4 ++++
Lib/test/_test_multiprocessing.py | 9 +++++++++
Misc/ACKS | 1 +
.../2024-11-04-16-40-02.gh-issue-126417.OWPqn0.rst | 3 +++
4 files changed, 17 insertions(+)
create mode 100644 Misc/NEWS.d/next/Library/2024-11-04-16-40-02.gh-issue-126417.OWPqn0.rst
diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py
index 75d9c18c201a86..b915e67c265b3d 100644
--- a/Lib/multiprocessing/managers.py
+++ b/Lib/multiprocessing/managers.py
@@ -18,6 +18,7 @@
import threading
import signal
import array
+import collections.abc
import queue
import time
import types
@@ -1160,6 +1161,8 @@ def __imul__(self, value):
return self
+collections.abc.MutableSequence.register(BaseListProxy)
+
DictProxy = MakeProxyType('DictProxy', (
'__contains__', '__delitem__', '__getitem__', '__iter__', '__len__',
'__setitem__', 'clear', 'copy', 'get', 'items',
@@ -1169,6 +1172,7 @@ def __imul__(self, value):
'__iter__': 'Iterator',
}
+collections.abc.MutableMapping.register(DictProxy)
ArrayProxy = MakeProxyType('ArrayProxy', (
'__len__', '__getitem__', '__setitem__'
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 607bfc02b12303..2213af52ca09ac 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -16,6 +16,7 @@
import functools
import signal
import array
+import collections.abc
import socket
import random
import logging
@@ -2331,6 +2332,10 @@ def test_list(self):
a.append('hello')
self.assertEqual(f[0][:], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 'hello'])
+ def test_list_isinstance(self):
+ a = self.list()
+ self.assertIsInstance(a, collections.abc.MutableSequence)
+
def test_list_iter(self):
a = self.list(list(range(10)))
it = iter(a)
@@ -2371,6 +2376,10 @@ def test_dict(self):
self.assertEqual(sorted(d.values()), [chr(i) for i in indices])
self.assertEqual(sorted(d.items()), [(i, chr(i)) for i in indices])
+ def test_dict_isinstance(self):
+ a = self.dict()
+ self.assertIsInstance(a, collections.abc.MutableMapping)
+
def test_dict_iter(self):
d = self.dict()
indices = list(range(65, 70))
diff --git a/Misc/ACKS b/Misc/ACKS
index 837ffbda18aea1..b5cf6acc55a88c 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1259,6 +1259,7 @@ Emily Morehouse
Derek Morr
James A Morrison
Martin Morrison
+Stephen Morton
Derek McTavish Mounce
Alessandro Moura
Pablo Mouzo
diff --git a/Misc/NEWS.d/next/Library/2024-11-04-16-40-02.gh-issue-126417.OWPqn0.rst b/Misc/NEWS.d/next/Library/2024-11-04-16-40-02.gh-issue-126417.OWPqn0.rst
new file mode 100644
index 00000000000000..c4a366343382f3
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-11-04-16-40-02.gh-issue-126417.OWPqn0.rst
@@ -0,0 +1,3 @@
+Register the :class:`!multiprocessing.managers.DictProxy` and :class:`!multiprocessing.managers.ListProxy` types in
+:mod:`multiprocessing.managers` to :class:`collections.abc.MutableMapping` and
+:class:`collections.abc.MutableSequence`, respectively.
From 786886a926fa025538f70a5a84259ab121e40782 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 5 Nov 2024 13:42:33 +0100
Subject: [PATCH 161/269] [3.12] gh-126425: Refactor `_lsprof_Profiler_enable`
(GH-126426) (#126443)
gh-126425: Refactor `_lsprof_Profiler_enable` (GH-126426)
- Explicit memory management for `None` objects (since we still try to treat immortal objects as regular objects)
- Respect possible errors of `sys.monitoring.register_callback` call
(cherry picked from commit 75872605aa78dbdfc5c4f025b0f90a7f37ba10c3)
Co-authored-by: sobolevn
---
Modules/_lsprof.c | 37 +++++++++++++++++++++++++------------
1 file changed, 25 insertions(+), 12 deletions(-)
diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c
index d11689906e574d..0464e7e8a44c0e 100644
--- a/Modules/_lsprof.c
+++ b/Modules/_lsprof.c
@@ -750,34 +750,47 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds)
return NULL;
}
- if (PyObject_CallMethod(monitoring, "use_tool_id", "is", self->tool_id, "cProfile") == NULL) {
+ PyObject *check = PyObject_CallMethod(monitoring,
+ "use_tool_id", "is",
+ self->tool_id, "cProfile");
+ if (check == NULL) {
PyErr_Format(PyExc_ValueError, "Another profiling tool is already active");
- Py_DECREF(monitoring);
- return NULL;
+ goto error;
}
+ Py_DECREF(check);
for (int i = 0; callback_table[i].callback_method; i++) {
+ int event = (1 << callback_table[i].event);
PyObject* callback = PyObject_GetAttrString((PyObject*)self, callback_table[i].callback_method);
if (!callback) {
- Py_DECREF(monitoring);
- return NULL;
+ goto error;
}
- Py_XDECREF(PyObject_CallMethod(monitoring, "register_callback", "iiO", self->tool_id,
- (1 << callback_table[i].event),
- callback));
+ PyObject *register_result = PyObject_CallMethod(monitoring, "register_callback",
+ "iiO", self->tool_id,
+ event, callback);
Py_DECREF(callback);
- all_events |= (1 << callback_table[i].event);
+ if (register_result == NULL) {
+ goto error;
+ }
+ Py_DECREF(register_result);
+ all_events |= event;
}
- if (!PyObject_CallMethod(monitoring, "set_events", "ii", self->tool_id, all_events)) {
- Py_DECREF(monitoring);
- return NULL;
+ PyObject *event_result = PyObject_CallMethod(monitoring, "set_events", "ii",
+ self->tool_id, all_events);
+ if (event_result == NULL) {
+ goto error;
}
+ Py_DECREF(event_result);
Py_DECREF(monitoring);
self->flags |= POF_ENABLED;
Py_RETURN_NONE;
+
+error:
+ Py_DECREF(monitoring);
+ return NULL;
}
static void
From b00887d176544377c607689c2132a47712afa6b3 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland"
Date: Tue, 5 Nov 2024 23:19:13 +0100
Subject: [PATCH 162/269] [3.12] gh-89640: harden float word ordering (#125571
and #126387) (#126430)
Properly detect float word ordering on Linux (gh-125571)
autoconf-archive patch by Dan Amelang.
(cherry picked from commit 26d627779f79d8d5650fe7be348432eccc28f8f9)
Hardcode WASM float word ordering to little endian (gh-126387)
(cherry picked from commit 532fc08102d62c04d55f5b8aac00bd9e7e12ff4b)
---
.github/workflows/build.yml | 2 +-
...4-10-16-09-37-51.gh-issue-89640.UDsW-j.rst | 2 +
...4-11-04-09-42-04.gh-issue-89640.QBv05o.rst | 1 +
Tools/build/regen-configure.sh | 2 +-
aclocal.m4 | 61 +++++++++++++++--
configure | 66 ++++++++++---------
configure.ac | 42 ++++++------
pyconfig.h.in | 4 --
8 files changed, 114 insertions(+), 66 deletions(-)
create mode 100644 Misc/NEWS.d/next/Build/2024-10-16-09-37-51.gh-issue-89640.UDsW-j.rst
create mode 100644 Misc/NEWS.d/next/Build/2024-11-04-09-42-04.gh-issue-89640.QBv05o.rst
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index a6daf0a436a459..325752ecde4967 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -97,7 +97,7 @@ jobs:
# reproducible: to get the same tools versions (autoconf, aclocal, ...)
runs-on: ubuntu-24.04
container:
- image: ghcr.io/python/autoconf:2024.10.11.11293396815
+ image: ghcr.io/python/autoconf:2024.10.16.11360930377
timeout-minutes: 60
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
diff --git a/Misc/NEWS.d/next/Build/2024-10-16-09-37-51.gh-issue-89640.UDsW-j.rst b/Misc/NEWS.d/next/Build/2024-10-16-09-37-51.gh-issue-89640.UDsW-j.rst
new file mode 100644
index 00000000000000..5aba2c789b6842
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2024-10-16-09-37-51.gh-issue-89640.UDsW-j.rst
@@ -0,0 +1,2 @@
+Improve detection of float word ordering on Linux when link-time optimizations
+are enabled.
diff --git a/Misc/NEWS.d/next/Build/2024-11-04-09-42-04.gh-issue-89640.QBv05o.rst b/Misc/NEWS.d/next/Build/2024-11-04-09-42-04.gh-issue-89640.QBv05o.rst
new file mode 100644
index 00000000000000..4fa44a1d6493b4
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2024-11-04-09-42-04.gh-issue-89640.QBv05o.rst
@@ -0,0 +1 @@
+Hard-code float word ordering as little endian on WASM.
diff --git a/Tools/build/regen-configure.sh b/Tools/build/regen-configure.sh
index efc80c8527885c..c3df291745a09e 100755
--- a/Tools/build/regen-configure.sh
+++ b/Tools/build/regen-configure.sh
@@ -5,7 +5,7 @@ set -e -x
# The check_generated_files job of .github/workflows/build.yml must kept in
# sync with this script. Use the same container image than the job so the job
# doesn't need to run autoreconf in a container.
-IMAGE="ghcr.io/python/autoconf:2024.10.06.11200919239"
+IMAGE="ghcr.io/python/autoconf:2024.10.16.11360930377"
AUTORECONF="autoreconf -ivf -Werror"
WORK_DIR="/src"
diff --git a/aclocal.m4 b/aclocal.m4
index 09ae5d1aa8a608..97514d838914a5 100644
--- a/aclocal.m4
+++ b/aclocal.m4
@@ -41,32 +41,81 @@ m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun
# If neither value is found, the user is instructed to specify the
# ordering.
#
+# Early versions of this macro (i.e., before serial 12) would not work
+# when interprocedural optimization (via link-time optimization) was
+# enabled. This would happen when, say, the GCC/clang "-flto" flag, or the
+# ICC "-ipo" flag was used, for example. The problem was that under
+# these conditions, the compiler did not allocate for and write the special
+# float value in the data segment of the object file, since doing so might
+# not prove optimal once more context was available. Thus, the special value
+# (in platform-dependent binary form) could not be found in the object file,
+# and the macro would fail.
+#
+# The solution to the above problem was to:
+#
+# 1) Compile and link a whole test program rather than just compile an
+# object file. This ensures that we reach the point where even an
+# interprocedural optimizing compiler writes values to the data segment.
+#
+# 2) Add code that requires the compiler to write the special value to
+# the data segment, as opposed to "optimizing away" the variable's
+# allocation. This could be done via compiler keywords or options, but
+# it's tricky to make this work for all versions of all compilers with
+# all optimization settings. The chosen solution was to make the exit
+# code of the test program depend on the storing of the special value
+# in memory (in the data segment). Because the exit code can be
+# verified, any compiler that aspires to be correct will produce a
+# program binary that contains the value, which the macro can then find.
+#
+# How does the exit code depend on the special value residing in memory?
+# Memory, unlike variables and registers, can be addressed indirectly at run
+# time. The exit code of this test program is a result of indirectly reading
+# and writing to the memory region where the special value is supposed to
+# reside. The actual memory addresses used and the values to be written are
+# derived from the the program input ("argv") and are therefore not known at
+# compile or link time. The compiler has no choice but to defer the
+# computation to run time, and to prepare by allocating and populating the
+# data segment with the special value. For further details, refer to the
+# source code of the test program.
+#
+# Note that the test program is never meant to be run. It only exists to host
+# a double float value in a given platform's binary format. Thus, error
+# handling is not included.
+#
# LICENSE
#
-# Copyright (c) 2008 Daniel Amelang
+# Copyright (c) 2008, 2023 Daniel Amelang
#
# Copying and distribution of this file, with or without modification, are
# permitted in any medium without royalty provided the copyright notice
# and this notice are preserved. This file is offered as-is, without any
# warranty.
-#serial 11
+#serial 12
AC_DEFUN([AX_C_FLOAT_WORDS_BIGENDIAN],
[AC_CACHE_CHECK(whether float word ordering is bigendian,
ax_cv_c_float_words_bigendian, [
ax_cv_c_float_words_bigendian=unknown
-AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
+AC_LINK_IFELSE([AC_LANG_SOURCE([[
+
+#include
+
+static double m[] = {9.090423496703681e+223, 0.0};
-double d = 90904234967036810337470478905505011476211692735615632014797120844053488865816695273723469097858056257517020191247487429516932130503560650002327564517570778480236724525140520121371739201496540132640109977779420565776568942592.0;
+int main (int argc, char *argv[])
+{
+ m[atoi (argv[1])] += atof (argv[2]);
+ return m[atoi (argv[3])] > 0.0;
+}
]])], [
-if grep noonsees conftest.$ac_objext >/dev/null ; then
+if grep noonsees conftest$EXEEXT >/dev/null ; then
ax_cv_c_float_words_bigendian=yes
fi
-if grep seesnoon conftest.$ac_objext >/dev/null ; then
+if grep seesnoon conftest$EXEEXT >/dev/null ; then
if test "$ax_cv_c_float_words_bigendian" = unknown; then
ax_cv_c_float_words_bigendian=no
else
diff --git a/configure b/configure
index 8cca82b6bc87b4..241cf8f3d4a57c 100755
--- a/configure
+++ b/configure
@@ -23525,18 +23525,26 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
-double d = 90904234967036810337470478905505011476211692735615632014797120844053488865816695273723469097858056257517020191247487429516932130503560650002327564517570778480236724525140520121371739201496540132640109977779420565776568942592.0;
+#include
+
+static double m[] = {9.090423496703681e+223, 0.0};
+
+int main (int argc, char *argv[])
+{
+ m[atoi (argv[1])] += atof (argv[2]);
+ return m[atoi (argv[3])] > 0.0;
+}
_ACEOF
-if ac_fn_c_try_compile "$LINENO"
+if ac_fn_c_try_link "$LINENO"
then :
-if grep noonsees conftest.$ac_objext >/dev/null ; then
+if grep noonsees conftest$EXEEXT >/dev/null ; then
ax_cv_c_float_words_bigendian=yes
fi
-if grep seesnoon conftest.$ac_objext >/dev/null ; then
+if grep seesnoon conftest$EXEEXT >/dev/null ; then
if test "$ax_cv_c_float_words_bigendian" = unknown; then
ax_cv_c_float_words_bigendian=no
else
@@ -23546,7 +23554,8 @@ fi
fi
-rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+ conftest$ac_exeext conftest.$ac_ext
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_c_float_words_bigendian" >&5
printf "%s\n" "$ax_cv_c_float_words_bigendian" >&6; }
@@ -23554,41 +23563,34 @@ printf "%s\n" "$ax_cv_c_float_words_bigendian" >&6; }
case $ax_cv_c_float_words_bigendian in
yes)
-printf "%s\n" "#define FLOAT_WORDS_BIGENDIAN 1" >>confdefs.h
+printf "%s\n" "#define DOUBLE_IS_BIG_ENDIAN_IEEE754 1" >>confdefs.h
;;
no)
- ;;
- *)
- as_fn_error $? "
-
-Unknown float word ordering. You need to manually preset
-ax_cv_c_float_words_bigendian=no (or yes) according to your system.
-
- " "$LINENO" 5 ;;
-esac
-
-if test "$ax_cv_c_float_words_bigendian" = "yes"
-then
-
-printf "%s\n" "#define DOUBLE_IS_BIG_ENDIAN_IEEE754 1" >>confdefs.h
+printf "%s\n" "#define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 1" >>confdefs.h
+ ;;
+ *)
+ case $host_cpu in #(
+ *arm*) :
+ # Some ARM platforms use a mixed-endian representation for
+ # doubles. While Python doesn't currently have full support
+ # for these platforms (see e.g., issue 1762561), we can at
+ # least make sure that float <-> string conversions work.
+ # FLOAT_WORDS_BIGENDIAN doesn't actually detect this case,
+ # but if it's not big or little, then it must be this?
-elif test "$ax_cv_c_float_words_bigendian" = "no"
-then
+printf "%s\n" "#define DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 1" >>confdefs.h
+ ;; #(
+ wasm*) :
printf "%s\n" "#define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 1" >>confdefs.h
+ ;; #(
+ *) :
+ ;;
+esac ;;
+esac
-else
- # Some ARM platforms use a mixed-endian representation for doubles.
- # While Python doesn't currently have full support for these platforms
- # (see e.g., issue 1762561), we can at least make sure that float <-> string
- # conversions work.
- # FLOAT_WORDS_BIGENDIAN doesn't actually detect this case, but if it's not big
- # or little, then it must be this?
-
-printf "%s\n" "#define DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 1" >>confdefs.h
-fi
# The short float repr introduced in Python 3.1 requires the
# correctly-rounded string <-> double conversion functions from
diff --git a/configure.ac b/configure.ac
index d3f0fb4d88f389..9270b5f7172d54 100644
--- a/configure.ac
+++ b/configure.ac
@@ -5679,28 +5679,26 @@ AS_VAR_IF([ac_cv_gcc_asm_for_x64], [yes], [
# * Check for various properties of floating point *
# **************************************************
-AX_C_FLOAT_WORDS_BIGENDIAN
-if test "$ax_cv_c_float_words_bigendian" = "yes"
-then
- AC_DEFINE([DOUBLE_IS_BIG_ENDIAN_IEEE754], [1],
- [Define if C doubles are 64-bit IEEE 754 binary format, stored
- with the most significant byte first])
-elif test "$ax_cv_c_float_words_bigendian" = "no"
-then
- AC_DEFINE([DOUBLE_IS_LITTLE_ENDIAN_IEEE754], [1],
- [Define if C doubles are 64-bit IEEE 754 binary format, stored
- with the least significant byte first])
-else
- # Some ARM platforms use a mixed-endian representation for doubles.
- # While Python doesn't currently have full support for these platforms
- # (see e.g., issue 1762561), we can at least make sure that float <-> string
- # conversions work.
- # FLOAT_WORDS_BIGENDIAN doesn't actually detect this case, but if it's not big
- # or little, then it must be this?
- AC_DEFINE([DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754], [1],
- [Define if C doubles are 64-bit IEEE 754 binary format, stored
- in ARM mixed-endian order (byte order 45670123)])
-fi
+AX_C_FLOAT_WORDS_BIGENDIAN(
+ [AC_DEFINE([DOUBLE_IS_BIG_ENDIAN_IEEE754], [1],
+ [Define if C doubles are 64-bit IEEE 754 binary format,
+ stored with the most significant byte first])],
+ [AC_DEFINE([DOUBLE_IS_LITTLE_ENDIAN_IEEE754], [1],
+ [Define if C doubles are 64-bit IEEE 754 binary format,
+ stored with the least significant byte first])],
+ [AS_CASE([$host_cpu],
+ [*arm*], [# Some ARM platforms use a mixed-endian representation for
+ # doubles. While Python doesn't currently have full support
+ # for these platforms (see e.g., issue 1762561), we can at
+ # least make sure that float <-> string conversions work.
+ # FLOAT_WORDS_BIGENDIAN doesn't actually detect this case,
+ # but if it's not big or little, then it must be this?
+ AC_DEFINE([DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754], [1],
+ [Define if C doubles are 64-bit IEEE 754 binary format,
+ stored in ARM mixed-endian order (byte order 45670123)])],
+ [wasm*], [AC_DEFINE([DOUBLE_IS_LITTLE_ENDIAN_IEEE754], [1],
+ [Define if C doubles are 64-bit IEEE 754 binary format,
+ stored with the least significant byte first])])])
# The short float repr introduced in Python 3.1 requires the
# correctly-rounded string <-> double conversion functions from
diff --git a/pyconfig.h.in b/pyconfig.h.in
index 6d370f6664c10c..7d3537e5dd8385 100644
--- a/pyconfig.h.in
+++ b/pyconfig.h.in
@@ -47,10 +47,6 @@
/* Define if --enable-ipv6 is specified */
#undef ENABLE_IPV6
-/* Define to 1 if your system stores words within floats with the most
- significant word first */
-#undef FLOAT_WORDS_BIGENDIAN
-
/* Define if getpgrp() must be called as getpgrp(0). */
#undef GETPGRP_HAVE_ARG
From fc01844019f63748e8334adca57a11577cc00a6d Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 6 Nov 2024 01:12:08 +0100
Subject: [PATCH 163/269] [3.12] gh-70764: inspect.getclosurevars now
identifies global variables with LOAD_GLOBAL (GH-120143) (#126460)
gh-70764: inspect.getclosurevars now identifies global variables with LOAD_GLOBAL (GH-120143)
(cherry picked from commit 83ba8c2bba834c0b92de669cac16fcda17485e0e)
Co-authored-by: blhsing
---
Lib/inspect.py | 14 +++++++++-----
Lib/test/test_inspect/test_inspect.py | 13 +++++++++++++
.../2024-06-06-04-06-05.gh-issue-70764.6511hw.rst | 1 +
3 files changed, 23 insertions(+), 5 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-06-06-04-06-05.gh-issue-70764.6511hw.rst
diff --git a/Lib/inspect.py b/Lib/inspect.py
index c43faa73159a1c..b630cb2835957b 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -1638,11 +1638,15 @@ def getclosurevars(func):
global_vars = {}
builtin_vars = {}
unbound_names = set()
- for name in code.co_names:
- if name in ("None", "True", "False"):
- # Because these used to be builtins instead of keywords, they
- # may still show up as name references. We ignore them.
- continue
+ global_names = set()
+ for instruction in dis.get_instructions(code):
+ opname = instruction.opname
+ name = instruction.argval
+ if opname == "LOAD_ATTR":
+ unbound_names.add(name)
+ elif opname == "LOAD_GLOBAL":
+ global_names.add(name)
+ for name in global_names:
try:
global_vars[name] = global_ns[name]
except KeyError:
diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py
index 36978e8217cf9b..7d3153db1070ec 100644
--- a/Lib/test/test_inspect/test_inspect.py
+++ b/Lib/test/test_inspect/test_inspect.py
@@ -1780,6 +1780,19 @@ def g(local_ref):
builtin_vars, unbound_names)
self.assertEqual(inspect.getclosurevars(C().f(_arg)), expected)
+ def test_attribute_same_name_as_global_var(self):
+ class C:
+ _global_ref = object()
+ def f():
+ print(C._global_ref, _global_ref)
+ nonlocal_vars = {"C": C}
+ global_vars = {"_global_ref": _global_ref}
+ builtin_vars = {"print": print}
+ unbound_names = {"_global_ref"}
+ expected = inspect.ClosureVars(nonlocal_vars, global_vars,
+ builtin_vars, unbound_names)
+ self.assertEqual(inspect.getclosurevars(f), expected)
+
def test_nonlocal_vars(self):
# More complex tests of nonlocal resolution
def _nonlocal_vars(f):
diff --git a/Misc/NEWS.d/next/Library/2024-06-06-04-06-05.gh-issue-70764.6511hw.rst b/Misc/NEWS.d/next/Library/2024-06-06-04-06-05.gh-issue-70764.6511hw.rst
new file mode 100644
index 00000000000000..4cfb66a6ccc6ee
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-06-06-04-06-05.gh-issue-70764.6511hw.rst
@@ -0,0 +1 @@
+Fixed an issue where :func:`inspect.getclosurevars` would incorrectly classify an attribute name as a global variable when the name exists both as an attribute name and a global variable.
From 2f39370f4aee651b9dfd882f5d739eeea52f05ce Mon Sep 17 00:00:00 2001
From: Valery Fedorenko
Date: Wed, 6 Nov 2024 10:56:12 +0300
Subject: [PATCH 164/269] [3.12] Fix possible null pointer dereference of
freevars in _PyCompile_LookupArg (gh-126238) (#126474)
[3.12] gh-126238: Fix possible null pointer dereference of freevars in _PyCompile_LookupArg (GH-126239)
* Replace Py_DECREF by Py_XDECREF
(cherry picked from commit 8525c9375f25e6ec0c0b5dfcab464703f6e78082)
Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>
Co-authored-by: Peter Bierma
---
Python/compile.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Python/compile.c b/Python/compile.c
index 6050b7281fa051..56fdbfae6f613e 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -1835,7 +1835,7 @@ compiler_make_closure(struct compiler *c, location loc,
c->u->u_metadata.u_name,
co->co_name,
freevars);
- Py_DECREF(freevars);
+ Py_XDECREF(freevars);
return ERROR;
}
ADDOP_I(c, loc, LOAD_CLOSURE, arg);
From 7a4262b064a2339c8d37e0267be91a0084f79c3c Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 6 Nov 2024 11:18:02 +0100
Subject: [PATCH 165/269] [3.12] gh-122544: Change OS image in Azure pipeline
to Ubuntu 24.04 (GH-125344) (#126480)
Co-authored-by: Damien <81557462+Damien-Chen@users.noreply.github.com>
---
.azure-pipelines/ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index d3e842d9f31d01..7490dd947e1504 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -5,7 +5,7 @@ jobs:
displayName: Pre-build checks
pool:
- vmImage: ubuntu-22.04
+ vmImage: ubuntu-24.04
steps:
- template: ./prebuild-checks.yml
From e6fd40ba1883040224128cbb30fa002827f18392 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:21:21 +0100
Subject: [PATCH 166/269] [3.12] gh-126455: Disallow _ssl.SSLSocket
instantiation (GH-126481) (#126487)
gh-126455: Disallow _ssl.SSLSocket instantiation (GH-126481)
Prevent creation of incomplete/invalid _ssl.SSLSocket objects when
created directly.
(cherry picked from commit b1c4ffc20573befb4db66bbbdd569b9bd13bb127)
Co-authored-by: Victor Stinner
---
Modules/_ssl.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Modules/_ssl.c b/Modules/_ssl.c
index 35e4ce7af50fc6..7a9f2c87239eca 100644
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -2900,7 +2900,7 @@ static PyType_Spec PySSLSocket_spec = {
.name = "_ssl._SSLSocket",
.basicsize = sizeof(PySSLSocket),
.flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE |
- Py_TPFLAGS_HAVE_GC),
+ Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_DISALLOW_INSTANTIATION),
.slots = PySSLSocket_slots,
};
From 97ab3cf3749085f6f42f38180c40aa88ca1b6355 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 6 Nov 2024 14:49:25 +0100
Subject: [PATCH 167/269] [3.12] gh-126461: Fix _Unpickler_ReadFromFile() error
handling (GH-126485) (#126496)
gh-126461: Fix _Unpickler_ReadFromFile() error handling (GH-126485)
Handle _Unpickler_SetStringInput() failure.
(cherry picked from commit a1c57bcfd2bcbc55ff858407e09c1d8d8cee44e6)
Co-authored-by: Victor Stinner
---
Modules/_pickle.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
index 879c18263d505e..179500d6956a78 100644
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -1344,6 +1344,10 @@ _Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n)
else {
read_size = _Unpickler_SetStringInput(self, data);
Py_DECREF(data);
+ if (read_size < 0) {
+ return -1;
+ }
+
self->prefetched_idx = 0;
if (n <= read_size)
return n;
From a19832bfbffcca0c224434a877f671137bc87137 Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka
Date: Wed, 6 Nov 2024 23:11:37 +0200
Subject: [PATCH 168/269] [3.12] gh-126489: Do not call persistent_id() for a
persistent id in Python pickle (GH-126490) (GH-126516)
(cherry picked from commit 8fa4dc4ba8646c59f945f2451c53e2919f066065)
---
Lib/pickle.py | 9 +++++----
.../2024-11-06-13-41-38.gh-issue-126489.toaf-0.rst | 3 +++
2 files changed, 8 insertions(+), 4 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-11-06-13-41-38.gh-issue-126489.toaf-0.rst
diff --git a/Lib/pickle.py b/Lib/pickle.py
index 01c1a102794d57..ea5f1c5dc36c91 100644
--- a/Lib/pickle.py
+++ b/Lib/pickle.py
@@ -533,10 +533,11 @@ def save(self, obj, save_persistent_id=True):
self.framer.commit_frame()
# Check for persistent id (defined by a subclass)
- pid = self.persistent_id(obj)
- if pid is not None and save_persistent_id:
- self.save_pers(pid)
- return
+ if save_persistent_id:
+ pid = self.persistent_id(obj)
+ if pid is not None:
+ self.save_pers(pid)
+ return
# Check the memo
x = self.memo.get(id(obj))
diff --git a/Misc/NEWS.d/next/Library/2024-11-06-13-41-38.gh-issue-126489.toaf-0.rst b/Misc/NEWS.d/next/Library/2024-11-06-13-41-38.gh-issue-126489.toaf-0.rst
new file mode 100644
index 00000000000000..8a6573cdea7b42
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-11-06-13-41-38.gh-issue-126489.toaf-0.rst
@@ -0,0 +1,3 @@
+The Python implementation of :mod:`pickle` no longer calls
+:meth:`pickle.Pickler.persistent_id` for the result of
+:meth:`!persistent_id`.
From d71da0feda679bea2d291ed59f121a4cdd38b118 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 6 Nov 2024 23:37:37 +0100
Subject: [PATCH 169/269] [3.12] gh-126451: Register contextvars.Context to
collections.abc.Mapping (GH-126452) (#126519)
gh-126451: Register contextvars.Context to collections.abc.Mapping (GH-126452)
(cherry picked from commit 5dc36dc5658f6ba9cfd9d7a2771baaf17d2ee23a)
Co-authored-by: Stephen Morton
Co-authored-by: sobolevn
Co-authored-by: Alex Waygood
Co-authored-by: Peter Bierma
---
Lib/contextvars.py | 4 ++++
Lib/test/test_context.py | 14 ++++++++++++++
.../2024-11-05-11-28-45.gh-issue-126451.XJMtqz.rst | 2 ++
3 files changed, 20 insertions(+)
create mode 100644 Misc/NEWS.d/next/Library/2024-11-05-11-28-45.gh-issue-126451.XJMtqz.rst
diff --git a/Lib/contextvars.py b/Lib/contextvars.py
index d78c80dfe6f99c..14514f185e069d 100644
--- a/Lib/contextvars.py
+++ b/Lib/contextvars.py
@@ -1,4 +1,8 @@
+import _collections_abc
from _contextvars import Context, ContextVar, Token, copy_context
__all__ = ('Context', 'ContextVar', 'Token', 'copy_context')
+
+
+_collections_abc.Mapping.register(Context)
diff --git a/Lib/test/test_context.py b/Lib/test/test_context.py
index dc6856509a40a0..d9e1c6214e7057 100644
--- a/Lib/test/test_context.py
+++ b/Lib/test/test_context.py
@@ -1,3 +1,4 @@
+import collections.abc
import concurrent.futures
import contextvars
import functools
@@ -342,6 +343,19 @@ def ctx2_fun():
ctx1.run(ctx1_fun)
+ def test_context_isinstance(self):
+ ctx = contextvars.Context()
+ self.assertIsInstance(ctx, collections.abc.Mapping)
+ self.assertTrue(issubclass(contextvars.Context, collections.abc.Mapping))
+
+ mapping_methods = (
+ '__contains__', '__eq__', '__getitem__', '__iter__', '__len__',
+ '__ne__', 'get', 'items', 'keys', 'values',
+ )
+ for name in mapping_methods:
+ with self.subTest(name=name):
+ self.assertTrue(callable(getattr(ctx, name)))
+
@isolated_context
@threading_helper.requires_working_threading()
def test_context_threads_1(self):
diff --git a/Misc/NEWS.d/next/Library/2024-11-05-11-28-45.gh-issue-126451.XJMtqz.rst b/Misc/NEWS.d/next/Library/2024-11-05-11-28-45.gh-issue-126451.XJMtqz.rst
new file mode 100644
index 00000000000000..563cb2515eca60
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-11-05-11-28-45.gh-issue-126451.XJMtqz.rst
@@ -0,0 +1,2 @@
+Register the :class:`contextvars.Context` type to
+:class:`collections.abc.Mapping`.
From 4f10b8eacfe563f69ce348674d8c81f8c88b4025 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 7 Nov 2024 10:50:46 +0100
Subject: [PATCH 170/269] [3.12] gh-125679: multiprocessing Lock and RLock -
fix invalid representation string on MacOSX. (GH-125680) (#126534)
gh-125679: multiprocessing Lock and RLock - fix invalid representation string on MacOSX. (GH-125680)
(cherry picked from commit 75f7cf91ec5afc6091a0fd442a1f0435c19300b2)
Co-authored-by: Duprat
---
Lib/multiprocessing/synchronize.py | 4 +-
Lib/test/_test_multiprocessing.py | 122 ++++++++++++++++++
...-11-06-23-40-28.gh-issue-125679.Qq9xF5.rst | 2 +
3 files changed, 126 insertions(+), 2 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-11-06-23-40-28.gh-issue-125679.Qq9xF5.rst
diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py
index 3ccbfe311c71f3..0f682b9a0944b8 100644
--- a/Lib/multiprocessing/synchronize.py
+++ b/Lib/multiprocessing/synchronize.py
@@ -174,7 +174,7 @@ def __repr__(self):
name = process.current_process().name
if threading.current_thread().name != 'MainThread':
name += '|' + threading.current_thread().name
- elif self._semlock._get_value() == 1:
+ elif not self._semlock._is_zero():
name = 'None'
elif self._semlock._count() > 0:
name = 'SomeOtherThread'
@@ -200,7 +200,7 @@ def __repr__(self):
if threading.current_thread().name != 'MainThread':
name += '|' + threading.current_thread().name
count = self._semlock._count()
- elif self._semlock._get_value() == 1:
+ elif not self._semlock._is_zero():
name, count = 'None', 0
elif self._semlock._count() > 0:
name, count = 'SomeOtherThread', 'nonzero'
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 2213af52ca09ac..f5dcfe644a3190 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -1363,6 +1363,66 @@ def test_closed_queue_put_get_exceptions(self):
class _TestLock(BaseTestCase):
+ @staticmethod
+ def _acquire(lock, l=None):
+ lock.acquire()
+ if l is not None:
+ l.append(repr(lock))
+
+ @staticmethod
+ def _acquire_event(lock, event):
+ lock.acquire()
+ event.set()
+ time.sleep(1.0)
+
+ def test_repr_lock(self):
+ if self.TYPE != 'processes':
+ self.skipTest('test not appropriate for {}'.format(self.TYPE))
+
+ lock = self.Lock()
+ self.assertEqual(f'', repr(lock))
+
+ lock.acquire()
+ self.assertEqual(f'', repr(lock))
+ lock.release()
+
+ tname = 'T1'
+ l = []
+ t = threading.Thread(target=self._acquire,
+ args=(lock, l),
+ name=tname)
+ t.start()
+ time.sleep(0.1)
+ self.assertEqual(f'', l[0])
+ lock.release()
+
+ t = threading.Thread(target=self._acquire,
+ args=(lock,),
+ name=tname)
+ t.start()
+ time.sleep(0.1)
+ self.assertEqual('', repr(lock))
+ lock.release()
+
+ pname = 'P1'
+ l = multiprocessing.Manager().list()
+ p = self.Process(target=self._acquire,
+ args=(lock, l),
+ name=pname)
+ p.start()
+ p.join()
+ self.assertEqual(f'', l[0])
+
+ lock = self.Lock()
+ event = self.Event()
+ p = self.Process(target=self._acquire_event,
+ args=(lock, event),
+ name='P2')
+ p.start()
+ event.wait()
+ self.assertEqual(f'', repr(lock))
+ p.terminate()
+
def test_lock(self):
lock = self.Lock()
self.assertEqual(lock.acquire(), True)
@@ -1370,6 +1430,68 @@ def test_lock(self):
self.assertEqual(lock.release(), None)
self.assertRaises((ValueError, threading.ThreadError), lock.release)
+ @staticmethod
+ def _acquire_release(lock, timeout, l=None, n=1):
+ for _ in range(n):
+ lock.acquire()
+ if l is not None:
+ l.append(repr(lock))
+ time.sleep(timeout)
+ for _ in range(n):
+ lock.release()
+
+ def test_repr_rlock(self):
+ if self.TYPE != 'processes':
+ self.skipTest('test not appropriate for {}'.format(self.TYPE))
+
+ lock = self.RLock()
+ self.assertEqual('', repr(lock))
+
+ n = 3
+ for _ in range(n):
+ lock.acquire()
+ self.assertEqual(f'', repr(lock))
+ for _ in range(n):
+ lock.release()
+
+ t, l = [], []
+ for i in range(n):
+ t.append(threading.Thread(target=self._acquire_release,
+ args=(lock, 0.1, l, i+1),
+ name=f'T{i+1}'))
+ t[-1].start()
+ for t_ in t:
+ t_.join()
+ for i in range(n):
+ self.assertIn(f'', l)
+
+
+ t = threading.Thread(target=self._acquire_release,
+ args=(lock, 0.2),
+ name=f'T1')
+ t.start()
+ time.sleep(0.1)
+ self.assertEqual('', repr(lock))
+ time.sleep(0.2)
+
+ pname = 'P1'
+ l = multiprocessing.Manager().list()
+ p = self.Process(target=self._acquire_release,
+ args=(lock, 0.1, l),
+ name=pname)
+ p.start()
+ p.join()
+ self.assertEqual(f'', l[0])
+
+ event = self.Event()
+ lock = self.RLock()
+ p = self.Process(target=self._acquire_event,
+ args=(lock, event))
+ p.start()
+ event.wait()
+ self.assertEqual('', repr(lock))
+ p.join()
+
def test_rlock(self):
lock = self.RLock()
self.assertEqual(lock.acquire(), True)
diff --git a/Misc/NEWS.d/next/Library/2024-11-06-23-40-28.gh-issue-125679.Qq9xF5.rst b/Misc/NEWS.d/next/Library/2024-11-06-23-40-28.gh-issue-125679.Qq9xF5.rst
new file mode 100644
index 00000000000000..ac6851e2689692
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-11-06-23-40-28.gh-issue-125679.Qq9xF5.rst
@@ -0,0 +1,2 @@
+The :class:`multiprocessing.Lock` and :class:`multiprocessing.RLock`
+``repr`` values no longer say "unknown" on macOS.
From a0866f4c81ecc057d4521e8e7a02f4e1fff175a1 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Thu, 7 Nov 2024 16:38:57 +0100
Subject: [PATCH 171/269] [3.12] Doc: C API: Demote sections to subsections for
consistency (GH-126535) (#126545)
Doc: C API: Demote sections to subsections for consistency (GH-126535)
The entire file should be a single section; the headings below the
first heading should be subsections.
(cherry picked from commit e3510bd3dd9ea8f2a30cb1128470aee3a48d8880)
Co-authored-by: Richard Hansen
---
Doc/c-api/typeobj.rst | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
index 610cb96e8885b6..9fc3d358371b17 100644
--- a/Doc/c-api/typeobj.rst
+++ b/Doc/c-api/typeobj.rst
@@ -2193,7 +2193,7 @@ This is done by filling a :c:type:`PyType_Spec` structure and calling
.. _number-structs:
Number Object Structures
-========================
+------------------------
.. sectionauthor:: Amaury Forgeot d'Arc
@@ -2307,7 +2307,7 @@ Number Object Structures
.. _mapping-structs:
Mapping Object Structures
-=========================
+-------------------------
.. sectionauthor:: Amaury Forgeot d'Arc
@@ -2344,7 +2344,7 @@ Mapping Object Structures
.. _sequence-structs:
Sequence Object Structures
-==========================
+--------------------------
.. sectionauthor:: Amaury Forgeot d'Arc
@@ -2424,7 +2424,7 @@ Sequence Object Structures
.. _buffer-structs:
Buffer Object Structures
-========================
+------------------------
.. sectionauthor:: Greg J. Stein
.. sectionauthor:: Benjamin Peterson
@@ -2519,7 +2519,7 @@ Buffer Object Structures
Async Object Structures
-=======================
+-----------------------
.. sectionauthor:: Yury Selivanov
@@ -2587,7 +2587,7 @@ Async Object Structures
.. _slot-typedefs:
Slot Type typedefs
-==================
+------------------
.. c:type:: PyObject *(*allocfunc)(PyTypeObject *cls, Py_ssize_t nitems)
@@ -2696,7 +2696,7 @@ Slot Type typedefs
.. _typedef-examples:
Examples
-========
+--------
The following are simple examples of Python type definitions. They
include common usage you may encounter. Some demonstrate tricky corner
From fc1e9e606b0f293a35f8e52d05b5c63ed4081ef3 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Fri, 8 Nov 2024 11:56:08 +0100
Subject: [PATCH 172/269] [3.12] gh-126171: fix possible null dereference in
_imp_find_frozen_impl (GH-126566) (#126568)
gh-126171: fix possible null dereference in _imp_find_frozen_impl (GH-126566)
(cherry picked from commit 9ecd8f7f40e6724a1c1d46c2665147aaabceb2d2)
Co-authored-by: Valery Fedorenko
---
Python/import.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Python/import.c b/Python/import.c
index db70909982fa3e..4d0b7fd95569b3 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -3539,7 +3539,7 @@ _imp_find_frozen_impl(PyObject *module, PyObject *name, int withdata)
if (info.origname != NULL && info.origname[0] != '\0') {
origname = PyUnicode_FromString(info.origname);
if (origname == NULL) {
- Py_DECREF(data);
+ Py_XDECREF(data);
return NULL;
}
}
From 69849ad28825c0ccc1ebd97f3eeddebd64a0e6ee Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Fri, 8 Nov 2024 16:47:46 +0100
Subject: [PATCH 173/269] gh-125298: Remove misleading text in os.kill
documentation (GH-125749)
Windows has not accepted process handles in many releases.
(cherry picked from commit 75ffac296ef24758b7e5bd9316f32a8170ade37f)
Co-authored-by: RUANG (James Roy)
---
Doc/library/os.rst | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/Doc/library/os.rst b/Doc/library/os.rst
index 423480e927c900..18e58249a81b08 100644
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -4245,8 +4245,7 @@ written in Python, such as a mail server's external command delivery program.
only be sent to console processes which share a common console window,
e.g., some subprocesses. Any other value for *sig* will cause the process
to be unconditionally killed by the TerminateProcess API, and the exit code
- will be set to *sig*. The Windows version of :func:`kill` additionally takes
- process handles to be killed.
+ will be set to *sig*.
See also :func:`signal.pthread_kill`.
From 5fb443d6254d53f43acb0b39a7615d56f4fc7355 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Fri, 8 Nov 2024 18:21:42 +0100
Subject: [PATCH 174/269] [3.12] GH-126212: Fix removal of slashes in file URIs
on Windows (GH-126214) (#126591)
GH-126212: Fix removal of slashes in file URIs on Windows (GH-126214)
Adjust `urllib.request.pathname2url()` and `url2pathname()` so that they
don't remove slashes from Windows DOS drive paths and URLs. There was no
basis for this behaviour, and it conflicts with how UNC and POSIX paths are
handled.
(cherry picked from commit 54c63a32d06cb5f07a66245c375eac7d7efb964a)
Co-authored-by: Barney Gale
---
Lib/nturl2path.py | 25 +++++--------------
Lib/test/test_urllib.py | 11 ++++++--
...-10-30-23-59-36.gh-issue-126212._9uYjT.rst | 3 +++
3 files changed, 18 insertions(+), 21 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-10-30-23-59-36.gh-issue-126212._9uYjT.rst
diff --git a/Lib/nturl2path.py b/Lib/nturl2path.py
index 6453f202c26d14..2f9fec7893afd1 100644
--- a/Lib/nturl2path.py
+++ b/Lib/nturl2path.py
@@ -24,23 +24,15 @@ def url2pathname(url):
# convert this to \\host\path\on\remote\host
# (notice halving of slashes at the start of the path)
url = url[2:]
- components = url.split('/')
# make sure not to convert quoted slashes :-)
- return urllib.parse.unquote('\\'.join(components))
+ return urllib.parse.unquote(url.replace('/', '\\'))
comp = url.split('|')
if len(comp) != 2 or comp[0][-1] not in string.ascii_letters:
error = 'Bad URL: ' + url
raise OSError(error)
drive = comp[0][-1].upper()
- components = comp[1].split('/')
- path = drive + ':'
- for comp in components:
- if comp:
- path = path + '\\' + urllib.parse.unquote(comp)
- # Issue #11474 - handing url such as |c/|
- if path.endswith(':') and url.endswith('/'):
- path += '\\'
- return path
+ tail = urllib.parse.unquote(comp[1].replace('/', '\\'))
+ return drive + ':' + tail
def pathname2url(p):
"""OS-specific conversion from a file system path to a relative URL
@@ -60,17 +52,12 @@ def pathname2url(p):
raise OSError('Bad path: ' + p)
if not ':' in p:
# No drive specifier, just convert slashes and quote the name
- components = p.split('\\')
- return urllib.parse.quote('/'.join(components))
+ return urllib.parse.quote(p.replace('\\', '/'))
comp = p.split(':', maxsplit=2)
if len(comp) != 2 or len(comp[0]) > 1:
error = 'Bad path: ' + p
raise OSError(error)
drive = urllib.parse.quote(comp[0].upper())
- components = comp[1].split('\\')
- path = '///' + drive + ':'
- for comp in components:
- if comp:
- path = path + '/' + urllib.parse.quote(comp)
- return path
+ tail = urllib.parse.quote(comp[1].replace('\\', '/'))
+ return '///' + drive + ':' + tail
diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py
index 0e1b13d31c9273..2b59fb617cbdda 100644
--- a/Lib/test/test_urllib.py
+++ b/Lib/test/test_urllib.py
@@ -1535,8 +1535,10 @@ def test_pathname2url_win(self):
self.assertEqual(fn('\\\\?\\C:\\dir'), '///C:/dir')
self.assertEqual(fn('\\\\?\\unc\\server\\share\\dir'), '//server/share/dir')
self.assertEqual(fn("C:"), '///C:')
- self.assertEqual(fn("C:\\"), '///C:')
+ self.assertEqual(fn("C:\\"), '///C:/')
self.assertEqual(fn('C:\\a\\b.c'), '///C:/a/b.c')
+ self.assertEqual(fn('C:\\a\\b.c\\'), '///C:/a/b.c/')
+ self.assertEqual(fn('C:\\a\\\\b.c'), '///C:/a//b.c')
self.assertEqual(fn('C:\\a\\b%#c'), '///C:/a/b%25%23c')
self.assertEqual(fn('C:\\a\\b\xe9'), '///C:/a/b%C3%A9')
self.assertEqual(fn('C:\\foo\\bar\\spam.foo'), "///C:/foo/bar/spam.foo")
@@ -1572,13 +1574,15 @@ def test_url2pathname_win(self):
self.assertEqual(fn("///C|"), 'C:')
self.assertEqual(fn("///C:"), 'C:')
self.assertEqual(fn('///C:/'), 'C:\\')
- self.assertEqual(fn('/C|//'), 'C:\\')
+ self.assertEqual(fn('/C|//'), 'C:\\\\')
self.assertEqual(fn('///C|/path'), 'C:\\path')
# No DOS drive
self.assertEqual(fn("///C/test/"), '\\\\\\C\\test\\')
self.assertEqual(fn("////C/test/"), '\\\\C\\test\\')
# DOS drive paths
self.assertEqual(fn('C:/path/to/file'), 'C:\\path\\to\\file')
+ self.assertEqual(fn('C:/path/to/file/'), 'C:\\path\\to\\file\\')
+ self.assertEqual(fn('C:/path/to//file'), 'C:\\path\\to\\\\file')
self.assertEqual(fn('C|/path/to/file'), 'C:\\path\\to\\file')
self.assertEqual(fn('/C|/path/to/file'), 'C:\\path\\to\\file')
self.assertEqual(fn('///C|/path/to/file'), 'C:\\path\\to\\file')
@@ -1592,6 +1596,9 @@ def test_url2pathname_win(self):
# Localhost paths
self.assertEqual(fn('//localhost/C:/path/to/file'), 'C:\\path\\to\\file')
self.assertEqual(fn('//localhost/C|/path/to/file'), 'C:\\path\\to\\file')
+ # Percent-encoded forward slashes are preserved for backwards compatibility
+ self.assertEqual(fn('C:/foo%2fbar'), 'C:\\foo/bar')
+ self.assertEqual(fn('//server/share/foo%2fbar'), '\\\\server\\share\\foo/bar')
# Round-tripping
paths = ['C:',
r'\\\C\test\\',
diff --git a/Misc/NEWS.d/next/Library/2024-10-30-23-59-36.gh-issue-126212._9uYjT.rst b/Misc/NEWS.d/next/Library/2024-10-30-23-59-36.gh-issue-126212._9uYjT.rst
new file mode 100644
index 00000000000000..047fe0f68048b5
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-10-30-23-59-36.gh-issue-126212._9uYjT.rst
@@ -0,0 +1,3 @@
+Fix issue where :func:`urllib.request.pathname2url` and
+:func:`~urllib.request.url2pathname` removed slashes from Windows DOS drive
+paths and URLs.
From 7bb92ed505920009ca097948553adc846596b246 Mon Sep 17 00:00:00 2001
From: "Gregory P. Smith"
Date: Sat, 9 Nov 2024 16:13:26 -0800
Subject: [PATCH 175/269] [3.12] gh-117378: Fix multiprocessing forkserver
preload sys.path inheritance. (GH-126538) (GH-126633)
gh-117378: Fix multiprocessing forkserver preload sys.path inheritance.
`sys.path` was not properly being sent from the parent process when launching
the multiprocessing forkserver process to preload imports. This bug has been
there since the forkserver start method was introduced in Python 3.4. It was
always _supposed_ to inherit `sys.path` the same way the spawn method does.
Observable behavior change: A `''` value in `sys.path` will now be replaced in
the forkserver's `sys.path` with an absolute pathname
`os.path.abspath(os.getcwd())` saved at the time that `multiprocessing` was
imported in the parent process as it already was when using the spawn start
method. **This will only be observable during forkserver preload imports**.
The code invoked before calling things in another process already correctly sets `sys.path`.
Which is likely why this went unnoticed for so long as a mere performance issue in
some configurations.
A workaround for the bug on impacted Pythons is to set PYTHONPATH in the
environment before multiprocessing's forkserver process was started. Not perfect
as that is then inherited by other children, etc, but likely good enough for many
people's purposes.
(cherry picked from commit 9d08423b6e0fa89ce9cfea08e580ed72e5db8c70)
Co-authored-by: Serhiy Storchaka
---
Lib/multiprocessing/forkserver.py | 2 +
Lib/test/_test_multiprocessing.py | 78 +++++++++++++++++++
...-11-07-01-40-11.gh-issue-117378.o9O5uM.rst | 17 ++++
3 files changed, 97 insertions(+)
create mode 100644 Misc/NEWS.d/next/Library/2024-11-07-01-40-11.gh-issue-117378.o9O5uM.rst
diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py
index 4642707dae2f4e..9fb563276e33b5 100644
--- a/Lib/multiprocessing/forkserver.py
+++ b/Lib/multiprocessing/forkserver.py
@@ -167,6 +167,8 @@ def ensure_running(self):
def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
'''Run forkserver.'''
if preload:
+ if sys_path is not None:
+ sys.path[:] = sys_path
if '__main__' in preload and main_path is not None:
process.current_process()._inheriting = True
try:
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index f5dcfe644a3190..63f8ef992c6d46 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -12,6 +12,7 @@
import sys
import os
import gc
+import importlib
import errno
import functools
import signal
@@ -20,8 +21,10 @@
import socket
import random
import logging
+import shutil
import subprocess
import struct
+import tempfile
import operator
import pickle
import weakref
@@ -6196,6 +6199,81 @@ def submain(): pass
self.assertFalse(err, msg=err.decode('utf-8'))
+class _TestSpawnedSysPath(BaseTestCase):
+ """Test that sys.path is setup in forkserver and spawn processes."""
+
+ ALLOWED_TYPES = ('processes',)
+
+ def setUp(self):
+ self._orig_sys_path = list(sys.path)
+ self._temp_dir = tempfile.mkdtemp(prefix="test_sys_path-")
+ self._mod_name = "unique_test_mod"
+ module_path = os.path.join(self._temp_dir, f"{self._mod_name}.py")
+ with open(module_path, "w", encoding="utf-8") as mod:
+ mod.write("# A simple test module\n")
+ sys.path[:] = [p for p in sys.path if p] # remove any existing ""s
+ sys.path.insert(0, self._temp_dir)
+ sys.path.insert(0, "") # Replaced with an abspath in child.
+ try:
+ self._ctx_forkserver = multiprocessing.get_context("forkserver")
+ except ValueError:
+ self._ctx_forkserver = None
+ self._ctx_spawn = multiprocessing.get_context("spawn")
+
+ def tearDown(self):
+ sys.path[:] = self._orig_sys_path
+ shutil.rmtree(self._temp_dir, ignore_errors=True)
+
+ @staticmethod
+ def enq_imported_module_names(queue):
+ queue.put(tuple(sys.modules))
+
+ def test_forkserver_preload_imports_sys_path(self):
+ ctx = self._ctx_forkserver
+ if not ctx:
+ self.skipTest("requires forkserver start method.")
+ self.assertNotIn(self._mod_name, sys.modules)
+ multiprocessing.forkserver._forkserver._stop() # Must be fresh.
+ ctx.set_forkserver_preload(
+ ["test.test_multiprocessing_forkserver", self._mod_name])
+ q = ctx.Queue()
+ proc = ctx.Process(target=self.enq_imported_module_names, args=(q,))
+ proc.start()
+ proc.join()
+ child_imported_modules = q.get()
+ q.close()
+ self.assertIn(self._mod_name, child_imported_modules)
+
+ @staticmethod
+ def enq_sys_path_and_import(queue, mod_name):
+ queue.put(sys.path)
+ try:
+ importlib.import_module(mod_name)
+ except ImportError as exc:
+ queue.put(exc)
+ else:
+ queue.put(None)
+
+ def test_child_sys_path(self):
+ for ctx in (self._ctx_spawn, self._ctx_forkserver):
+ if not ctx:
+ continue
+ with self.subTest(f"{ctx.get_start_method()} start method"):
+ q = ctx.Queue()
+ proc = ctx.Process(target=self.enq_sys_path_and_import,
+ args=(q, self._mod_name))
+ proc.start()
+ proc.join()
+ child_sys_path = q.get()
+ import_error = q.get()
+ q.close()
+ self.assertNotIn("", child_sys_path) # replaced by an abspath
+ self.assertIn(self._temp_dir, child_sys_path) # our addition
+ # ignore the first element, it is the absolute "" replacement
+ self.assertEqual(child_sys_path[1:], sys.path[1:])
+ self.assertIsNone(import_error, msg=f"child could not import {self._mod_name}")
+
+
class MiscTestCase(unittest.TestCase):
def test__all__(self):
# Just make sure names in not_exported are excluded
diff --git a/Misc/NEWS.d/next/Library/2024-11-07-01-40-11.gh-issue-117378.o9O5uM.rst b/Misc/NEWS.d/next/Library/2024-11-07-01-40-11.gh-issue-117378.o9O5uM.rst
new file mode 100644
index 00000000000000..cdbe21f9f9a663
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-11-07-01-40-11.gh-issue-117378.o9O5uM.rst
@@ -0,0 +1,17 @@
+Fixed the :mod:`multiprocessing` ``"forkserver"`` start method forkserver
+process to correctly inherit the parent's :data:`sys.path` during the importing
+of :func:`multiprocessing.set_forkserver_preload` modules in the same manner as
+:data:`sys.path` is configured in workers before executing work items.
+
+This bug caused some forkserver module preloading to silently fail to preload.
+This manifested as a performance degration in child processes when the
+``sys.path`` was required due to additional repeated work in every worker.
+
+It could also have a side effect of ``""`` remaining in :data:`sys.path` during
+forkserver preload imports instead of the absolute path from :func:`os.getcwd`
+at multiprocessing import time used in the worker ``sys.path``.
+
+Potentially leading to incorrect imports from the wrong location during
+preload. We are unaware of that actually happening. The issue was discovered
+by someone observing unexpected preload performance gains.
+
From 61e6f0992f50a32509e94b050decce70ada8885f Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sun, 10 Nov 2024 01:57:54 +0100
Subject: [PATCH 176/269] [3.12] Postpone `module.__loader__` deprecation to
Python 3.16 (GH-126482) (#126637)
Postpone `module.__loader__` deprecation to Python 3.16 (GH-126482)
(cherry picked from commit 450db61a78989c5a1f1106be01e071798c783cf9)
Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
---
Doc/deprecations/pending-removal-in-3.14.rst | 7 -------
Doc/deprecations/pending-removal-in-3.16.rst | 7 +++++++
Doc/reference/datamodel.rst | 4 ++--
3 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst
index 0e6a100574c3be..b506b0f02fbf53 100644
--- a/Doc/deprecations/pending-removal-in-3.14.rst
+++ b/Doc/deprecations/pending-removal-in-3.14.rst
@@ -1,13 +1,6 @@
Pending Removal in Python 3.14
------------------------------
-* The import system:
-
- * Setting :attr:`~module.__loader__` on a module while
- failing to set :attr:`__spec__.loader `
- is deprecated. In Python 3.14, :attr:`!__loader__` will cease to be set or
- taken into consideration by the import system or the standard library.
-
* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters
of :class:`!argparse.BooleanOptionalAction` are deprecated
and will be removed in 3.14.
diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst
index 82d40091576815..f59a40373e48a8 100644
--- a/Doc/deprecations/pending-removal-in-3.16.rst
+++ b/Doc/deprecations/pending-removal-in-3.16.rst
@@ -1,6 +1,13 @@
Pending Removal in Python 3.16
------------------------------
+* The import system:
+
+ * Setting :attr:`~module.__loader__` on a module while
+ failing to set :attr:`__spec__.loader `
+ is deprecated. In Python 3.16, :attr:`!__loader__` will cease to be set or
+ taken into consideration by the import system or the standard library.
+
* :mod:`array`:
:class:`array.array` ``'u'`` type (:c:type:`wchar_t`):
use the ``'w'`` type instead (``Py_UCS4``).
diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index bdb83790653661..e19a365fa4bd09 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -1007,9 +1007,9 @@ this approach.
using the :class:`types.ModuleType` constructor.
Previously the attribute was optional.
- .. deprecated-removed:: 3.12 3.14
+ .. deprecated-removed:: 3.12 3.16
Setting :attr:`!__loader__` on a module while failing to set
- :attr:`!__spec__.loader` is deprecated. In Python 3.14,
+ :attr:`!__spec__.loader` is deprecated. In Python 3.16,
:attr:`!__loader__` will cease to be set or taken into consideration by
the import system or the standard library.
From 5030e81dedd482fd313884991d8b3447dd153fcb Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sun, 10 Nov 2024 16:14:48 +0100
Subject: [PATCH 177/269] [3.12] gh-126565: Skip `zipfile.Path.exists` check in
write mode (GH-126576) (#126643)
gh-126565: Skip `zipfile.Path.exists` check in write mode (GH-126576)
When `zipfile.Path.open` is called, the implementation will check
whether the path already exists in the ZIP file. However, this check is
only required when the ZIP file is in read mode. By swapping arguments
of the `and` operator, the short-circuiting will prevent the check from
being run in write mode.
This change will improve the performance of `open()`, because checking
whether a file exists is slow in write mode, especially when the archive
has many members.
(cherry picked from commit 160758a574d12bf0d965d8206136e7da4f4fd6c3)
Co-authored-by: Jan Hicken
---
Lib/zipfile/_path/__init__.py | 2 +-
.../next/Library/2024-11-08-11-06-14.gh-issue-126565.dFFO22.rst | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-11-08-11-06-14.gh-issue-126565.dFFO22.rst
diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py
index 8db5ef18d7c5b0..645cfafdd623d9 100644
--- a/Lib/zipfile/_path/__init__.py
+++ b/Lib/zipfile/_path/__init__.py
@@ -303,7 +303,7 @@ def open(self, mode='r', *args, pwd=None, **kwargs):
if self.is_dir():
raise IsADirectoryError(self)
zip_mode = mode[0]
- if not self.exists() and zip_mode == 'r':
+ if zip_mode == 'r' and not self.exists():
raise FileNotFoundError(self)
stream = self.root.open(self.at, zip_mode, pwd=pwd)
if 'b' in mode:
diff --git a/Misc/NEWS.d/next/Library/2024-11-08-11-06-14.gh-issue-126565.dFFO22.rst b/Misc/NEWS.d/next/Library/2024-11-08-11-06-14.gh-issue-126565.dFFO22.rst
new file mode 100644
index 00000000000000..22858570bbe03c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-11-08-11-06-14.gh-issue-126565.dFFO22.rst
@@ -0,0 +1 @@
+Improve performances of :meth:`zipfile.Path.open` for non-reading modes.
From c47c6416c3614f5a58f5d4fd86714ed4b6bd60c4 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sun, 10 Nov 2024 16:34:52 +0100
Subject: [PATCH 178/269] [3.12] Skip test in test_socket.py if
`sys.getrefcount` isn't available (GH-126640) (#126646)
Skip test in test_socket.py if `sys.getrefcount` isn't available (GH-126640)
Skip `testMakefileCloseSocketDestroy` test if `sys.getrefcount` isn't available. This is necessary for PyPy and other Python implementations that do not have `sys.getrefcount`.
(cherry picked from commit 0f6bb28ff3ba152faf7523ea9aaf0094cc39bdda)
Co-authored-by: CF Bolz-Tereick
---
Lib/test/test_socket.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py
index acfa1cd503b86b..fd328a74134bcf 100644
--- a/Lib/test/test_socket.py
+++ b/Lib/test/test_socket.py
@@ -5261,6 +5261,8 @@ def _testMakefileClose(self):
self.write_file.write(self.write_msg)
self.write_file.flush()
+ @unittest.skipUnless(hasattr(sys, 'getrefcount'),
+ 'test needs sys.getrefcount()')
def testMakefileCloseSocketDestroy(self):
refcount_before = sys.getrefcount(self.cli_conn)
self.read_file.close()
From abb8265f0206ab209bad62581eb48d8a764038a1 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Sun, 10 Nov 2024 22:33:26 +0100
Subject: [PATCH 179/269] [3.12] gh-117378: Only run the new multiprocessing
SysPath test when appropriate (GH-126635) (GH-126653)
gh-117378: Only run the new multiprocessing SysPath test when appropriate (GH-126635)
The first version had it running two forkserver and one spawn tests underneath each of the _fork, _forkserver, and _spawn test suites that build off the generic one.
This adds to the existing complexity of the multiprocessing test suite by offering BaseTestCase classes another attribute to control which suites they are invoked under. Practicality vs purity here. :/
Net result: we don't over-run the new test and their internal logic is simplified.
(cherry picked from commit ca878b6e45f9c7934842f7bb94274e671b155e09)
Co-authored-by: Gregory P. Smith
---
Lib/test/_test_multiprocessing.py | 59 ++++++++++++++++---------------
1 file changed, 30 insertions(+), 29 deletions(-)
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index 63f8ef992c6d46..c190dc918dac64 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -258,6 +258,9 @@ def __call__(self, *args, **kwds):
class BaseTestCase(object):
ALLOWED_TYPES = ('processes', 'manager', 'threads')
+ # If not empty, limit which start method suites run this class.
+ START_METHODS: set[str] = set()
+ start_method = None # set by install_tests_in_module_dict()
def assertTimingAlmostEqual(self, a, b):
if CHECK_TIMINGS:
@@ -6202,7 +6205,9 @@ def submain(): pass
class _TestSpawnedSysPath(BaseTestCase):
"""Test that sys.path is setup in forkserver and spawn processes."""
- ALLOWED_TYPES = ('processes',)
+ ALLOWED_TYPES = {'processes'}
+ # Not applicable to fork which inherits everything from the process as is.
+ START_METHODS = {"forkserver", "spawn"}
def setUp(self):
self._orig_sys_path = list(sys.path)
@@ -6214,11 +6219,8 @@ def setUp(self):
sys.path[:] = [p for p in sys.path if p] # remove any existing ""s
sys.path.insert(0, self._temp_dir)
sys.path.insert(0, "") # Replaced with an abspath in child.
- try:
- self._ctx_forkserver = multiprocessing.get_context("forkserver")
- except ValueError:
- self._ctx_forkserver = None
- self._ctx_spawn = multiprocessing.get_context("spawn")
+ self.assertIn(self.start_method, self.START_METHODS)
+ self._ctx = multiprocessing.get_context(self.start_method)
def tearDown(self):
sys.path[:] = self._orig_sys_path
@@ -6229,15 +6231,15 @@ def enq_imported_module_names(queue):
queue.put(tuple(sys.modules))
def test_forkserver_preload_imports_sys_path(self):
- ctx = self._ctx_forkserver
- if not ctx:
- self.skipTest("requires forkserver start method.")
+ if self._ctx.get_start_method() != "forkserver":
+ self.skipTest("forkserver specific test.")
self.assertNotIn(self._mod_name, sys.modules)
multiprocessing.forkserver._forkserver._stop() # Must be fresh.
- ctx.set_forkserver_preload(
+ self._ctx.set_forkserver_preload(
["test.test_multiprocessing_forkserver", self._mod_name])
- q = ctx.Queue()
- proc = ctx.Process(target=self.enq_imported_module_names, args=(q,))
+ q = self._ctx.Queue()
+ proc = self._ctx.Process(
+ target=self.enq_imported_module_names, args=(q,))
proc.start()
proc.join()
child_imported_modules = q.get()
@@ -6255,23 +6257,19 @@ def enq_sys_path_and_import(queue, mod_name):
queue.put(None)
def test_child_sys_path(self):
- for ctx in (self._ctx_spawn, self._ctx_forkserver):
- if not ctx:
- continue
- with self.subTest(f"{ctx.get_start_method()} start method"):
- q = ctx.Queue()
- proc = ctx.Process(target=self.enq_sys_path_and_import,
- args=(q, self._mod_name))
- proc.start()
- proc.join()
- child_sys_path = q.get()
- import_error = q.get()
- q.close()
- self.assertNotIn("", child_sys_path) # replaced by an abspath
- self.assertIn(self._temp_dir, child_sys_path) # our addition
- # ignore the first element, it is the absolute "" replacement
- self.assertEqual(child_sys_path[1:], sys.path[1:])
- self.assertIsNone(import_error, msg=f"child could not import {self._mod_name}")
+ q = self._ctx.Queue()
+ proc = self._ctx.Process(
+ target=self.enq_sys_path_and_import, args=(q, self._mod_name))
+ proc.start()
+ proc.join()
+ child_sys_path = q.get()
+ import_error = q.get()
+ q.close()
+ self.assertNotIn("", child_sys_path) # replaced by an abspath
+ self.assertIn(self._temp_dir, child_sys_path) # our addition
+ # ignore the first element, it is the absolute "" replacement
+ self.assertEqual(child_sys_path[1:], sys.path[1:])
+ self.assertIsNone(import_error, msg=f"child could not import {self._mod_name}")
class MiscTestCase(unittest.TestCase):
@@ -6450,6 +6448,8 @@ def install_tests_in_module_dict(remote_globs, start_method,
if base is BaseTestCase:
continue
assert set(base.ALLOWED_TYPES) <= ALL_TYPES, base.ALLOWED_TYPES
+ if base.START_METHODS and start_method not in base.START_METHODS:
+ continue # class not intended for this start method.
for type_ in base.ALLOWED_TYPES:
if only_type and type_ != only_type:
continue
@@ -6463,6 +6463,7 @@ class Temp(base, Mixin, unittest.TestCase):
Temp = hashlib_helper.requires_hashdigest('sha256')(Temp)
Temp.__name__ = Temp.__qualname__ = newname
Temp.__module__ = __module__
+ Temp.start_method = start_method
remote_globs[newname] = Temp
elif issubclass(base, unittest.TestCase):
if only_type:
From 55c53056906cd962d977d209af7672bc6b2cb560 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 11 Nov 2024 03:52:20 +0100
Subject: [PATCH 180/269] [3.12] gh-126543: Docs: change "bound type var" to
"bounded" when used in the context of the 'bound' kw argument to TypeVar
(GH-126584) (#126658)
(cherry picked from commit 434b29767f2fdef9f35c8e93303cf6aca4a66a80)
Co-authored-by: Pedro Fonini
---
Doc/library/typing.rst | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst
index 8b464be14b059e..54a19ae0b6bebc 100644
--- a/Doc/library/typing.rst
+++ b/Doc/library/typing.rst
@@ -1609,11 +1609,11 @@ without the dedicated syntax, as documented below.
class Sequence[T]: # T is a TypeVar
...
- This syntax can also be used to create bound and constrained type
+ This syntax can also be used to create bounded and constrained type
variables::
- class StrSequence[S: str]: # S is a TypeVar bound to str
- ...
+ class StrSequence[S: str]: # S is a TypeVar with a `str` upper bound;
+ ... # we can say that S is "bounded by `str`"
class StrOrBytesSequence[A: (str, bytes)]: # A is a TypeVar constrained to str or bytes
@@ -1646,8 +1646,8 @@ without the dedicated syntax, as documented below.
"""Add two strings or bytes objects together."""
return x + y
- Note that type variables can be *bound*, *constrained*, or neither, but
- cannot be both bound *and* constrained.
+ Note that type variables can be *bounded*, *constrained*, or neither, but
+ cannot be both bounded *and* constrained.
The variance of type variables is inferred by type checkers when they are created
through the :ref:`type parameter syntax ` or when
@@ -1657,8 +1657,8 @@ without the dedicated syntax, as documented below.
By default, manually created type variables are invariant.
See :pep:`484` and :pep:`695` for more details.
- Bound type variables and constrained type variables have different
- semantics in several important ways. Using a *bound* type variable means
+ Bounded type variables and constrained type variables have different
+ semantics in several important ways. Using a *bounded* type variable means
that the ``TypeVar`` will be solved using the most specific type possible::
x = print_capitalized('a string')
@@ -1672,8 +1672,8 @@ without the dedicated syntax, as documented below.
z = print_capitalized(45) # error: int is not a subtype of str
- Type variables can be bound to concrete types, abstract types (ABCs or
- protocols), and even unions of types::
+ The upper bound of a type variable can be a concrete type, abstract type
+ (ABC or Protocol), or even a union of types::
# Can be anything with an __abs__ method
def print_abs[T: SupportsAbs](arg: T) -> None:
@@ -1717,7 +1717,7 @@ without the dedicated syntax, as documented below.
.. attribute:: __bound__
- The bound of the type variable, if any.
+ The upper bound of the type variable, if any.
.. versionchanged:: 3.12
@@ -1903,7 +1903,7 @@ without the dedicated syntax, as documented below.
return x + y
Without ``ParamSpec``, the simplest way to annotate this previously was to
- use a :class:`TypeVar` with bound ``Callable[..., Any]``. However this
+ use a :class:`TypeVar` with upper bound ``Callable[..., Any]``. However this
causes two problems:
1. The type checker can't type check the ``inner`` function because
From 7050aba5d7fbbdb8f42cd559072af9b49ea8a28b Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 11 Nov 2024 07:48:23 +0100
Subject: [PATCH 181/269] [3.12] gh-117378: Clear up the NEWS entry wording
(GH-126634) (#126669)
gh-117378: Clear up the NEWS entry wording (GH-126634)
gh-117378: Clear up the NEWS entry wording.
Docs are hard. Lets go shopping!
(cherry picked from commit 5c488caeb858690a696bc9f74fc74a274a3aa51c)
Co-authored-by: Gregory P. Smith
---
.../Library/2024-11-07-01-40-11.gh-issue-117378.o9O5uM.rst | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/Misc/NEWS.d/next/Library/2024-11-07-01-40-11.gh-issue-117378.o9O5uM.rst b/Misc/NEWS.d/next/Library/2024-11-07-01-40-11.gh-issue-117378.o9O5uM.rst
index cdbe21f9f9a663..d7d4477ec17814 100644
--- a/Misc/NEWS.d/next/Library/2024-11-07-01-40-11.gh-issue-117378.o9O5uM.rst
+++ b/Misc/NEWS.d/next/Library/2024-11-07-01-40-11.gh-issue-117378.o9O5uM.rst
@@ -11,7 +11,7 @@ It could also have a side effect of ``""`` remaining in :data:`sys.path` during
forkserver preload imports instead of the absolute path from :func:`os.getcwd`
at multiprocessing import time used in the worker ``sys.path``.
-Potentially leading to incorrect imports from the wrong location during
-preload. We are unaware of that actually happening. The issue was discovered
-by someone observing unexpected preload performance gains.
+The ``sys.path`` differences between phases in the child process could
+potentially have caused preload to import incorrect things from the wrong
+location. We are unaware of that actually having happened in practice.
From fb82e10ce4a5f5b08ed769078f8a3ed51456a06c Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 11 Nov 2024 07:57:15 +0100
Subject: [PATCH 182/269] [3.12] gh-126664: Use `else` instead of `finally` in
"The with statement" documentation. (GH-126665) (#126671)
gh-126664: Use `else` instead of `finally` in "The with statement" documentation. (GH-126665)
(cherry picked from commit 25257d61cfccc3b4189f96390a5c4db73fd5302c)
Co-authored-by: vivodi <103735539+vivodi@users.noreply.github.com>
---
Doc/reference/compound_stmts.rst | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst
index 81399901c9cf46..ddb1337f949e70 100644
--- a/Doc/reference/compound_stmts.rst
+++ b/Doc/reference/compound_stmts.rst
@@ -534,18 +534,15 @@ is semantically equivalent to::
enter = type(manager).__enter__
exit = type(manager).__exit__
value = enter(manager)
- hit_except = False
try:
TARGET = value
SUITE
except:
- hit_except = True
if not exit(manager, *sys.exc_info()):
raise
- finally:
- if not hit_except:
- exit(manager, None, None, None)
+ else:
+ exit(manager, None, None, None)
With more than one item, the context managers are processed as if multiple
:keyword:`with` statements were nested::
From 75d251bb04a7fe6386c2d7675bb9cd750add5fe5 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 11 Nov 2024 08:08:37 +0100
Subject: [PATCH 183/269] [3.12] Add missing fullstop `.` to whatsnew/3.8.rst
(GH-126553) (#126673)
Add missing fullstop `.` to whatsnew/3.8.rst (GH-126553)
(cherry picked from commit 82269c7d580e1aad71ff11fe891cf7f97eb45703)
Co-authored-by: Rafael Fontenelle
---
Doc/whatsnew/3.8.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 75e200905811af..4d0b9f2c886f2b 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -936,7 +936,7 @@ Add option ``--json-lines`` to parse every input line as a separate JSON object.
logging
-------
-Added a *force* keyword argument to :func:`logging.basicConfig`
+Added a *force* keyword argument to :func:`logging.basicConfig`.
When set to true, any existing handlers attached
to the root logger are removed and closed before carrying out the
configuration specified by the other arguments.
From d62f100fe1184c3d14d1ee51cfaaf3542c236ae7 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 11 Nov 2024 13:56:08 +0100
Subject: [PATCH 184/269] [3.12] gh-126499: test_ssl: Don't assume err.reason
is a string (GH-126501) (GH-126573)
gh-126499: test_ssl: Don't assume err.reason is a string (GH-126501)
The skipping machinery called `getattr(err, "reason", "")` on an arbitrary
exception. As intermittent Buildbot failures show, sometimes it's set
to None.
Convert it to string for this specific check.
(cherry picked from commit 78ad7e632248dc989378cabeb797b9f3d940d9f2)
Co-authored-by: Petr Viktorin
---
Lib/test/test_ssl.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 9b59ddd887aa0b..1775234266227a 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -4817,7 +4817,7 @@ def non_linux_skip_if_other_okay_error(self, err):
return # Expect the full test setup to always work on Linux.
if (isinstance(err, ConnectionResetError) or
(isinstance(err, OSError) and err.errno == errno.EINVAL) or
- re.search('wrong.version.number', getattr(err, "reason", ""), re.I)):
+ re.search('wrong.version.number', str(getattr(err, "reason", "")), re.I)):
# On Windows the TCP RST leads to a ConnectionResetError
# (ECONNRESET) which Linux doesn't appear to surface to userspace.
# If wrap_socket() winds up on the "if connected:" path and doing
From aee80cd5e7c6be90c69b9aa9c09faa19b91cdccd Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 11 Nov 2024 15:26:10 +0100
Subject: [PATCH 185/269] [3.12] gh-126500: test_ssl: Don't stop
ThreadedEchoServer on OSError in ConnectionHandler; rely on __exit__
(GH-126503) (GH-126572)
gh-126500: test_ssl: Don't stop ThreadedEchoServer on OSError in ConnectionHandler; rely on __exit__ (GH-126503)
If `read()` in the ConnectionHandler thread raises `OSError` (except `ConnectionError`),
the ConnectionHandler shuts down the entire ThreadedEchoServer,
preventing further connections.
It also does that for `EPROTOTYPE` in `wrap_conn`.
As far as I can see, this is done to avoid the server thread getting stuck,
forgotten, in its accept loop. However, since 2011 (5b95eb90a7167285b6544b50865227c584943c9a)
the server is used as a context manager, and its `__exit__` does `stop()` and `join()`.
(I'm not sure if we *always* used `with` since that commit, but currently we do.)
Make sure that the context manager *is* used, and remove the `server.stop()`
calls from ConnectionHandler.
(cherry picked from commit c9cda1608edf7664c10f4f467e24591062c2fe62)
Co-authored-by: Petr Viktorin
---
Lib/test/test_ssl.py | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 1775234266227a..aaddb7597dfc70 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -2300,7 +2300,6 @@ def wrap_conn(self):
# See also http://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/
if e.errno != errno.EPROTOTYPE and sys.platform != "darwin":
self.running = False
- self.server.stop()
self.close()
return False
else:
@@ -2435,10 +2434,6 @@ def run(self):
self.close()
self.running = False
- # normally, we'd just stop here, but for the test
- # harness, we want to stop the server
- self.server.stop()
-
def __init__(self, certificate=None, ssl_version=None,
certreqs=None, cacerts=None,
chatty=True, connectionchatty=False, starttls_server=False,
@@ -2472,21 +2467,33 @@ def __init__(self, certificate=None, ssl_version=None,
self.conn_errors = []
threading.Thread.__init__(self)
self.daemon = True
+ self._in_context = False
def __enter__(self):
+ if self._in_context:
+ raise ValueError('Re-entering ThreadedEchoServer context')
+ self._in_context = True
self.start(threading.Event())
self.flag.wait()
return self
def __exit__(self, *args):
+ assert self._in_context
+ self._in_context = False
self.stop()
self.join()
def start(self, flag=None):
+ if not self._in_context:
+ raise ValueError(
+ 'ThreadedEchoServer must be used as a context manager')
self.flag = flag
threading.Thread.start(self)
def run(self):
+ if not self._in_context:
+ raise ValueError(
+ 'ThreadedEchoServer must be used as a context manager')
self.sock.settimeout(1.0)
self.sock.listen(5)
self.active = True
From 7d091d54c53b535897158326a853b7439fec463b Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 11 Nov 2024 17:47:37 +0100
Subject: [PATCH 186/269] [3.12] gh-126505: Fix bugs in compiling
case-insensitive character classes (GH-126557) (GH-126690)
* upper-case non-BMP character was ignored
* the ASCII flag was ignored when matching a character range whose
upper bound is beyond the BMP region
(cherry picked from commit 819830f34a11ecaa3aada174ca8eedeb3f260630)
Co-authored-by: Serhiy Storchaka
---
Lib/re/_compiler.py | 23 +++++---
Lib/test/test_re.py | 55 +++++++++++++++++++
...-11-07-22-41-47.gh-issue-126505.iztYE1.rst | 4 ++
3 files changed, 73 insertions(+), 9 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-11-07-22-41-47.gh-issue-126505.iztYE1.rst
diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py
index 285c21936f2cfa..bb97f9fdd10de1 100644
--- a/Lib/re/_compiler.py
+++ b/Lib/re/_compiler.py
@@ -250,11 +250,11 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None):
while True:
try:
if op is LITERAL:
- if fixup:
- lo = fixup(av)
- charmap[lo] = 1
- if fixes and lo in fixes:
- for k in fixes[lo]:
+ if fixup: # IGNORECASE and not LOCALE
+ av = fixup(av)
+ charmap[av] = 1
+ if fixes and av in fixes:
+ for k in fixes[av]:
charmap[k] = 1
if not hascased and iscased(av):
hascased = True
@@ -262,7 +262,7 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None):
charmap[av] = 1
elif op is RANGE:
r = range(av[0], av[1]+1)
- if fixup:
+ if fixup: # IGNORECASE and not LOCALE
if fixes:
for i in map(fixup, r):
charmap[i] = 1
@@ -289,8 +289,7 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None):
# Character set contains non-BMP character codes.
# For range, all BMP characters in the range are already
# proceeded.
- if fixup:
- hascased = True
+ if fixup: # IGNORECASE and not LOCALE
# For now, IN_UNI_IGNORE+LITERAL and
# IN_UNI_IGNORE+RANGE_UNI_IGNORE work for all non-BMP
# characters, because two characters (at least one of
@@ -301,7 +300,13 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None):
# Also, both c.lower() and c.lower().upper() are single
# characters for every non-BMP character.
if op is RANGE:
- op = RANGE_UNI_IGNORE
+ if fixes: # not ASCII
+ op = RANGE_UNI_IGNORE
+ hascased = True
+ else:
+ assert op is LITERAL
+ if not hascased and iscased(av):
+ hascased = True
tail.append((op, av))
break
diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py
index cc39c9d40f7ffd..756a7ccd506be4 100644
--- a/Lib/test/test_re.py
+++ b/Lib/test/test_re.py
@@ -1073,6 +1073,39 @@ def test_ignore_case_set(self):
self.assertTrue(re.match(br'[19a]', b'a', re.I))
self.assertTrue(re.match(br'[19a]', b'A', re.I))
self.assertTrue(re.match(br'[19A]', b'a', re.I))
+ self.assertTrue(re.match(r'[19\xc7]', '\xc7', re.I))
+ self.assertTrue(re.match(r'[19\xc7]', '\xe7', re.I))
+ self.assertTrue(re.match(r'[19\xe7]', '\xc7', re.I))
+ self.assertTrue(re.match(r'[19\xe7]', '\xe7', re.I))
+ self.assertTrue(re.match(r'[19\u0400]', '\u0400', re.I))
+ self.assertTrue(re.match(r'[19\u0400]', '\u0450', re.I))
+ self.assertTrue(re.match(r'[19\u0450]', '\u0400', re.I))
+ self.assertTrue(re.match(r'[19\u0450]', '\u0450', re.I))
+ self.assertTrue(re.match(r'[19\U00010400]', '\U00010400', re.I))
+ self.assertTrue(re.match(r'[19\U00010400]', '\U00010428', re.I))
+ self.assertTrue(re.match(r'[19\U00010428]', '\U00010400', re.I))
+ self.assertTrue(re.match(r'[19\U00010428]', '\U00010428', re.I))
+
+ self.assertTrue(re.match(br'[19A]', b'A', re.I))
+ self.assertTrue(re.match(br'[19a]', b'a', re.I))
+ self.assertTrue(re.match(br'[19a]', b'A', re.I))
+ self.assertTrue(re.match(br'[19A]', b'a', re.I))
+ self.assertTrue(re.match(r'[19A]', 'A', re.I|re.A))
+ self.assertTrue(re.match(r'[19a]', 'a', re.I|re.A))
+ self.assertTrue(re.match(r'[19a]', 'A', re.I|re.A))
+ self.assertTrue(re.match(r'[19A]', 'a', re.I|re.A))
+ self.assertTrue(re.match(r'[19\xc7]', '\xc7', re.I|re.A))
+ self.assertIsNone(re.match(r'[19\xc7]', '\xe7', re.I|re.A))
+ self.assertIsNone(re.match(r'[19\xe7]', '\xc7', re.I|re.A))
+ self.assertTrue(re.match(r'[19\xe7]', '\xe7', re.I|re.A))
+ self.assertTrue(re.match(r'[19\u0400]', '\u0400', re.I|re.A))
+ self.assertIsNone(re.match(r'[19\u0400]', '\u0450', re.I|re.A))
+ self.assertIsNone(re.match(r'[19\u0450]', '\u0400', re.I|re.A))
+ self.assertTrue(re.match(r'[19\u0450]', '\u0450', re.I|re.A))
+ self.assertTrue(re.match(r'[19\U00010400]', '\U00010400', re.I|re.A))
+ self.assertIsNone(re.match(r'[19\U00010400]', '\U00010428', re.I|re.A))
+ self.assertIsNone(re.match(r'[19\U00010428]', '\U00010400', re.I|re.A))
+ self.assertTrue(re.match(r'[19\U00010428]', '\U00010428', re.I|re.A))
# Two different characters have the same lowercase.
assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K'
@@ -1109,8 +1142,10 @@ def test_ignore_case_range(self):
self.assertTrue(re.match(br'[9-a]', b'_', re.I))
self.assertIsNone(re.match(br'[9-A]', b'_', re.I))
self.assertTrue(re.match(r'[\xc0-\xde]', '\xd7', re.I))
+ self.assertTrue(re.match(r'[\xc0-\xde]', '\xe7', re.I))
self.assertIsNone(re.match(r'[\xc0-\xde]', '\xf7', re.I))
self.assertTrue(re.match(r'[\xe0-\xfe]', '\xf7', re.I))
+ self.assertTrue(re.match(r'[\xe0-\xfe]', '\xc7', re.I))
self.assertIsNone(re.match(r'[\xe0-\xfe]', '\xd7', re.I))
self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0450', re.I))
self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0400', re.I))
@@ -1121,6 +1156,26 @@ def test_ignore_case_range(self):
self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I))
self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I))
+ self.assertTrue(re.match(r'[\xc0-\xde]', '\xd7', re.I|re.A))
+ self.assertIsNone(re.match(r'[\xc0-\xde]', '\xe7', re.I|re.A))
+ self.assertTrue(re.match(r'[\xe0-\xfe]', '\xf7', re.I|re.A))
+ self.assertIsNone(re.match(r'[\xe0-\xfe]', '\xc7', re.I|re.A))
+ self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0450', re.I|re.A))
+ self.assertIsNone(re.match(r'[\u0430-\u045f]', '\u0400', re.I|re.A))
+ self.assertIsNone(re.match(r'[\u0400-\u042f]', '\u0450', re.I|re.A))
+ self.assertTrue(re.match(r'[\u0400-\u042f]', '\u0400', re.I|re.A))
+ self.assertTrue(re.match(r'[\U00010428-\U0001044f]', '\U00010428', re.I|re.A))
+ self.assertIsNone(re.match(r'[\U00010428-\U0001044f]', '\U00010400', re.I|re.A))
+ self.assertIsNone(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I|re.A))
+ self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I|re.A))
+
+ self.assertTrue(re.match(r'[N-\x7f]', 'A', re.I|re.A))
+ self.assertTrue(re.match(r'[n-\x7f]', 'Z', re.I|re.A))
+ self.assertTrue(re.match(r'[N-\uffff]', 'A', re.I|re.A))
+ self.assertTrue(re.match(r'[n-\uffff]', 'Z', re.I|re.A))
+ self.assertTrue(re.match(r'[N-\U00010000]', 'A', re.I|re.A))
+ self.assertTrue(re.match(r'[n-\U00010000]', 'Z', re.I|re.A))
+
# Two different characters have the same lowercase.
assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K'
self.assertTrue(re.match(r'[J-M]', '\u212a', re.I))
diff --git a/Misc/NEWS.d/next/Library/2024-11-07-22-41-47.gh-issue-126505.iztYE1.rst b/Misc/NEWS.d/next/Library/2024-11-07-22-41-47.gh-issue-126505.iztYE1.rst
new file mode 100644
index 00000000000000..0a0f893a2688a0
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-11-07-22-41-47.gh-issue-126505.iztYE1.rst
@@ -0,0 +1,4 @@
+Fix bugs in compiling case-insensitive :mod:`regular expressions ` with
+character classes containing non-BMP characters: upper-case non-BMP
+character did was ignored and the ASCII flag was ignored when
+matching a character range whose upper bound is beyond the BMP region.
From 5cd35fb8e0ed18188467ac3bca2c56815b02f956 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Mon, 11 Nov 2024 18:20:33 +0100
Subject: [PATCH 187/269] Update documentation links to Microsoft's
documentation pages (GH-126379)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
(cherry picked from commit 6e25eb15410f781f632d536d555f38879432522c)
Co-authored-by: 谭九鼎 <109224573@qq.com>
---
Doc/library/asyncio-eventloop.rst | 2 +-
Doc/library/time.rst | 2 +-
Doc/using/windows.rst | 14 +++++++-------
3 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
index bdcea54f6a0450..53aaae413027d4 100644
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -1750,7 +1750,7 @@ on Unix and :class:`ProactorEventLoop` on Windows.
.. seealso::
`MSDN documentation on I/O Completion Ports
- `_.
+ `_.
.. class:: AbstractEventLoop
diff --git a/Doc/library/time.rst b/Doc/library/time.rst
index 5d6365f23fdc2b..db53296e2a5698 100644
--- a/Doc/library/time.rst
+++ b/Doc/library/time.rst
@@ -372,7 +372,7 @@ Functions
threads ready to run, the function returns immediately, and the thread
continues execution. On Windows 8.1 and newer the implementation uses
a `high-resolution timer
- `_
+ `_
which provides resolution of 100 nanoseconds. If *secs* is zero, ``Sleep(0)`` is used.
Unix implementation:
diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst
index 87fccfbc7e591e..2a507675044666 100644
--- a/Doc/using/windows.rst
+++ b/Doc/using/windows.rst
@@ -394,7 +394,7 @@ When writing to the Windows Registry, the following behaviors exist:
For more detail on the technical basis for these limitations, please consult
Microsoft's documentation on packaged full-trust apps, currently available at
`docs.microsoft.com/en-us/windows/msix/desktop/desktop-to-uwp-behind-the-scenes
-`_
+`_
.. _windows-nuget:
@@ -475,7 +475,7 @@ dependents, such as Idle), pip and the Python documentation are not included.
.. note::
The embedded distribution does not include the `Microsoft C Runtime
- `_ and it is
+ `_ and it is
the responsibility of the application installer to provide this. The
runtime may have already been installed on a user's system previously or
automatically via Windows Update, and can be detected by finding
@@ -618,13 +618,13 @@ System variables, you need non-restricted access to your machine
.. seealso::
- https://docs.microsoft.com/en-us/windows/win32/procthread/environment-variables
+ https://learn.microsoft.com/windows/win32/procthread/environment-variables
Overview of environment variables on Windows
- https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/set_1
+ https://learn.microsoft.com/windows-server/administration/windows-commands/set_1
The ``set`` command, for temporarily modifying environment variables
- https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/setx
+ https://learn.microsoft.com/windows-server/administration/windows-commands/setx
The ``setx`` command, for permanently modifying environment variables
@@ -1222,13 +1222,13 @@ is a collection of modules for advanced Windows-specific support. This includes
utilities for:
* `Component Object Model
- `_
+ `_
(COM)
* Win32 API calls
* Registry
* Event log
* `Microsoft Foundation Classes
- `_
+ `_
(MFC) user interfaces
`PythonWin
Date: Tue, 12 Nov 2024 09:10:55 +0100
Subject: [PATCH 188/269] [3.12] Fix error message of "Check if Autoconf files
are up to date" job (GH-126683) (#126718)
Fix error message of "Check if Autoconf files are up to date" job (GH-126683)
(cherry picked from commit 0052a8c638518447baf39ae02b6ff6a309efd4ce)
Co-authored-by: sobolevn
---
.github/workflows/build.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 325752ecde4967..329ebc42a301b4 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -127,7 +127,7 @@ jobs:
# Check for changes in regenerated files
if test -n "$changes"; then
echo "Generated files not up to date."
- echo "Perhaps you forgot to run make regen-all or build.bat --regen. ;)"
+ echo "Perhaps you forgot to run make regen-configure ;)"
echo "configure files must be regenerated with a specific version of autoconf."
echo "$changes"
echo ""
From bc31c73f5aaa1c3d03f1552be6be8b44b4604b4c Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 12 Nov 2024 11:41:28 +0100
Subject: [PATCH 189/269] [3.12] gh-126525: Fix `makeunicodedata.py` output on
macOS and Windows (GH-126526) (#126726)
gh-126525: Fix `makeunicodedata.py` output on macOS and Windows (GH-126526)
(cherry picked from commit f223efb2a2d6a3e86556be7295cbbd3ef839f489)
Co-authored-by: sobolevn
---
Tools/unicode/makeunicodedata.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py
index 034642db06e48b..8732db22dbd630 100644
--- a/Tools/unicode/makeunicodedata.py
+++ b/Tools/unicode/makeunicodedata.py
@@ -35,7 +35,7 @@
from textwrap import dedent
from typing import Iterator, List, Optional, Set, Tuple
-SCRIPT = sys.argv[0]
+SCRIPT = os.path.normpath(sys.argv[0])
VERSION = "3.3"
# The Unicode Database
From 86efa2f1fa5ad8ddcf5966104a580cd33218547e Mon Sep 17 00:00:00 2001
From: Alex Waygood
Date: Tue, 12 Nov 2024 12:26:23 +0000
Subject: [PATCH 190/269] [3.12] gh-126451: Revert backports of ABC
registrations for contextvars.Context and multiprocessing proxies (#126735)
---
Lib/contextvars.py | 4 ----
Lib/multiprocessing/managers.py | 4 ----
Lib/test/_test_multiprocessing.py | 9 ---------
Lib/test/test_context.py | 14 --------------
Misc/ACKS | 1 -
.../2024-11-04-16-40-02.gh-issue-126417.OWPqn0.rst | 3 ---
.../2024-11-05-11-28-45.gh-issue-126451.XJMtqz.rst | 2 --
7 files changed, 37 deletions(-)
delete mode 100644 Misc/NEWS.d/next/Library/2024-11-04-16-40-02.gh-issue-126417.OWPqn0.rst
delete mode 100644 Misc/NEWS.d/next/Library/2024-11-05-11-28-45.gh-issue-126451.XJMtqz.rst
diff --git a/Lib/contextvars.py b/Lib/contextvars.py
index 14514f185e069d..d78c80dfe6f99c 100644
--- a/Lib/contextvars.py
+++ b/Lib/contextvars.py
@@ -1,8 +1,4 @@
-import _collections_abc
from _contextvars import Context, ContextVar, Token, copy_context
__all__ = ('Context', 'ContextVar', 'Token', 'copy_context')
-
-
-_collections_abc.Mapping.register(Context)
diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py
index b915e67c265b3d..75d9c18c201a86 100644
--- a/Lib/multiprocessing/managers.py
+++ b/Lib/multiprocessing/managers.py
@@ -18,7 +18,6 @@
import threading
import signal
import array
-import collections.abc
import queue
import time
import types
@@ -1161,8 +1160,6 @@ def __imul__(self, value):
return self
-collections.abc.MutableSequence.register(BaseListProxy)
-
DictProxy = MakeProxyType('DictProxy', (
'__contains__', '__delitem__', '__getitem__', '__iter__', '__len__',
'__setitem__', 'clear', 'copy', 'get', 'items',
@@ -1172,7 +1169,6 @@ def __imul__(self, value):
'__iter__': 'Iterator',
}
-collections.abc.MutableMapping.register(DictProxy)
ArrayProxy = MakeProxyType('ArrayProxy', (
'__len__', '__getitem__', '__setitem__'
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index c190dc918dac64..3b4415b50ae2c2 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -17,7 +17,6 @@
import functools
import signal
import array
-import collections.abc
import socket
import random
import logging
@@ -2460,10 +2459,6 @@ def test_list(self):
a.append('hello')
self.assertEqual(f[0][:], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 'hello'])
- def test_list_isinstance(self):
- a = self.list()
- self.assertIsInstance(a, collections.abc.MutableSequence)
-
def test_list_iter(self):
a = self.list(list(range(10)))
it = iter(a)
@@ -2504,10 +2499,6 @@ def test_dict(self):
self.assertEqual(sorted(d.values()), [chr(i) for i in indices])
self.assertEqual(sorted(d.items()), [(i, chr(i)) for i in indices])
- def test_dict_isinstance(self):
- a = self.dict()
- self.assertIsInstance(a, collections.abc.MutableMapping)
-
def test_dict_iter(self):
d = self.dict()
indices = list(range(65, 70))
diff --git a/Lib/test/test_context.py b/Lib/test/test_context.py
index d9e1c6214e7057..dc6856509a40a0 100644
--- a/Lib/test/test_context.py
+++ b/Lib/test/test_context.py
@@ -1,4 +1,3 @@
-import collections.abc
import concurrent.futures
import contextvars
import functools
@@ -343,19 +342,6 @@ def ctx2_fun():
ctx1.run(ctx1_fun)
- def test_context_isinstance(self):
- ctx = contextvars.Context()
- self.assertIsInstance(ctx, collections.abc.Mapping)
- self.assertTrue(issubclass(contextvars.Context, collections.abc.Mapping))
-
- mapping_methods = (
- '__contains__', '__eq__', '__getitem__', '__iter__', '__len__',
- '__ne__', 'get', 'items', 'keys', 'values',
- )
- for name in mapping_methods:
- with self.subTest(name=name):
- self.assertTrue(callable(getattr(ctx, name)))
-
@isolated_context
@threading_helper.requires_working_threading()
def test_context_threads_1(self):
diff --git a/Misc/ACKS b/Misc/ACKS
index b5cf6acc55a88c..837ffbda18aea1 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1259,7 +1259,6 @@ Emily Morehouse
Derek Morr
James A Morrison
Martin Morrison
-Stephen Morton
Derek McTavish Mounce
Alessandro Moura
Pablo Mouzo
diff --git a/Misc/NEWS.d/next/Library/2024-11-04-16-40-02.gh-issue-126417.OWPqn0.rst b/Misc/NEWS.d/next/Library/2024-11-04-16-40-02.gh-issue-126417.OWPqn0.rst
deleted file mode 100644
index c4a366343382f3..00000000000000
--- a/Misc/NEWS.d/next/Library/2024-11-04-16-40-02.gh-issue-126417.OWPqn0.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Register the :class:`!multiprocessing.managers.DictProxy` and :class:`!multiprocessing.managers.ListProxy` types in
-:mod:`multiprocessing.managers` to :class:`collections.abc.MutableMapping` and
-:class:`collections.abc.MutableSequence`, respectively.
diff --git a/Misc/NEWS.d/next/Library/2024-11-05-11-28-45.gh-issue-126451.XJMtqz.rst b/Misc/NEWS.d/next/Library/2024-11-05-11-28-45.gh-issue-126451.XJMtqz.rst
deleted file mode 100644
index 563cb2515eca60..00000000000000
--- a/Misc/NEWS.d/next/Library/2024-11-05-11-28-45.gh-issue-126451.XJMtqz.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Register the :class:`contextvars.Context` type to
-:class:`collections.abc.Mapping`.
From b13b84ab707bbc06b26a1ed4fb58253ff309c5dd Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 12 Nov 2024 14:04:25 +0100
Subject: [PATCH 191/269] [3.12] gh-126405: fix use-after-free in
`_asyncio.Future.remove_done_callback` (GH-126733) (#126737)
gh-126405: fix use-after-free in `_asyncio.Future.remove_done_callback` (GH-126733)
(cherry picked from commit 37c57dfad12744608091653fd753a1f770e2479b)
Co-authored-by: Kumar Aditya
---
Modules/_asynciomodule.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c
index 20a9d992834b0e..2356b708ee88b4 100644
--- a/Modules/_asynciomodule.c
+++ b/Modules/_asynciomodule.c
@@ -1073,8 +1073,10 @@ _asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls,
if (len == 1) {
PyObject *cb_tup = PyList_GET_ITEM(self->fut_callbacks, 0);
+ Py_INCREF(cb_tup);
int cmp = PyObject_RichCompareBool(
PyTuple_GET_ITEM(cb_tup, 0), fn, Py_EQ);
+ Py_DECREF(cb_tup);
if (cmp == -1) {
return NULL;
}
From 46710ca5f263936a2e36fa5d0f140cf9f50b2618 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?=
<10796600+picnixz@users.noreply.github.com>
Date: Tue, 12 Nov 2024 15:05:23 +0100
Subject: [PATCH 192/269] [3.12] gh-126595: fix a crash when calling
`itertools.count(sys.maxsize)` (GH-126617) (#126740)
gh-126595: fix a crash when calling `itertools.count(sys.maxsize)` (#126617)
---
Lib/test/test_itertools.py | 8 ++++++++
.../2024-11-09-10-31-10.gh-issue-126595.A-7MyC.rst | 2 ++
Modules/itertoolsmodule.c | 3 +++
3 files changed, 13 insertions(+)
create mode 100644 Misc/NEWS.d/next/Library/2024-11-09-10-31-10.gh-issue-126595.A-7MyC.rst
diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py
index b6404f4366ca0e..89eb78ce42994f 100644
--- a/Lib/test/test_itertools.py
+++ b/Lib/test/test_itertools.py
@@ -599,6 +599,8 @@ def test_count(self):
self.assertEqual(take(2, zip('abc',count(-3))), [('a', -3), ('b', -2)])
self.assertRaises(TypeError, count, 2, 3, 4)
self.assertRaises(TypeError, count, 'a')
+ self.assertEqual(take(3, count(maxsize)),
+ [maxsize, maxsize + 1, maxsize + 2])
self.assertEqual(take(10, count(maxsize-5)),
list(range(maxsize-5, maxsize+5)))
self.assertEqual(take(10, count(-maxsize-5)),
@@ -654,6 +656,12 @@ def test_count_with_stride(self):
self.assertEqual(take(20, count(-maxsize-15, 3)), take(20, range(-maxsize-15,-maxsize+100, 3)))
self.assertEqual(take(3, count(10, maxsize+5)),
list(range(10, 10+3*(maxsize+5), maxsize+5)))
+ self.assertEqual(take(3, count(maxsize, 2)),
+ [maxsize, maxsize + 2, maxsize + 4])
+ self.assertEqual(take(3, count(maxsize, maxsize)),
+ [maxsize, 2 * maxsize, 3 * maxsize])
+ self.assertEqual(take(3, count(-maxsize, maxsize)),
+ [-maxsize, 0, maxsize])
self.assertEqual(take(3, count(2, 1.25)), [2, 3.25, 4.5])
self.assertEqual(take(3, count(2, 3.25-4j)), [2, 5.25-4j, 8.5-8j])
self.assertEqual(take(3, count(Decimal('1.1'), Decimal('.1'))),
diff --git a/Misc/NEWS.d/next/Library/2024-11-09-10-31-10.gh-issue-126595.A-7MyC.rst b/Misc/NEWS.d/next/Library/2024-11-09-10-31-10.gh-issue-126595.A-7MyC.rst
new file mode 100644
index 00000000000000..84a5dc0b23922f
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-11-09-10-31-10.gh-issue-126595.A-7MyC.rst
@@ -0,0 +1,2 @@
+Fix a crash when instantiating :class:`itertools.count` with an initial
+count of :data:`sys.maxsize` on debug builds. Patch by Bénédikt Tran.
diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c
index e87c753113563f..17e0eecbad2008 100644
--- a/Modules/itertoolsmodule.c
+++ b/Modules/itertoolsmodule.c
@@ -4037,6 +4037,9 @@ itertools_count_impl(PyTypeObject *type, PyObject *long_cnt,
PyErr_Clear();
fast_mode = 0;
}
+ else if (cnt == PY_SSIZE_T_MAX) {
+ fast_mode = 0;
+ }
}
} else {
cnt = 0;
From 361dda59a3a73e4bd2b6498d16dcc22195670d1d Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Tue, 12 Nov 2024 22:53:58 +0100
Subject: [PATCH 193/269] [3.12] GH-120423: `pathname2url()`: handle forward
slashes in Windows paths (GH-126593) (#126763)
GH-120423: `pathname2url()`: handle forward slashes in Windows paths (GH-126593)
Adjust `urllib.request.pathname2url()` so that forward slashes in Windows
paths are handled identically to backward slashes.
(cherry picked from commit bf224bd7cef5d24eaff35945ebe7ffe14df7710f)
Co-authored-by: Barney Gale
---
Lib/nturl2path.py | 13 +++++++------
Lib/test/test_urllib.py | 5 +++++
.../2024-11-08-17-05-10.gh-issue-120423.7rdLVV.rst | 2 ++
3 files changed, 14 insertions(+), 6 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-11-08-17-05-10.gh-issue-120423.7rdLVV.rst
diff --git a/Lib/nturl2path.py b/Lib/nturl2path.py
index 2f9fec7893afd1..9ecabff21c33e1 100644
--- a/Lib/nturl2path.py
+++ b/Lib/nturl2path.py
@@ -44,20 +44,21 @@ def pathname2url(p):
import urllib.parse
# First, clean up some special forms. We are going to sacrifice
# the additional information anyway
- if p[:4] == '\\\\?\\':
+ p = p.replace('\\', '/')
+ if p[:4] == '//?/':
p = p[4:]
- if p[:4].upper() == 'UNC\\':
- p = '\\\\' + p[4:]
+ if p[:4].upper() == 'UNC/':
+ p = '//' + p[4:]
elif p[1:2] != ':':
raise OSError('Bad path: ' + p)
if not ':' in p:
- # No drive specifier, just convert slashes and quote the name
- return urllib.parse.quote(p.replace('\\', '/'))
+ # No DOS drive specified, just quote the pathname
+ return urllib.parse.quote(p)
comp = p.split(':', maxsplit=2)
if len(comp) != 2 or len(comp[0]) > 1:
error = 'Bad path: ' + p
raise OSError(error)
drive = urllib.parse.quote(comp[0].upper())
- tail = urllib.parse.quote(comp[1].replace('\\', '/'))
+ tail = urllib.parse.quote(comp[1])
return '///' + drive + ':' + tail
diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py
index 2b59fb617cbdda..5433072bcda741 100644
--- a/Lib/test/test_urllib.py
+++ b/Lib/test/test_urllib.py
@@ -1551,6 +1551,11 @@ def test_pathname2url_win(self):
self.assertEqual(fn('\\\\some\\share\\'), '//some/share/')
self.assertEqual(fn('\\\\some\\share\\a\\b.c'), '//some/share/a/b.c')
self.assertEqual(fn('\\\\some\\share\\a\\b%#c\xe9'), '//some/share/a/b%25%23c%C3%A9')
+ # Alternate path separator
+ self.assertEqual(fn('C:/a/b.c'), '///C:/a/b.c')
+ self.assertEqual(fn('//some/share/a/b.c'), '//some/share/a/b.c')
+ self.assertEqual(fn('//?/C:/dir'), '///C:/dir')
+ self.assertEqual(fn('//?/unc/server/share/dir'), '//server/share/dir')
# Round-tripping
urls = ['///C:',
'///folder/test/',
diff --git a/Misc/NEWS.d/next/Library/2024-11-08-17-05-10.gh-issue-120423.7rdLVV.rst b/Misc/NEWS.d/next/Library/2024-11-08-17-05-10.gh-issue-120423.7rdLVV.rst
new file mode 100644
index 00000000000000..b475257ceb6610
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-11-08-17-05-10.gh-issue-120423.7rdLVV.rst
@@ -0,0 +1,2 @@
+Fix issue where :func:`urllib.request.pathname2url` mishandled Windows paths
+with embedded forward slashes.
From 951ed3305489ef2180972c9786de0ff61042128b Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 13 Nov 2024 09:37:34 +0100
Subject: [PATCH 194/269] [3.12] gh-104745: Limit starting a patcher more than
once without stopping it (GH-126649) (#126773)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
gh-104745: Limit starting a patcher more than once without stopping it (GH-126649)
Previously, this would cause an `AttributeError` if the patch stopped more than once after this, and would also disrupt the original patched object.
---------
(cherry picked from commit 1e40c5ba47780ddd91868abb3aa064f5ba3015e4)
Co-authored-by: Red4Ru <39802734+Red4Ru@users.noreply.github.com>
Co-authored-by: Peter Bierma
Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
---
Lib/test/test_unittest/testmock/testpatch.py | 52 ++++++++++++++++++-
Lib/unittest/mock.py | 9 ++++
...-11-10-18-14-51.gh-issue-104745.zAa5Ke.rst | 3 ++
3 files changed, 62 insertions(+), 2 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-11-10-18-14-51.gh-issue-104745.zAa5Ke.rst
diff --git a/Lib/test/test_unittest/testmock/testpatch.py b/Lib/test/test_unittest/testmock/testpatch.py
index f26e74ce0bc1ba..037c021e6eafcf 100644
--- a/Lib/test/test_unittest/testmock/testpatch.py
+++ b/Lib/test/test_unittest/testmock/testpatch.py
@@ -745,6 +745,54 @@ def test_stop_idempotent(self):
self.assertIsNone(patcher.stop())
+ def test_exit_idempotent(self):
+ patcher = patch(foo_name, 'bar', 3)
+ with patcher:
+ patcher.stop()
+
+
+ def test_second_start_failure(self):
+ patcher = patch(foo_name, 'bar', 3)
+ patcher.start()
+ try:
+ self.assertRaises(RuntimeError, patcher.start)
+ finally:
+ patcher.stop()
+
+
+ def test_second_enter_failure(self):
+ patcher = patch(foo_name, 'bar', 3)
+ with patcher:
+ self.assertRaises(RuntimeError, patcher.start)
+
+
+ def test_second_start_after_stop(self):
+ patcher = patch(foo_name, 'bar', 3)
+ patcher.start()
+ patcher.stop()
+ patcher.start()
+ patcher.stop()
+
+
+ def test_property_setters(self):
+ mock_object = Mock()
+ mock_bar = mock_object.bar
+ patcher = patch.object(mock_object, 'bar', 'x')
+ with patcher:
+ self.assertEqual(patcher.is_local, False)
+ self.assertIs(patcher.target, mock_object)
+ self.assertEqual(patcher.temp_original, mock_bar)
+ patcher.is_local = True
+ patcher.target = mock_bar
+ patcher.temp_original = mock_object
+ self.assertEqual(patcher.is_local, True)
+ self.assertIs(patcher.target, mock_bar)
+ self.assertEqual(patcher.temp_original, mock_object)
+ # if changes are left intact, they may lead to disruption as shown below (it might be what someone needs though)
+ self.assertEqual(mock_bar.bar, mock_object)
+ self.assertEqual(mock_object.bar, 'x')
+
+
def test_patchobject_start_stop(self):
original = something
patcher = patch.object(PTModule, 'something', 'foo')
@@ -1098,7 +1146,7 @@ def test_new_callable_patch(self):
self.assertIsNot(m1, m2)
for mock in m1, m2:
- self.assertNotCallable(m1)
+ self.assertNotCallable(mock)
def test_new_callable_patch_object(self):
@@ -1111,7 +1159,7 @@ def test_new_callable_patch_object(self):
self.assertIsNot(m1, m2)
for mock in m1, m2:
- self.assertNotCallable(m1)
+ self.assertNotCallable(mock)
def test_new_callable_keyword_arguments(self):
diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 3e9791b22dc450..c4ce8f8a3eb930 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -1329,6 +1329,7 @@ def __init__(
self.autospec = autospec
self.kwargs = kwargs
self.additional_patchers = []
+ self.is_started = False
def copy(self):
@@ -1441,6 +1442,9 @@ def get_original(self):
def __enter__(self):
"""Perform the patch."""
+ if self.is_started:
+ raise RuntimeError("Patch is already started")
+
new, spec, spec_set = self.new, self.spec, self.spec_set
autospec, kwargs = self.autospec, self.kwargs
new_callable = self.new_callable
@@ -1572,6 +1576,7 @@ def __enter__(self):
self.temp_original = original
self.is_local = local
self._exit_stack = contextlib.ExitStack()
+ self.is_started = True
try:
setattr(self.target, self.attribute, new_attr)
if self.attribute_name is not None:
@@ -1591,6 +1596,9 @@ def __enter__(self):
def __exit__(self, *exc_info):
"""Undo the patch."""
+ if not self.is_started:
+ return
+
if self.is_local and self.temp_original is not DEFAULT:
setattr(self.target, self.attribute, self.temp_original)
else:
@@ -1607,6 +1615,7 @@ def __exit__(self, *exc_info):
del self.target
exit_stack = self._exit_stack
del self._exit_stack
+ self.is_started = False
return exit_stack.__exit__(*exc_info)
diff --git a/Misc/NEWS.d/next/Library/2024-11-10-18-14-51.gh-issue-104745.zAa5Ke.rst b/Misc/NEWS.d/next/Library/2024-11-10-18-14-51.gh-issue-104745.zAa5Ke.rst
new file mode 100644
index 00000000000000..c83a10769820cf
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-11-10-18-14-51.gh-issue-104745.zAa5Ke.rst
@@ -0,0 +1,3 @@
+Limit starting a patcher (from :func:`unittest.mock.patch` or
+:func:`unittest.mock.patch.object`) more than
+once without stopping it
From dd212592834bb3a4be43b5d7d1a90876701e9885 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 13 Nov 2024 10:20:47 +0100
Subject: [PATCH 195/269] [3.12] bpo-46128: Strip IsolatedAsyncioTestCase
frames from reported stacktraces (GH-30196) (#126771)
bpo-46128: Strip IsolatedAsyncioTestCase frames from reported stacktraces (GH-30196)
---------
(cherry picked from commit 2e39d77ddeb51505d65fd54ccfcd72615c6b1927)
Co-authored-by: Andrew Svetlov
Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com>
Co-authored-by: Ezio Melotti
Co-authored-by: Hugo van Kemenade
---
Lib/unittest/async_case.py | 1 +
.../next/Library/2021-12-19-10-47-24.bpo-46128.Qv3EK1.rst | 2 ++
2 files changed, 3 insertions(+)
create mode 100644 Misc/NEWS.d/next/Library/2021-12-19-10-47-24.bpo-46128.Qv3EK1.rst
diff --git a/Lib/unittest/async_case.py b/Lib/unittest/async_case.py
index bd2a471156065b..2abfb79079822a 100644
--- a/Lib/unittest/async_case.py
+++ b/Lib/unittest/async_case.py
@@ -5,6 +5,7 @@
from .case import TestCase
+__unittest = True
class IsolatedAsyncioTestCase(TestCase):
# Names intentionally have a long prefix
diff --git a/Misc/NEWS.d/next/Library/2021-12-19-10-47-24.bpo-46128.Qv3EK1.rst b/Misc/NEWS.d/next/Library/2021-12-19-10-47-24.bpo-46128.Qv3EK1.rst
new file mode 100644
index 00000000000000..7d11d20d94e8a3
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-12-19-10-47-24.bpo-46128.Qv3EK1.rst
@@ -0,0 +1,2 @@
+Strip :class:`unittest.IsolatedAsyncioTestCase` stack frames from reported
+stacktraces.
From 0bdcc84ae99ff695b10b2f13c5a00497ce0840e5 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 13 Nov 2024 19:44:01 +0100
Subject: [PATCH 196/269] [3.12] gh-126341: add release check to `__iter__`
method of `memoryview` (GH-126759) (#126779)
gh-126341: add release check to `__iter__` method of `memoryview` (GH-126759)
(cherry picked from commit a12690ef49e8fc8a3af4c5f1757eb3caffb35e03)
Co-authored-by: Ritvik Pasham
Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>
Co-authored-by: Peter Bierma
Co-authored-by: Victor Stinner
Co-authored-by: sobolevn
---
Lib/test/test_buffer.py | 2 ++
.../2024-11-12-19-24-00.gh-issue-126341.5SdAe1.rst | 1 +
Objects/memoryobject.c | 1 +
3 files changed, 4 insertions(+)
create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-12-19-24-00.gh-issue-126341.5SdAe1.rst
diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py
index aafbb8a993def5..84a34bccbc9af1 100644
--- a/Lib/test/test_buffer.py
+++ b/Lib/test/test_buffer.py
@@ -3906,6 +3906,8 @@ def test_memoryview_check_released(self):
self.assertRaises(ValueError, memoryview, m)
# memoryview.cast()
self.assertRaises(ValueError, m.cast, 'c')
+ # memoryview.__iter__()
+ self.assertRaises(ValueError, m.__iter__)
# getbuffer()
self.assertRaises(ValueError, ndarray, m)
# memoryview.tolist()
diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-12-19-24-00.gh-issue-126341.5SdAe1.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-12-19-24-00.gh-issue-126341.5SdAe1.rst
new file mode 100644
index 00000000000000..c2436d2ebf4d09
--- /dev/null
+++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-12-19-24-00.gh-issue-126341.5SdAe1.rst
@@ -0,0 +1 @@
+Now :exc:`ValueError` is raised instead of :exc:`SystemError` when trying to iterate over a released :class:`memoryview` object.
diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c
index 26871612ea794d..9a5f9c665b2a9f 100644
--- a/Objects/memoryobject.c
+++ b/Objects/memoryobject.c
@@ -3322,6 +3322,7 @@ memory_iter(PyObject *seq)
PyErr_BadInternalCall();
return NULL;
}
+ CHECK_RELEASED(seq);
PyMemoryViewObject *obj = (PyMemoryViewObject *)seq;
int ndims = obj->view.ndim;
if (ndims == 0) {
From 9e86d211fce5d60a67468777664c6246c9caa818 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 13 Nov 2024 19:50:29 +0100
Subject: [PATCH 197/269] [3.12] gh-126623: Update libexpat to 2.6.4, make
future updates easier (GH-126792) (GH-126797)
gh-126623: Update libexpat to 2.6.4, make future updates easier (GH-126792)
Update libexpat to 2.6.4, make future updates easier.
(cherry picked from commit 3c9996909402fadc98e6ca2a64e75a71a7427352)
Co-authored-by: Seth Michael Larson
---
...-11-13-11-09-12.gh-issue-126623.TO7NnR.rst | 1 +
Misc/sbom.spdx.json | 22 +++----
Modules/expat/expat.h | 6 +-
Modules/expat/expat_external.h | 9 ++-
Modules/expat/refresh.sh | 57 +++++++++++++++++++
Modules/expat/xmlparse.c | 18 ++++--
Tools/build/generate_sbom.py | 28 +++++++++
7 files changed, 119 insertions(+), 22 deletions(-)
create mode 100644 Misc/NEWS.d/next/Security/2024-11-13-11-09-12.gh-issue-126623.TO7NnR.rst
create mode 100755 Modules/expat/refresh.sh
diff --git a/Misc/NEWS.d/next/Security/2024-11-13-11-09-12.gh-issue-126623.TO7NnR.rst b/Misc/NEWS.d/next/Security/2024-11-13-11-09-12.gh-issue-126623.TO7NnR.rst
new file mode 100644
index 00000000000000..f09a158af2a475
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2024-11-13-11-09-12.gh-issue-126623.TO7NnR.rst
@@ -0,0 +1 @@
+Upgrade libexpat to 2.6.4
diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json
index b33c66ab89399a..eb13ecff51528d 100644
--- a/Misc/sbom.spdx.json
+++ b/Misc/sbom.spdx.json
@@ -48,11 +48,11 @@
"checksums": [
{
"algorithm": "SHA1",
- "checksumValue": "6aaee1b194bea30f0a60d1cce71eada8b14d3526"
+ "checksumValue": "373cc00d87782a736970644d863ff2ebbd0e4886"
},
{
"algorithm": "SHA256",
- "checksumValue": "7bd4e53a8015534b5bbb58afe1a131b3989d3d4fca29bca685c44d34bcaa2555"
+ "checksumValue": "0f750bc336e510d14ac9a3e63fc2399f60f3f04f0061c426e86751ed5fba90e4"
}
],
"fileName": "Modules/expat/expat.h"
@@ -62,11 +62,11 @@
"checksums": [
{
"algorithm": "SHA1",
- "checksumValue": "b70ce53fdc25ae482681ae2f6623c3c8edc9c1b7"
+ "checksumValue": "9e615c6e5c3ba00670f674a6b071bb855b0b563d"
},
{
"algorithm": "SHA256",
- "checksumValue": "86afb425ec9999eb4f1ec9ab2fb41c58c4aa5cb9bf934b8c94264670fc5a961d"
+ "checksumValue": "3d90a4b65c40a3f848c36100f4d73b933a015c7b7cd85c28e4331a6b845c1ad0"
}
],
"fileName": "Modules/expat/expat_external.h"
@@ -128,18 +128,18 @@
"fileName": "Modules/expat/nametab.h"
},
{
- "SPDXID": "SPDXRef-FILE-Modules-expat-pyexpatns.h",
+ "SPDXID": "SPDXRef-FILE-Modules-expat-refresh.sh",
"checksums": [
{
"algorithm": "SHA1",
- "checksumValue": "f50c899172acd93fc539007bfb43315b83d407e4"
+ "checksumValue": "a9b0a33b8359cfe94b23972a1605daf8dcc605d9"
},
{
"algorithm": "SHA256",
- "checksumValue": "d571b8258cfaa067a20adef553e5fcedd6671ca4a8841483496de031bd904567"
+ "checksumValue": "19eb541460bc2ca8b87118acd3c048f6af77affbf8719ac29aa7b6c8d70f83fd"
}
],
- "fileName": "Modules/expat/pyexpatns.h"
+ "fileName": "Modules/expat/refresh.sh"
},
{
"SPDXID": "SPDXRef-FILE-Modules-expat-siphash.h",
@@ -188,11 +188,11 @@
"checksums": [
{
"algorithm": "SHA1",
- "checksumValue": "b2ec0ad170ccc21e63fbcfc8d7404cdd756eedd3"
+ "checksumValue": "3199fbd38b6fb158f73d5c8de6b6e6e3812ef803"
},
{
"algorithm": "SHA256",
- "checksumValue": "92159d4e17393e56ee85f47d9fb31348695a58589899aa01e7536cdc88f60b85"
+ "checksumValue": "c1518244dd5ea397e345d00e12cc45d42f43453ed208218559c981c97a0583e2"
}
],
"fileName": "Modules/expat/xmlparse.c"
@@ -1715,7 +1715,7 @@
"spdxElementId": "SPDXRef-PACKAGE-expat"
},
{
- "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-pyexpatns.h",
+ "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-refresh.sh",
"relationshipType": "CONTAINS",
"spdxElementId": "SPDXRef-PACKAGE-expat"
},
diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h
index d0d6015a66283f..523b37d8d5787d 100644
--- a/Modules/expat/expat.h
+++ b/Modules/expat/expat.h
@@ -130,7 +130,9 @@ enum XML_Error {
/* Added in 2.3.0. */
XML_ERROR_NO_BUFFER,
/* Added in 2.4.0. */
- XML_ERROR_AMPLIFICATION_LIMIT_BREACH
+ XML_ERROR_AMPLIFICATION_LIMIT_BREACH,
+ /* Added in 2.6.4. */
+ XML_ERROR_NOT_STARTED,
};
enum XML_Content_Type {
@@ -1066,7 +1068,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled);
*/
#define XML_MAJOR_VERSION 2
#define XML_MINOR_VERSION 6
-#define XML_MICRO_VERSION 3
+#define XML_MICRO_VERSION 4
#ifdef __cplusplus
}
diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h
index 12c560e14716ff..567872b09836e1 100644
--- a/Modules/expat/expat_external.h
+++ b/Modules/expat/expat_external.h
@@ -40,6 +40,10 @@
#ifndef Expat_External_INCLUDED
#define Expat_External_INCLUDED 1
+/* Namespace external symbols to allow multiple libexpat version to
+ co-exist. */
+#include "pyexpatns.h"
+
/* External API definitions */
/* Expat tries very hard to make the API boundary very specifically
@@ -64,11 +68,6 @@
compiled with the cdecl calling convention as the default since
system headers may assume the cdecl convention.
*/
-
-/* Namespace external symbols to allow multiple libexpat version to
- co-exist. */
-#include "pyexpatns.h"
-
#ifndef XMLCALL
# if defined(_MSC_VER)
# define XMLCALL __cdecl
diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh
new file mode 100755
index 00000000000000..82a9dbc23ad26b
--- /dev/null
+++ b/Modules/expat/refresh.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+#
+# Use this script to update libexpat
+
+set -e
+set -o pipefail
+
+if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
+ echo "A bash version >= 4 required. Got: $BASH_VERSION" >&2
+ exit 1
+fi
+
+# Update this when updating to a new version after verifying that the changes
+# the update brings in are good. These values are used for verifying the SBOM, too.
+expected_libexpat_tag="R_2_6_4"
+expected_libexpat_version="2.6.4"
+expected_libexpat_sha256="fd03b7172b3bd7427a3e7a812063f74754f24542429b634e0db6511b53fb2278"
+
+expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")"
+cd ${expat_dir}
+
+# Step 1: download and copy files
+curl --location "https://github.com/libexpat/libexpat/releases/download/${expected_libexpat_tag}/expat-${expected_libexpat_version}.tar.gz" > libexpat.tar.gz
+echo "${expected_libexpat_sha256} libexpat.tar.gz" | sha256sum --check
+
+# Step 2: Pull files from the libexpat distribution
+declare -a lib_files
+lib_files=(
+ ascii.h
+ asciitab.h
+ expat.h
+ expat_external.h
+ iasciitab.h
+ internal.h
+ latin1tab.h
+ nametab.h
+ siphash.h
+ utf8tab.h
+ winconfig.h
+ xmlparse.c
+ xmlrole.c
+ xmlrole.h
+ xmltok.c
+ xmltok.h
+ xmltok_impl.c
+ xmltok_impl.h
+ xmltok_ns.c
+)
+for f in "${lib_files[@]}"; do
+ tar xzvf libexpat.tar.gz "expat-${expected_libexpat_version}/lib/${f}" --strip-components 2
+done
+rm libexpat.tar.gz
+
+# Step 3: Add the namespacing include to expat_external.h
+sed -i 's/#define Expat_External_INCLUDED 1/&\n\n\/* Namespace external symbols to allow multiple libexpat version to\n co-exist. \*\/\n#include "pyexpatns.h"/' expat_external.h
+
+echo "Updated; verify all is okay using git diff and git status."
diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c
index d9285b213b38bd..a4e091e7c33c0a 100644
--- a/Modules/expat/xmlparse.c
+++ b/Modules/expat/xmlparse.c
@@ -1,4 +1,4 @@
-/* ba4cdf9bdb534f355a9def4c9e25d20ee8e72f95b0a4d930be52e563f5080196 (2.6.3+)
+/* c5625880f4bf417c1463deee4eb92d86ff413f802048621c57e25fe483eb59e4 (2.6.4+)
__ __ _
___\ \/ /_ __ __ _| |_
/ _ \\ /| '_ \ / _` | __|
@@ -40,6 +40,7 @@
Copyright (c) 2023 Owain Davies
Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow
Copyright (c) 2024 Berkay Eren Ürün
+ Copyright (c) 2024 Hanno Böck
Licensed under the MIT license:
Permission is hereby granted, free of charge, to any person obtaining
@@ -2234,6 +2235,9 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable) {
if (parser == NULL)
return XML_STATUS_ERROR;
switch (parser->m_parsingStatus.parsing) {
+ case XML_INITIALIZED:
+ parser->m_errorCode = XML_ERROR_NOT_STARTED;
+ return XML_STATUS_ERROR;
case XML_SUSPENDED:
if (resumable) {
parser->m_errorCode = XML_ERROR_SUSPENDED;
@@ -2244,7 +2248,7 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable) {
case XML_FINISHED:
parser->m_errorCode = XML_ERROR_FINISHED;
return XML_STATUS_ERROR;
- default:
+ case XML_PARSING:
if (resumable) {
#ifdef XML_DTD
if (parser->m_isParamEntity) {
@@ -2255,6 +2259,9 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable) {
parser->m_parsingStatus.parsing = XML_SUSPENDED;
} else
parser->m_parsingStatus.parsing = XML_FINISHED;
+ break;
+ default:
+ assert(0);
}
return XML_STATUS_OK;
}
@@ -2519,6 +2526,9 @@ XML_ErrorString(enum XML_Error code) {
case XML_ERROR_AMPLIFICATION_LIMIT_BREACH:
return XML_L(
"limit on input amplification factor (from DTD and entities) breached");
+ /* Added in 2.6.4. */
+ case XML_ERROR_NOT_STARTED:
+ return XML_L("parser not started");
}
return NULL;
}
@@ -7856,7 +7866,7 @@ accountingReportDiff(XML_Parser rootParser,
assert(! rootParser->m_parentParser);
fprintf(stderr,
- " (+" EXPAT_FMT_PTRDIFF_T("6") " bytes %s|%d, xmlparse.c:%d) %*s\"",
+ " (+" EXPAT_FMT_PTRDIFF_T("6") " bytes %s|%u, xmlparse.c:%d) %*s\"",
bytesMore, (account == XML_ACCOUNT_DIRECT) ? "DIR" : "EXP",
levelsAwayFromRootParser, source_line, 10, "");
@@ -7969,7 +7979,7 @@ entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity,
fprintf(
stderr,
- "expat: Entities(%p): Count %9d, depth %2d/%2d %*s%s%s; %s length %d (xmlparse.c:%d)\n",
+ "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s; %s length %d (xmlparse.c:%d)\n",
(void *)rootParser, rootParser->m_entity_stats.countEverOpened,
rootParser->m_entity_stats.currentDepth,
rootParser->m_entity_stats.maximumDepthSeen,
diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py
index 9cc89b8caee150..3299e4479e4a2e 100644
--- a/Tools/build/generate_sbom.py
+++ b/Tools/build/generate_sbom.py
@@ -59,6 +59,8 @@ class PackageFiles(typing.NamedTuple):
include=["Modules/expat/**"],
exclude=[
"Modules/expat/expat_config.h",
+ "Modules/expat/pyexpatns.h",
+ "Modules/_hacl/refresh.sh",
]
),
"macholib": PackageFiles(
@@ -221,6 +223,32 @@ def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None:
"HACL* SBOM version doesn't match value in 'Modules/_hacl/refresh.sh'"
)
+ # libexpat specifies its expected rev in a refresh script.
+ if package["name"] == "libexpat":
+ libexpat_refresh_sh = (CPYTHON_ROOT_DIR / "Modules/expat/refresh.sh").read_text()
+ libexpat_expected_version_match = re.search(
+ r"expected_libexpat_version=\"([0-9]+\.[0-9]+\.[0-9]+)\"",
+ libexpat_refresh_sh
+ )
+ libexpat_expected_sha256_match = re.search(
+ r"expected_libexpat_sha256=\"[a-f0-9]{40}\"",
+ libexpat_refresh_sh
+ )
+ libexpat_expected_version = libexpat_expected_version_match and libexpat_expected_version_match.group(1)
+ libexpat_expected_sha256 = libexpat_expected_sha256_match and libexpat_expected_sha256_match.group(1)
+
+ error_if(
+ libexpat_expected_version != version,
+ "libexpat SBOM version doesn't match value in 'Modules/expat/refresh.sh'"
+ )
+ error_if(
+ package["checksums"] != [{
+ "algorithm": "SHA256",
+ "checksumValue": libexpat_expected_sha256
+ }],
+ "libexpat SBOM checksum doesn't match value in 'Modules/expat/refresh.sh'"
+ )
+
# License must be on the approved list for SPDX.
license_concluded = package["licenseConcluded"]
error_if(
From df59b64589e32a2b2b43703cce62aad2ea0cfe1b Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
<31488909+miss-islington@users.noreply.github.com>
Date: Wed, 13 Nov 2024 23:49:47 +0100
Subject: [PATCH 198/269] [3.12] GH-126606: don't write incomplete pyc files
(GH-126627) (GH-126810)
GH-126606: don't write incomplete pyc files (GH-126627)
(cherry picked from commit c695e37a3f95c225ee08d1e882d23fa200b5ec34)
Co-authored-by: CF Bolz-Tereick
Co-authored-by: Kirill Podoprigora
Co-authored-by: Brett Cannon