diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0d7545368d7a70..8edf2d0f6cfb37 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -185,7 +185,10 @@ jobs: steps: - uses: actions/checkout@v4 - name: Install Homebrew dependencies - run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk + run: | + brew install pkg-config openssl@3.0 xz gdbm tcl-tk@8 + # Because alternate versions are not symlinked into place by default: + brew link tcl-tk@8 - name: Configure CPython run: | CPPFLAGS="-I$(brew --prefix gdbm)/include -I$(brew --prefix xz)/include" \ @@ -193,7 +196,9 @@ jobs: ./configure \ --with-pydebug \ --prefix=/opt/python-dev \ - --with-openssl="$(brew --prefix openssl@3.0)" + --with-openssl="$(brew --prefix openssl@3.0)" \ + --with-dbmliborder=gdbm:ndbm + # (--with-dbmliborder needed for homebrew's gdbm 1.24: see gh-89452) - name: Build CPython run: make -j4 - name: Display build info diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 6d50b2fda26771..161e088a02b55a 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 10 -#define PY_MICRO_VERSION 15 +#define PY_MICRO_VERSION 16 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.10.15" +#define PY_VERSION "3.10.16" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 94b1761b5f7bf8..ac284ca125bead 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -310,7 +310,7 @@ def collapse_addresses(addresses): [IPv4Network('192.0.2.0/24')] Args: - addresses: An iterator of IPv4Network or IPv6Network objects. + addresses: An iterable of IPv4Network or IPv6Network objects. Returns: An iterator of the collapsed IPv(4|6)Network objects. @@ -1840,9 +1840,6 @@ def _string_from_ip_int(cls, ip_int=None): def _explode_shorthand_ip_string(self): """Expand a shortened IPv6 address. - Args: - ip_str: A string, the IPv6 address. - Returns: A string, the expanded IPv6 address. @@ -1986,6 +1983,9 @@ def is_multicast(self): See RFC 2373 2.7 for details. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_multicast return self in self._constants._multicast_network @property @@ -1997,6 +1997,9 @@ def is_reserved(self): reserved IPv6 Network ranges. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_reserved return any(self in x for x in self._constants._reserved_networks) @property @@ -2007,6 +2010,9 @@ def is_link_local(self): A boolean, True if the address is reserved per RFC 4291. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_link_local return self in self._constants._linklocal_network @property @@ -2063,6 +2069,9 @@ def is_global(self): ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` IPv4 range where they are both ``False``. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_global return not self.is_private @property @@ -2074,6 +2083,9 @@ def is_unspecified(self): RFC 2373 2.5.2. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_unspecified return self._ip == 0 @property @@ -2085,6 +2097,9 @@ def is_loopback(self): RFC 2373 2.5.3. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_loopback return self._ip == 1 @property @@ -2201,7 +2216,7 @@ def is_unspecified(self): @property def is_loopback(self): - return self._ip == 1 and self.network.is_loopback + return super().is_loopback and self.network.is_loopback class IPv6Network(_BaseV6, _BaseNetwork): @@ -2314,6 +2329,8 @@ class _IPv6Constants: IPv6Network('2001:db8::/32'), # IANA says N/A, let's consider it not globally reachable to be safe IPv6Network('2002::/16'), + # RFC 9637: https://www.rfc-editor.org/rfc/rfc9637.html#section-6-2.2 + IPv6Network('3fff::/20'), IPv6Network('fc00::/7'), IPv6Network('fe80::/10'), ] diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index ccb2a7b1b42829..8db507a61f2a66 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Sat Sep 7 01:19:53 2024 +# Autogenerated by Sphinx on Tue Dec 3 12:26:47 2024 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index ccacfd606d186f..86bcafcf0e3cb3 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -739,6 +739,9 @@ def test_nonexisting_script(self): self.assertNotEqual(proc.returncode, 0) @unittest.skipUnless(os.path.exists('/dev/fd/0'), 'requires /dev/fd platform') + @unittest.skipIf(sys.platform.startswith("freebsd") and + os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, + "Requires fdescfs mounted on /dev/fd on FreeBSD") def test_script_as_dev_fd(self): # GH-87235: On macOS passing a non-trivial script to /dev/fd/N can cause # problems because all open /dev/fd/N file descriptors share the same diff --git a/Lib/test/test_email/test_utils.py b/Lib/test/test_email/test_utils.py index 78afb358035e81..51a5bac6dfaa8c 100644 --- a/Lib/test/test_email/test_utils.py +++ b/Lib/test/test_email/test_utils.py @@ -152,6 +152,8 @@ def test_localtime_epoch_notz_daylight_false(self): def test_variable_tzname(self): t0 = datetime.datetime(1984, 1, 1, tzinfo=datetime.timezone.utc) t1 = utils.localtime(t0) + if t1.tzname() == 'Europe': + self.skipTest("Can't find a Kyiv timezone database") self.assertEqual(t1.tzname(), 'MSK') t0 = datetime.datetime(1994, 1, 1, tzinfo=datetime.timezone.utc) t1 = utils.localtime(t0) diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 86f3bb504a23ba..47ad8f44a51a80 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -2415,6 +2415,30 @@ def testIpv4Mapped(self): self.assertEqual(ipaddress.ip_address('::ffff:c0a8:101').ipv4_mapped, ipaddress.ip_address('192.168.1.1')) + def testIpv4MappedProperties(self): + # Test that an IPv4 mapped IPv6 address has + # the same properties as an IPv4 address. + for addr4 in ( + "178.62.3.251", # global + "169.254.169.254", # link local + "127.0.0.1", # loopback + "224.0.0.1", # multicast + "192.168.0.1", # private + "0.0.0.0", # unspecified + "100.64.0.1", # public and not global + ): + with self.subTest(addr4): + ipv4 = ipaddress.IPv4Address(addr4) + ipv6 = ipaddress.IPv6Address(f"::ffff:{addr4}") + + self.assertEqual(ipv4.is_global, ipv6.is_global) + self.assertEqual(ipv4.is_private, ipv6.is_private) + self.assertEqual(ipv4.is_reserved, ipv6.is_reserved) + self.assertEqual(ipv4.is_multicast, ipv6.is_multicast) + self.assertEqual(ipv4.is_unspecified, ipv6.is_unspecified) + self.assertEqual(ipv4.is_link_local, ipv6.is_link_local) + self.assertEqual(ipv4.is_loopback, ipv6.is_loopback) + def testIpv4MappedPrivateCheck(self): self.assertEqual( True, ipaddress.ip_address('::ffff:192.168.1.1').is_private) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 4c8f55d3e1352b..0050298b416495 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -6409,12 +6409,16 @@ def test_sha256(self): self.assertEqual(op.recv(512), expected) def test_hmac_sha1(self): - expected = bytes.fromhex("effcdf6ae5eb2fa2d27416d5f184df9c259a7c79") + # gh-109396: In FIPS mode, Linux 6.5 requires a key + # of at least 112 bits. Use a key of 152 bits. + key = b"Python loves AF_ALG" + data = b"what do ya want for nothing?" + expected = bytes.fromhex("193dbb43c6297b47ea6277ec0ce67119a3f3aa66") with self.create_alg('hash', 'hmac(sha1)') as algo: - algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, b"Jefe") + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, key) op, _ = algo.accept() with op: - op.sendall(b"what do ya want for nothing?") + op.sendall(data) self.assertEqual(op.recv(512), expected) # Although it should work with 3.19 and newer the test blocks on diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 139ef41e57370b..0d3afe612a3afe 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -2795,7 +2795,7 @@ def test_close_fds(self): @unittest.skipIf(sys.platform.startswith("freebsd") and os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, - "Requires fdescfs mounted on /dev/fd on FreeBSD.") + "Requires fdescfs mounted on /dev/fd on FreeBSD") def test_close_fds_when_max_fd_is_lowered(self): """Confirm that issue21618 is fixed (may fail under valgrind).""" fd_status = support.findfile("fd_status.py", subdir="subprocessdata") diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 007c3e94acb876..63ab254c2a3655 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1,3 +1,4 @@ +import errno import sys import os import io @@ -3691,34 +3692,55 @@ def test_modes(self): arc.add('read_group_only', mode='?---r-----') arc.add('no_bits', mode='?---------') arc.add('dir/', mode='?---rwsrwt') + arc.add('dir_all_bits/', mode='?rwsrwsrwt') - # On some systems, setting the sticky bit is a no-op. - # Check if that's the case. + # On some systems, setting the uid, gid, and/or sticky bit is a no-ops. + # Check which bits we can set, so we can compare tarfile machinery to + # a simple chmod. tmp_filename = os.path.join(TEMPDIR, "tmp.file") with open(tmp_filename, 'w'): pass - os.chmod(tmp_filename, os.stat(tmp_filename).st_mode | stat.S_ISVTX) - have_sticky_files = (os.stat(tmp_filename).st_mode & stat.S_ISVTX) - os.unlink(tmp_filename) + try: + new_mode = (os.stat(tmp_filename).st_mode + | stat.S_ISVTX | stat.S_ISGID | stat.S_ISUID) + try: + os.chmod(tmp_filename, new_mode) + except OSError as exc: + if exc.errno == getattr(errno, "EFTYPE", 0): + # gh-108948: On FreeBSD, regular users cannot set + # the sticky bit. + self.skipTest("chmod() failed with EFTYPE: " + "regular users cannot set sticky bit") + else: + raise + + got_mode = os.stat(tmp_filename).st_mode + _t_file = 't' if (got_mode & stat.S_ISVTX) else 'x' + _suid_file = 's' if (got_mode & stat.S_ISUID) else 'x' + _sgid_file = 's' if (got_mode & stat.S_ISGID) else 'x' + finally: + os.unlink(tmp_filename) os.mkdir(tmp_filename) - os.chmod(tmp_filename, os.stat(tmp_filename).st_mode | stat.S_ISVTX) - have_sticky_dirs = (os.stat(tmp_filename).st_mode & stat.S_ISVTX) + new_mode = (os.stat(tmp_filename).st_mode + | stat.S_ISVTX | stat.S_ISGID | stat.S_ISUID) + os.chmod(tmp_filename, new_mode) + got_mode = os.stat(tmp_filename).st_mode + _t_dir = 't' if (got_mode & stat.S_ISVTX) else 'x' + _suid_dir = 's' if (got_mode & stat.S_ISUID) else 'x' + _sgid_dir = 's' if (got_mode & stat.S_ISGID) else 'x' os.rmdir(tmp_filename) with self.check_context(arc.open(), 'fully_trusted'): - if have_sticky_files: - self.expect_file('all_bits', mode='?rwsrwsrwt') - else: - self.expect_file('all_bits', mode='?rwsrwsrwx') + self.expect_file('all_bits', + mode=f'?rw{_suid_file}rw{_sgid_file}rw{_t_file}') self.expect_file('perm_bits', mode='?rwxrwxrwx') self.expect_file('exec_group_other', mode='?rw-rwxrwx') self.expect_file('read_group_only', mode='?---r-----') self.expect_file('no_bits', mode='?---------') - if have_sticky_dirs: - self.expect_file('dir/', mode='?---rwsrwt') - else: - self.expect_file('dir/', mode='?---rwsrwx') + self.expect_file('dir/', mode=f'?---rw{_sgid_dir}rw{_t_dir}') + self.expect_file('dir_all_bits/', + mode=f'?rw{_suid_dir}rw{_sgid_dir}rw{_t_dir}') with self.check_context(arc.open(), 'tar'): self.expect_file('all_bits', mode='?rwxr-xr-x') @@ -3727,6 +3749,7 @@ def test_modes(self): self.expect_file('read_group_only', mode='?---r-----') self.expect_file('no_bits', mode='?---------') self.expect_file('dir/', mode='?---r-xr-x') + self.expect_file('dir_all_bits/', mode='?rwxr-xr-x') with self.check_context(arc.open(), 'data'): normal_dir_mode = stat.filemode(stat.S_IMODE( @@ -3737,6 +3760,7 @@ def test_modes(self): self.expect_file('read_group_only', mode='?rw-r-----') self.expect_file('no_bits', mode='?rw-------') self.expect_file('dir/', mode=normal_dir_mode) + self.expect_file('dir_all_bits/', mode=normal_dir_mode) def test_pipe(self): # Test handling of a special file diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 5e08aa1bbad89d..f2ffc452e566c0 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -1138,6 +1138,32 @@ def test_issue14072(self): self.assertEqual(p2.scheme, 'tel') self.assertEqual(p2.path, '+31641044153') + def test_invalid_bracketed_hosts(self): + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[192.0.2.146]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[important.com:8000]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v123r.IP]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v12ae]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v.IP]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v123.]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af::2309::fae7:1234]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af:2309::fae7:1234:2342:438e:192.0.2.146]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@]v6a.ip[/Path') + + def test_splitting_bracketed_hosts(self): + p1 = urllib.parse.urlsplit('scheme://user@[v6a.ip]/path?query') + self.assertEqual(p1.hostname, 'v6a.ip') + self.assertEqual(p1.username, 'user') + self.assertEqual(p1.path, '/path') + p2 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7%test]/path?query') + self.assertEqual(p2.hostname, '0439:23af:2309::fae7%test') + self.assertEqual(p2.username, 'user') + self.assertEqual(p2.path, '/path') + p3 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7:1234:192.0.2.146%test]/path?query') + self.assertEqual(p3.hostname, '0439:23af:2309::fae7:1234:192.0.2.146%test') + self.assertEqual(p3.username, 'user') + self.assertEqual(p3.path, '/path') + def test_port_casting_failure_message(self): message = "Port could not be cast to integer value as 'oracle'" p1 = urllib.parse.urlparse('http://Server=sde; Service=sde:oracle') diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index fea986b9d86840..d8ee66e4990f53 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -15,6 +15,7 @@ import subprocess import sys import tempfile +import shlex from test.support import (captured_stdout, captured_stderr, requires_zlib, skip_if_broken_multiprocessing_synchronize) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) @@ -85,6 +86,10 @@ def get_text_file_contents(self, *args, encoding='utf-8'): result = f.read() return result + def assertEndsWith(self, string, tail): + if not string.endswith(tail): + self.fail(f"String {string!r} does not end with {tail!r}") + class BasicTest(BaseTest): """Test venv module functionality.""" @@ -342,6 +347,82 @@ def test_executable_symlinks(self): 'import sys; print(sys.executable)']) self.assertEqual(out.strip(), envpy.encode()) + # gh-124651: test quoted strings + @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + def test_special_chars_bash(self): + """ + Test that the template strings are quoted properly (bash) + """ + rmtree(self.env_dir) + bash = shutil.which('bash') + if bash is None: + self.skipTest('bash required for this test') + env_name = '"\';&&$e|\'"' + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate') + test_script = os.path.join(self.env_dir, 'test_special_chars.sh') + with open(test_script, "w") as f: + f.write(f'source {shlex.quote(activate)}\n' + 'python -c \'import sys; print(sys.executable)\'\n' + 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n' + 'deactivate\n') + out, err = check_output([bash, test_script]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + + # gh-124651: test quoted strings + @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + def test_special_chars_csh(self): + """ + Test that the template strings are quoted properly (csh) + """ + rmtree(self.env_dir) + csh = shutil.which('tcsh') or shutil.which('csh') + if csh is None: + self.skipTest('csh required for this test') + env_name = '"\';&&$e|\'"' + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.csh') + test_script = os.path.join(self.env_dir, 'test_special_chars.csh') + with open(test_script, "w") as f: + f.write(f'source {shlex.quote(activate)}\n' + 'python -c \'import sys; print(sys.executable)\'\n' + 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n' + 'deactivate\n') + out, err = check_output([csh, test_script]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + + # gh-124651: test quoted strings on Windows + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_special_chars_windows(self): + """ + Test that the template strings are quoted properly on Windows + """ + rmtree(self.env_dir) + env_name = "'&&^$e" + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.bat') + test_batch = os.path.join(self.env_dir, 'test_special_chars.bat') + with open(test_batch, "w") as f: + f.write('@echo off\n' + f'"{activate}" & ' + f'{self.exe} -c "import sys; print(sys.executable)" & ' + f'{self.exe} -c "import os; print(os.environ[\'VIRTUAL_ENV\'])" & ' + 'deactivate') + out, err = check_output([test_batch]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') def test_unicode_in_batch_file(self): """ diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 208bd1e9e9fc7f..c945ad12944357 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -3,6 +3,7 @@ from test.support import import_helper import binascii import copy +import os import pickle import random import sys @@ -32,6 +33,16 @@ def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION): ZLIB_RUNTIME_VERSION_TUPLE = _zlib_runtime_version_tuple() +# bpo-46623: When a hardware accelerator is used (currently only on s390x), +# using different ways to compress data with zlib can produce different +# compressed data. +# +# To simplify the skip condition, make the assumption that s390x always has an +# accelerator, and nothing else has it. +# Windows doesn't have os.uname() but it doesn't support s390x. +HW_ACCELERATED = hasattr(os, 'uname') and os.uname().machine == 's390x' + + class VersionTestCase(unittest.TestCase): def test_library_version(self): @@ -199,7 +210,10 @@ def test_speech128(self): # compress more data data = HAMLET_SCENE * 128 x = zlib.compress(data) - self.assertEqual(zlib.compress(bytearray(data)), x) + # With hardware acceleration, the compressed bytes + # might not be identical. + if not HW_ACCELERATED: + self.assertEqual(zlib.compress(bytearray(data)), x) for ob in x, bytearray(x): self.assertEqual(zlib.decompress(ob), data) @@ -256,7 +270,10 @@ def test_pair(self): x1 = co.compress(data) x2 = co.flush() self.assertRaises(zlib.error, co.flush) # second flush should not work - self.assertEqual(x1 + x2, datazip) + # With hardware acceleration, the compressed bytes might not + # be identical. + if not HW_ACCELERATED: + self.assertEqual(x1 + x2, datazip) for v1, v2 in ((x1, x2), (bytearray(x1), bytearray(x2))): dco = zlib.decompressobj() y1 = dco.decompress(v1 + v2) diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 44806e67a8fc01..07e3248504b988 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -36,6 +36,7 @@ import types import collections import warnings +import ipaddress __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag", "urlsplit", "urlunsplit", "urlencode", "parse_qs", @@ -441,6 +442,17 @@ def _checknetloc(netloc): raise ValueError("netloc '" + netloc + "' contains invalid " + "characters under NFKC normalization") +# Valid bracketed hosts are defined in +# https://www.rfc-editor.org/rfc/rfc3986#page-49 and https://url.spec.whatwg.org/ +def _check_bracketed_host(hostname): + if hostname.startswith('v'): + if not re.match(r"\Av[a-fA-F0-9]+\..+\Z", hostname): + raise ValueError(f"IPvFuture address is invalid") + else: + ip = ipaddress.ip_address(hostname) # Throws Value Error if not IPv6 or IPv4 + if isinstance(ip, ipaddress.IPv4Address): + raise ValueError(f"An IPv4 address cannot be in brackets") + def urlsplit(url, scheme='', allow_fragments=True): """Parse a URL into 5 components: :///?# @@ -487,12 +499,14 @@ def urlsplit(url, scheme='', allow_fragments=True): break else: scheme, url = url[:i].lower(), url[i+1:] - if url[:2] == '//': netloc, url = _splitnetloc(url, 2) if (('[' in netloc and ']' not in netloc) or (']' in netloc and '[' not in netloc)): raise ValueError("Invalid IPv6 URL") + if '[' in netloc and ']' in netloc: + bracketed_host = netloc.partition('[')[2].partition(']')[0] + _check_bracketed_host(bracketed_host) if allow_fragments and '#' in url: url, fragment = url.split('#', 1) if '?' in url: diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 69155e5cc2ffba..fe468a9d8984ff 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -11,6 +11,7 @@ import sys import sysconfig import types +import shlex CORE_VENV_DEPS = ('pip', 'setuptools') @@ -364,11 +365,41 @@ def replace_variables(self, text, context): :param context: The information for the environment creation request being processed. """ - text = text.replace('__VENV_DIR__', context.env_dir) - text = text.replace('__VENV_NAME__', context.env_name) - text = text.replace('__VENV_PROMPT__', context.prompt) - text = text.replace('__VENV_BIN_NAME__', context.bin_name) - text = text.replace('__VENV_PYTHON__', context.env_exe) + replacements = { + '__VENV_DIR__': context.env_dir, + '__VENV_NAME__': context.env_name, + '__VENV_PROMPT__': context.prompt, + '__VENV_BIN_NAME__': context.bin_name, + '__VENV_PYTHON__': context.env_exe, + } + + def quote_ps1(s): + """ + This should satisfy PowerShell quoting rules [1], unless the quoted + string is passed directly to Windows native commands [2]. + [1]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_quoting_rules + [2]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_parsing#passing-arguments-that-contain-quote-characters + """ + s = s.replace("'", "''") + return f"'{s}'" + + def quote_bat(s): + return s + + # gh-124651: need to quote the template strings properly + quote = shlex.quote + script_path = context.script_path + if script_path.endswith('.ps1'): + quote = quote_ps1 + elif script_path.endswith('.bat'): + quote = quote_bat + else: + # fallbacks to POSIX shell compliant quote + quote = shlex.quote + + replacements = {key: quote(s) for key, s in replacements.items()} + for key, quoted in replacements.items(): + text = text.replace(key, quoted) return text def install_scripts(self, context, path): @@ -408,6 +439,7 @@ def install_scripts(self, context, path): with open(srcfile, 'rb') as f: data = f.read() if not srcfile.endswith(('.exe', '.pdb')): + context.script_path = srcfile try: data = data.decode('utf-8') data = self.replace_variables(data, context) diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 6fbc2b8801da04..104399d55f8cc0 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -38,11 +38,11 @@ deactivate () { # unset irrelevant variables deactivate nondestructive -VIRTUAL_ENV="__VENV_DIR__" +VIRTUAL_ENV=__VENV_DIR__ export VIRTUAL_ENV _OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" +PATH="$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH" export PATH # unset PYTHONHOME if set @@ -55,9 +55,9 @@ fi if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then _OLD_VIRTUAL_PS1="${PS1:-}" - PS1="__VENV_PROMPT__${PS1:-}" + PS1=__VENV_PROMPT__"${PS1:-}" export PS1 - VIRTUAL_ENV_PROMPT="__VENV_PROMPT__" + VIRTUAL_ENV_PROMPT=__VENV_PROMPT__ export VIRTUAL_ENV_PROMPT fi diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat index c1c3c82ee37f10..715b21b13fbe35 100644 --- a/Lib/venv/scripts/nt/activate.bat +++ b/Lib/venv/scripts/nt/activate.bat @@ -8,7 +8,7 @@ if defined _OLD_CODEPAGE ( "%SystemRoot%\System32\chcp.com" 65001 > nul ) -set VIRTUAL_ENV=__VENV_DIR__ +set "VIRTUAL_ENV=__VENV_DIR__" if not defined PROMPT set PROMPT=$P$G @@ -24,8 +24,8 @@ set PYTHONHOME= if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH% if not defined _OLD_VIRTUAL_PATH set _OLD_VIRTUAL_PATH=%PATH% -set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH% -set VIRTUAL_ENV_PROMPT=__VENV_PROMPT__ +set "PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%" +set "VIRTUAL_ENV_PROMPT=__VENV_PROMPT__" :END if defined _OLD_CODEPAGE ( diff --git a/Lib/venv/scripts/posix/activate.csh b/Lib/venv/scripts/posix/activate.csh index d6f697c55ed81c..c47702127eff71 100644 --- a/Lib/venv/scripts/posix/activate.csh +++ b/Lib/venv/scripts/posix/activate.csh @@ -8,17 +8,17 @@ alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PA # Unset irrelevant variables. deactivate nondestructive -setenv VIRTUAL_ENV "__VENV_DIR__" +setenv VIRTUAL_ENV __VENV_DIR__ set _OLD_VIRTUAL_PATH="$PATH" -setenv PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" +setenv PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH" set _OLD_VIRTUAL_PROMPT="$prompt" if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - set prompt = "__VENV_PROMPT__$prompt" - setenv VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" + set prompt = __VENV_PROMPT__"$prompt" + setenv VIRTUAL_ENV_PROMPT __VENV_PROMPT__ endif alias pydoc python -m pydoc diff --git a/Lib/venv/scripts/posix/activate.fish b/Lib/venv/scripts/posix/activate.fish index 9aa4446005f4d8..dc3a6c88270c18 100644 --- a/Lib/venv/scripts/posix/activate.fish +++ b/Lib/venv/scripts/posix/activate.fish @@ -33,10 +33,10 @@ end # Unset irrelevant variables. deactivate nondestructive -set -gx VIRTUAL_ENV "__VENV_DIR__" +set -gx VIRTUAL_ENV __VENV_DIR__ set -gx _OLD_VIRTUAL_PATH $PATH -set -gx PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__" $PATH +set -gx PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__ $PATH # Unset PYTHONHOME if set. if set -q PYTHONHOME @@ -56,7 +56,7 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" set -l old_status $status # Output the venv prompt; color taken from the blue of the Python logo. - printf "%s%s%s" (set_color 4B8BBE) "__VENV_PROMPT__" (set_color normal) + printf "%s%s%s" (set_color 4B8BBE) __VENV_PROMPT__ (set_color normal) # Restore the return status of the previous command. echo "exit $old_status" | . @@ -65,5 +65,5 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" end set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" - set -gx VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" + set -gx VIRTUAL_ENV_PROMPT __VENV_PROMPT__ end diff --git a/Misc/NEWS.d/3.10.16.rst b/Misc/NEWS.d/3.10.16.rst new file mode 100644 index 00000000000000..edc30ab7d430c4 --- /dev/null +++ b/Misc/NEWS.d/3.10.16.rst @@ -0,0 +1,60 @@ +.. date: 2024-10-07-14-13-38 +.. gh-issue: 125041 +.. nonce: PKLWDf +.. release date: 2024-12-03 +.. section: Tests + +Re-enable skipped tests for :mod:`zlib` on the s390x architecture: only skip +checks of the compressed bytes, which can be different between zlib's +software implementation and the hardware-accelerated implementation. + +.. + +.. date: 2023-09-14-22-58-47 +.. gh-issue: 109396 +.. nonce: J1a4jR +.. section: Tests + +Fix ``test_socket.test_hmac_sha1()`` in FIPS mode. Use a longer key: FIPS +mode requires at least of at least 112 bits. The previous key was only 32 +bits. Patch by Victor Stinner. + +.. + +.. date: 2024-11-13-11-09-12 +.. gh-issue: 126623 +.. nonce: TO7NnR +.. section: Security + +Upgrade libexpat to 2.6.4 + +.. + +.. date: 2024-08-07-10-42-13 +.. gh-issue: 122792 +.. nonce: oiTMo9 +.. section: Security + +Changed IPv4-mapped ``ipaddress.IPv6Address`` to consistently use the mapped +IPv4 address value for deciding properties. Properties which have their +behavior fixed are ``is_multicast``, ``is_reserved``, ``is_link_local``, +``is_global``, and ``is_unspecified``. + +.. + +.. date: 2024-09-28-02-03-04 +.. gh-issue: 124651 +.. nonce: bLBGtH +.. section: Library + +Properly quote template strings in :mod:`venv` activation scripts. + +.. + +.. date: 2023-04-26-09-54-25 +.. gh-issue: 103848 +.. nonce: aDSnpR +.. section: Library + +Add checks to ensure that ``[`` bracketed ``]`` hosts found by +:func:`urllib.parse.urlsplit` are of IPv6 or IPvFuture format. diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index d0d6015a66283f..523b37d8d5787d 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -130,7 +130,9 @@ enum XML_Error { /* Added in 2.3.0. */ XML_ERROR_NO_BUFFER, /* Added in 2.4.0. */ - XML_ERROR_AMPLIFICATION_LIMIT_BREACH + XML_ERROR_AMPLIFICATION_LIMIT_BREACH, + /* Added in 2.6.4. */ + XML_ERROR_NOT_STARTED, }; enum XML_Content_Type { @@ -1066,7 +1068,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ #define XML_MAJOR_VERSION 2 #define XML_MINOR_VERSION 6 -#define XML_MICRO_VERSION 3 +#define XML_MICRO_VERSION 4 #ifdef __cplusplus } diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index 12c560e14716ff..567872b09836e1 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -40,6 +40,10 @@ #ifndef Expat_External_INCLUDED #define Expat_External_INCLUDED 1 +/* Namespace external symbols to allow multiple libexpat version to + co-exist. */ +#include "pyexpatns.h" + /* External API definitions */ /* Expat tries very hard to make the API boundary very specifically @@ -64,11 +68,6 @@ compiled with the cdecl calling convention as the default since system headers may assume the cdecl convention. */ - -/* Namespace external symbols to allow multiple libexpat version to - co-exist. */ -#include "pyexpatns.h" - #ifndef XMLCALL # if defined(_MSC_VER) # define XMLCALL __cdecl diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh new file mode 100755 index 00000000000000..82a9dbc23ad26b --- /dev/null +++ b/Modules/expat/refresh.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# Use this script to update libexpat + +set -e +set -o pipefail + +if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then + echo "A bash version >= 4 required. Got: $BASH_VERSION" >&2 + exit 1 +fi + +# Update this when updating to a new version after verifying that the changes +# the update brings in are good. These values are used for verifying the SBOM, too. +expected_libexpat_tag="R_2_6_4" +expected_libexpat_version="2.6.4" +expected_libexpat_sha256="fd03b7172b3bd7427a3e7a812063f74754f24542429b634e0db6511b53fb2278" + +expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" +cd ${expat_dir} + +# Step 1: download and copy files +curl --location "https://github.com/libexpat/libexpat/releases/download/${expected_libexpat_tag}/expat-${expected_libexpat_version}.tar.gz" > libexpat.tar.gz +echo "${expected_libexpat_sha256} libexpat.tar.gz" | sha256sum --check + +# Step 2: Pull files from the libexpat distribution +declare -a lib_files +lib_files=( + ascii.h + asciitab.h + expat.h + expat_external.h + iasciitab.h + internal.h + latin1tab.h + nametab.h + siphash.h + utf8tab.h + winconfig.h + xmlparse.c + xmlrole.c + xmlrole.h + xmltok.c + xmltok.h + xmltok_impl.c + xmltok_impl.h + xmltok_ns.c +) +for f in "${lib_files[@]}"; do + tar xzvf libexpat.tar.gz "expat-${expected_libexpat_version}/lib/${f}" --strip-components 2 +done +rm libexpat.tar.gz + +# Step 3: Add the namespacing include to expat_external.h +sed -i 's/#define Expat_External_INCLUDED 1/&\n\n\/* Namespace external symbols to allow multiple libexpat version to\n co-exist. \*\/\n#include "pyexpatns.h"/' expat_external.h + +echo "Updated; verify all is okay using git diff and git status." diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index d9285b213b38bd..a4e091e7c33c0a 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* ba4cdf9bdb534f355a9def4c9e25d20ee8e72f95b0a4d930be52e563f5080196 (2.6.3+) +/* c5625880f4bf417c1463deee4eb92d86ff413f802048621c57e25fe483eb59e4 (2.6.4+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -40,6 +40,7 @@ Copyright (c) 2023 Owain Davies Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow Copyright (c) 2024 Berkay Eren Ürün + Copyright (c) 2024 Hanno Böck Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -2234,6 +2235,9 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable) { if (parser == NULL) return XML_STATUS_ERROR; switch (parser->m_parsingStatus.parsing) { + case XML_INITIALIZED: + parser->m_errorCode = XML_ERROR_NOT_STARTED; + return XML_STATUS_ERROR; case XML_SUSPENDED: if (resumable) { parser->m_errorCode = XML_ERROR_SUSPENDED; @@ -2244,7 +2248,7 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable) { case XML_FINISHED: parser->m_errorCode = XML_ERROR_FINISHED; return XML_STATUS_ERROR; - default: + case XML_PARSING: if (resumable) { #ifdef XML_DTD if (parser->m_isParamEntity) { @@ -2255,6 +2259,9 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable) { parser->m_parsingStatus.parsing = XML_SUSPENDED; } else parser->m_parsingStatus.parsing = XML_FINISHED; + break; + default: + assert(0); } return XML_STATUS_OK; } @@ -2519,6 +2526,9 @@ XML_ErrorString(enum XML_Error code) { case XML_ERROR_AMPLIFICATION_LIMIT_BREACH: return XML_L( "limit on input amplification factor (from DTD and entities) breached"); + /* Added in 2.6.4. */ + case XML_ERROR_NOT_STARTED: + return XML_L("parser not started"); } return NULL; } @@ -7856,7 +7866,7 @@ accountingReportDiff(XML_Parser rootParser, assert(! rootParser->m_parentParser); fprintf(stderr, - " (+" EXPAT_FMT_PTRDIFF_T("6") " bytes %s|%d, xmlparse.c:%d) %*s\"", + " (+" EXPAT_FMT_PTRDIFF_T("6") " bytes %s|%u, xmlparse.c:%d) %*s\"", bytesMore, (account == XML_ACCOUNT_DIRECT) ? "DIR" : "EXP", levelsAwayFromRootParser, source_line, 10, ""); @@ -7969,7 +7979,7 @@ entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, fprintf( stderr, - "expat: Entities(%p): Count %9d, depth %2d/%2d %*s%s%s; %s length %d (xmlparse.c:%d)\n", + "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s; %s length %d (xmlparse.c:%d)\n", (void *)rootParser, rootParser->m_entity_stats.countEverOpened, rootParser->m_entity_stats.currentDepth, rootParser->m_entity_stats.maximumDepthSeen, diff --git a/README.rst b/README.rst index 884b077edc2ecd..1fd7ca3e1d2865 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.10.15 +This is Python version 3.10.16 ============================== .. image:: https://travis-ci.com/python/cpython.svg?branch=master diff --git a/Tools/peg_generator/pegen/grammar_parser.py b/Tools/peg_generator/pegen/grammar_parser.py index 892df5cf8ccca1..de735649d46159 100644 --- a/Tools/peg_generator/pegen/grammar_parser.py +++ b/Tools/peg_generator/pegen/grammar_parser.py @@ -420,7 +420,7 @@ def named_item(self) -> Optional[NamedItem]: and (item := self.item()) ): - return NamedItem ( name . string , item , f"{type.string}*" ) + return NamedItem ( name . string , item , type . string + "*" ) self.reset(mark) if cut: return None cut = False diff --git a/Tools/peg_generator/pegen/metagrammar.gram b/Tools/peg_generator/pegen/metagrammar.gram index bb4355fd189e05..77695d5e0a91a0 100644 --- a/Tools/peg_generator/pegen/metagrammar.gram +++ b/Tools/peg_generator/pegen/metagrammar.gram @@ -84,7 +84,7 @@ items[NamedItemList]: | named_item { [named_item] } named_item[NamedItem]: - | NAME '[' type=NAME '*' ']' '=' ~ item {NamedItem(name.string, item, f"{type.string}*")} + | NAME '[' type=NAME '*' ']' '=' ~ item {NamedItem(name.string, item, type.string+"*")} | NAME '[' type=NAME ']' '=' ~ item {NamedItem(name.string, item, type.string)} | NAME '=' ~ item {NamedItem(name.string, item)} | item {NamedItem(None, item)}