From 88f46e47b83a980a1445086f8abff5e689e9c297 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 1 Sep 2024 16:00:05 +0900 Subject: [PATCH 1/9] gh-121313: multiprocessing: change connection buffer size --- Lib/multiprocessing/connection.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index d84b52fe6d4f88..ede5596e4fcf2e 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -13,7 +13,6 @@ import io import itertools import os -import stat import sys import socket import struct @@ -40,7 +39,8 @@ # # -BUFSIZE = 8192 +# Default PIPE buffer size of many posix platforms. +BUFSIZE = 64*1024 # A very generous timeout when it comes to local connections... CONNECTION_TIMEOUT = 20. @@ -361,11 +361,6 @@ def _get_more_data(self, ov, maxsize): f.write(ov.getbuffer()) return f -""" -The default size of a pipe on Linux systems is 16 times the base page size: -https://man7.org/linux/man-pages/man7/pipe.7.html -""" -PAGES_PER_PIPE = 16 class Connection(_ConnectionBase): """ @@ -384,8 +379,6 @@ def _close(self, _close=os.close): _close(self._handle) _write = os.write _read = os.read - _base_page_size = os.sysconf(os.sysconf_names['SC_PAGESIZE']) - _default_pipe_size = _base_page_size * PAGES_PER_PIPE def _send(self, buf, write=_write): remaining = len(buf) @@ -400,13 +393,8 @@ def _recv(self, size, read=_read): buf = io.BytesIO() handle = self._handle remaining = size - is_pipe = False - if size > self._default_pipe_size > 0: - mode = os.fstat(handle).st_mode - is_pipe = stat.S_ISFIFO(mode) - limit = self._default_pipe_size if is_pipe else remaining while remaining > 0: - to_read = min(limit, remaining) + to_read = min(BUFSIZE, remaining) chunk = read(handle, to_read) n = len(chunk) if n == 0: From b24b46f0f36581c8dce82b19ea7f57231ddd9c82 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 1 Sep 2024 07:17:30 +0000 Subject: [PATCH 2/9] update changelog --- .../next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst b/Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst index 06abce9e67da44..7c3f74a5342ef6 100644 --- a/Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst +++ b/Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst @@ -1 +1 @@ -Limit reading size in multiprocessing connection._recv for pipes to default pipe size of 16 times base page size, in order to avoid memory overallocation and unnecessary memory management system calls. +Limit reading size in multiprocessing ``connection._recv`` to 64KiB, in order to avoid memory overallocation and unnecessary memory management system calls. From 0e0933d3ec691219a56e6467431a2ae1f989c1aa Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 1 Sep 2024 17:59:42 +0900 Subject: [PATCH 3/9] update comment --- Lib/multiprocessing/connection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index ede5596e4fcf2e..499ecbc1463549 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -39,8 +39,9 @@ # # -# Default PIPE buffer size of many posix platforms. +# 64KiB is the default PIPE buffer size of most posix platforms. BUFSIZE = 64*1024 + # A very generous timeout when it comes to local connections... CONNECTION_TIMEOUT = 20. From 205ec7b814f90fbde594147076d83712e7b3d4fd Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 2 Sep 2024 13:28:49 +0900 Subject: [PATCH 4/9] remove unnecessary code --- Lib/multiprocessing/connection.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 499ecbc1463549..d54251d90dd36a 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -374,7 +374,6 @@ def _close(self, _close=_multiprocessing.closesocket): _close(self._handle) _write = _multiprocessing.send _read = _multiprocessing.recv - _default_pipe_size = 0 else: def _close(self, _close=os.close): _close(self._handle) From bfbf9bdf305d563caf2c750489774fd26222b14a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 2 Sep 2024 19:28:03 +0900 Subject: [PATCH 5/9] Update Lib/multiprocessing/connection.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/multiprocessing/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index d54251d90dd36a..b2664c70756317 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -39,7 +39,7 @@ # # -# 64KiB is the default PIPE buffer size of most posix platforms. +# 64KiB is the default PIPE buffer size of most POSIX platforms. BUFSIZE = 64*1024 # A very generous timeout when it comes to local connections... From 5ea7d022d0617fd5cc58f99f7a68c1563836b396 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 2 Sep 2024 19:30:18 +0900 Subject: [PATCH 6/9] fix news category --- .../2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Misc/NEWS.d/next/{C API => Library}/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst (100%) diff --git a/Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst b/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst similarity index 100% rename from Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst rename to Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst From 3dbf77d52c60cac91f52c5257fa652ff56272477 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 2 Sep 2024 19:41:38 +0900 Subject: [PATCH 7/9] revice changelog --- .../next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst b/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst index 7c3f74a5342ef6..1032b86378358d 100644 --- a/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst +++ b/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst @@ -1 +1 @@ -Limit reading size in multiprocessing ``connection._recv`` to 64KiB, in order to avoid memory overallocation and unnecessary memory management system calls. +Limit the reading size in the :class:`multiprocessing.Connection` class to 64KiB to prevent memory overallocation and unnecessary memory management system calls. From 353558641ddcfa74a093012e3a3f8d7988109777 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 2 Sep 2024 20:32:42 +0900 Subject: [PATCH 8/9] Apply suggestions from code review Co-authored-by: Victor Stinner --- Lib/multiprocessing/connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index b2664c70756317..7e901cf2fb9852 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -39,8 +39,8 @@ # # -# 64KiB is the default PIPE buffer size of most POSIX platforms. -BUFSIZE = 64*1024 +# 64 KiB is the default PIPE buffer size of most POSIX platforms. +BUFSIZE = 64 * 1024 # A very generous timeout when it comes to local connections... CONNECTION_TIMEOUT = 20. From 7dc489bbc61e83c9c62971aa51af7706e0c9c353 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 2 Sep 2024 20:33:22 +0900 Subject: [PATCH 9/9] fix link --- .../next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst b/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst index 1032b86378358d..bb41063e684da4 100644 --- a/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst +++ b/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst @@ -1 +1 @@ -Limit the reading size in the :class:`multiprocessing.Connection` class to 64KiB to prevent memory overallocation and unnecessary memory management system calls. +Limit the reading size in the :class:`multiprocessing.connection.Connection` class to 64 KiB to prevent memory overallocation and unnecessary memory management system calls.