From 4f452fea5278f283894c8a8b04ff82300438dbf4 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 28 Jan 2025 22:07:02 -0800 Subject: [PATCH 1/6] gh-129205: Update multiprocessing.forkserver to use os.readinto --- Lib/multiprocessing/forkserver.py | 15 ++++++++------- ...2025-01-28-22-11-41.gh-issue-129205.C-vE8S.rst | 2 ++ 2 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-28-22-11-41.gh-issue-129205.C-vE8S.rst diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index df9b9be9d1898b..2b47b34593dc7e 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -382,13 +382,14 @@ def _serve_one(child_r, fds, unused_fds, handlers): # def read_signed(fd): - data = b'' - length = SIGNED_STRUCT.size - while len(data) < length: - s = os.read(fd, length - len(data)) - if not s: - raise EOFError('unexpected EOF') - data += s + data = bytearray(SIGNED_STRUCT.size) + bytes_read = 0 + while count := os.readinto(fd, memoryview(data)[bytes_read:]): + bytes_read += count + + if bytes_read < SIGNED_STRUCT.size: + raise EOFError('unexpected EOF') + return SIGNED_STRUCT.unpack(data)[0] def write_signed(fd, n): diff --git a/Misc/NEWS.d/next/Library/2025-01-28-22-11-41.gh-issue-129205.C-vE8S.rst b/Misc/NEWS.d/next/Library/2025-01-28-22-11-41.gh-issue-129205.C-vE8S.rst new file mode 100644 index 00000000000000..721cde163b0ea9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-28-22-11-41.gh-issue-129205.C-vE8S.rst @@ -0,0 +1,2 @@ +:mod:`multiprocessing` forkserver now uses :func:`os.readinto` to read pid +of child processes saving memory allocations and copies.. From d2d5baf32d801161623fa50290b8bd76956eec78 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 28 Jan 2025 22:28:56 -0800 Subject: [PATCH 2/6] Update 2025-01-28-22-11-41.gh-issue-129205.C-vE8S.rst --- .../next/Library/2025-01-28-22-11-41.gh-issue-129205.C-vE8S.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2025-01-28-22-11-41.gh-issue-129205.C-vE8S.rst b/Misc/NEWS.d/next/Library/2025-01-28-22-11-41.gh-issue-129205.C-vE8S.rst index 721cde163b0ea9..ac8316986a931c 100644 --- a/Misc/NEWS.d/next/Library/2025-01-28-22-11-41.gh-issue-129205.C-vE8S.rst +++ b/Misc/NEWS.d/next/Library/2025-01-28-22-11-41.gh-issue-129205.C-vE8S.rst @@ -1,2 +1,2 @@ :mod:`multiprocessing` forkserver now uses :func:`os.readinto` to read pid -of child processes saving memory allocations and copies.. +of child processes saving memory allocations and copies. From 9227d4e58118ea9aae8a2ad081ee6456cb245158 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 28 Jan 2025 23:32:17 -0800 Subject: [PATCH 3/6] Rework loop to only use one read in got it all case --- Lib/multiprocessing/forkserver.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 2b47b34593dc7e..f2918845244b39 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -382,13 +382,14 @@ def _serve_one(child_r, fds, unused_fds, handlers): # def read_signed(fd): - data = bytearray(SIGNED_STRUCT.size) - bytes_read = 0 - while count := os.readinto(fd, memoryview(data)[bytes_read:]): - bytes_read += count - - if bytes_read < SIGNED_STRUCT.size: - raise EOFError('unexpected EOF') + to_read = SIGNED_STRUCT.size + data = bytearray(to_read) + while to_read: + count = os.readinto(fd, memoryview(data)[-to_read:]) + + if count == 0: + raise EOFError('unexpected EOF') + to_read -= count return SIGNED_STRUCT.unpack(data)[0] From 4409ae38da9aab166e3b1dc1f8c7cbdb90c23742 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 28 Jan 2025 23:37:17 -0800 Subject: [PATCH 4/6] Work in positives --- Lib/multiprocessing/forkserver.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index f2918845244b39..a07eb0a1d7d198 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -382,14 +382,13 @@ def _serve_one(child_r, fds, unused_fds, handlers): # def read_signed(fd): - to_read = SIGNED_STRUCT.size - data = bytearray(to_read) - while to_read: - count = os.readinto(fd, memoryview(data)[-to_read:]) - + data = bytearray(SIGNED_STRUCT.size) + bytes_read = 0 + while bytes_read < SIGNED_STRUCT.size: + count = os.readinto(fd, memoryview(data)[bytes_read:]) if count == 0: raise EOFError('unexpected EOF') - to_read -= count + bytes_read += count return SIGNED_STRUCT.unpack(data)[0] From 3ef05bad4d8f2e1817177a1440b74d6ec7115092 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 29 Jan 2025 12:19:10 -0800 Subject: [PATCH 5/6] Rely on memoryview to track length, reduce temporaries --- Lib/multiprocessing/forkserver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index a07eb0a1d7d198..681af2610e9b37 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -383,12 +383,12 @@ def _serve_one(child_r, fds, unused_fds, handlers): def read_signed(fd): data = bytearray(SIGNED_STRUCT.size) - bytes_read = 0 - while bytes_read < SIGNED_STRUCT.size: - count = os.readinto(fd, memoryview(data)[bytes_read:]) + unread = memoryview(data) + while unread: + count = os.readinto(fd, unread) if count == 0: raise EOFError('unexpected EOF') - bytes_read += count + unread = unread[count:] return SIGNED_STRUCT.unpack(data)[0] From 22e6821d273b9b9fce2a881d5df2dc4ed8173065 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Thu, 30 Jan 2025 14:00:03 -0800 Subject: [PATCH 6/6] Remove news --- .../next/Library/2025-01-28-22-11-41.gh-issue-129205.C-vE8S.rst | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2025-01-28-22-11-41.gh-issue-129205.C-vE8S.rst diff --git a/Misc/NEWS.d/next/Library/2025-01-28-22-11-41.gh-issue-129205.C-vE8S.rst b/Misc/NEWS.d/next/Library/2025-01-28-22-11-41.gh-issue-129205.C-vE8S.rst deleted file mode 100644 index ac8316986a931c..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-01-28-22-11-41.gh-issue-129205.C-vE8S.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`multiprocessing` forkserver now uses :func:`os.readinto` to read pid -of child processes saving memory allocations and copies.