8000 gh-94026: Buffer regrtest worker stdout in temporary file (GH-94253) · vstinner/cpython@24d94ca · GitHub
[go: up one dir, main page]
Skip to content

Commit 24d94ca

Browse files
tiranvstinner
tiran
and
vstinner
committed
pythongh-94026: Buffer regrtest worker stdout in temporary file (pythonGH-94253)
Co-authored-by: Victor Stinner <vstinner@python.org> (cherry picked from commit 199ba23)
1 parent 5c87ac3 commit 24d94ca

File tree

1 file changed

+38
-39
lines changed

1 file changed

+38
-39
lines changed

‎Lib/test/libregrtest/runtest_mp.py

Lines changed: 38 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import threading
1010
import time
1111
import traceback
12-
from typing import NamedTuple, NoReturn, Literal, Any
12+
from typing import NamedTuple, NoReturn, Literal, Any, TextIO
1313

1414
from test import support
1515
from test.support import os_helper
@@ -53,7 +53,7 @@ def parse_worker_args(worker_args) -> tuple[Namespace, str]:
5353
return (ns, test_name)
5454

5555

56-
def run_test_in_subprocess(testname: str, ns: Namespace, tmp_dir: str) -> subprocess.Popen:
56+
def run_test_in_subprocess(testname: str, ns: Namespace, tmp_dir: str, stdout_fh: TextIO) -> subprocess.Popen:
5757
ns_dict = vars(ns)
5858
worker_args = (ns_dict, testname)
5959
worker_args = json.dumps(worker_args)
@@ -75,18 +75,18 @@ def run_test_in_subprocess(testname: str, ns: Namespace, tmp_dir: str) -> subpro
7575
# Running the child from the same working directory as regrtest's original
7676
# invocation ensures that TEMPDIR for the child is the same when
7777
# sysconfig.is_python_build() is true. See issue 15300.
78-
kw = {'env': env}
78+
kw = dict(
79+
env=env,
80+
stdout=stdout_fh,
81+
# bpo-45410: Write stderr into stdout to keep messages order
82+
stderr=stdout_fh,
83+
text=True,
84+
close_fds=(os.name != 'nt'),
85+
cwd=os_helper.SAVEDCWD,
86+
)
7987
if USE_PROCESS_GROUP:
8088
kw['start_new_session'] = True
81-
return subprocess.Popen(cmd,
82-
stdout=subprocess.PIPE,
83-
# bpo-45410: Write stderr into stdout to keep
84-
# messages order
85-
stderr=subprocess.STDOUT,
86-
universal_newlines=True,
87-
close_fds=(os.name != 'nt'),
88-
cwd=os_helper.SAVEDCWD,
89-
**kw)
89+
return subprocess.Popen(cmd, **kw)
9090

9191

9292
def run_tests_worker(ns: Namespace, test_name: str) -> NoReturn:
@@ -212,12 +212,12 @@ def mp_result_error(
212212
test_result.duration_sec = time.monotonic() - self.start_time
213213
return MultiprocessResult(test_result, stdout, err_msg)
214214

215-
def _run_process(self, test_name: str, tmp_dir: str) -> tuple[int, str, str]:
215+
def _run_process(self, test_name: str, tmp_dir: str, stdout_fh: TextIO) -> int:
216216
self.start_time = time.monotonic()
217217

218218
self.current_test_name = test_name
219219
try:
220-
popen = run_test_in_subprocess(test_name, self.ns, tmp_dir)
220+
popen = run_test_in_subprocess(test_name, self.ns, tmp_dir, stdout_fh)
221221

222222
self._killed = False
223223
self._popen = popen
@@ -234,10 +234,10 @@ def _run_process(self, test_name: str, tmp_dir: str) -> tuple[int, str, str]:
234234
raise ExitThread
235235

236236
try:
237-
# bpo-45410: stderr is written into stdout
238-
stdout, _ = popen.communicate(timeout=self.timeout)
239-
retcode = popen.returncode
237+
# gh-94026: stdout+stderr are written to tempfile
238+
retcode = popen.wait(timeout=self.timeout)
240239
assert retcode is not None
240+
return retcode
241241
except subprocess.TimeoutExpired:
242242
if self._stopped:
243243
# kill() has been called: communicate() fails on reading
@@ -252,17 +252,12 @@ def _run_process(self, test_name: str, tmp_dir: str) -> tuple[int, str, str]:
252252
# bpo-38207: Don't attempt to call communicate() again: on it
253253
# can hang until all child processes using stdout
254254
# pipes completes.
255-
stdout = ''
256255
except OSError:
257256
if self._stopped:
258257
# kill() has been called: communicate() fails
259258
# on reading closed stdout
260259
raise ExitThread
261260
raise
262-
else:
263-
stdout = stdout.strip()
264-
265-
return (retcode, stdout)
266261
except:
267262
self._kill()
268263
raise
@@ -272,23 +267,30 @@ def _run_process(self, test_name: str, tmp_dir: str) -> tuple[int, str, str]:
272267
self.current_test_name = None
273268

274269
def _runtest(self, test_name: str) -> MultiprocessResult:
275-
# Don't check for leaked temporary files and directories if Python is
276-
# run on WASI. WASI don't pass environment variables like TMPDIR to
277-
# worker processes.
278-
if not support.is_wasi:
270+
# gh-94026: Write stdout+stderr to a tempfile as workaround for
271+
# non-blocking pipes on Emscripten with NodeJS.
272+
with tempfile.TemporaryFile(
273+
'w+', encoding=sys.stdout.encoding
274+
) as stdout_fh:
279275
# gh-93353: Check for leaked temporary files in the parent process,
280276
# since the deletion of temporary files can happen late during
281277
# Python finalization: too late for libregrtest.
282-
tmp_dir = tempfile.mkdtemp(prefix="test_python_")
283-
tmp_dir = os.path.abspath(tmp_dir)
284-
try:
285-
retcode, stdout = self._run_process(test_name, tmp_dir)
286-
finally:
287-
tmp_files = os.listdir(tmp_dir)
288-
os_helper.rmtree(tmp_dir)
289-
else:
290-
retcode, stdout = self._run_process(test_name, None)
291-
tmp_files = ()
278+
if not support.is_wasi:
279+
# Don't check for leaked temporary files and directories if Python is
280+
# run on WASI. WASI don't pass environment variables like TMPDIR to
281+
# worker processes.
282+
tmp_dir = tempfile.mkdtemp(prefix="test_python_")
283+
tmp_dir = os.path.abspath(tmp_dir)
284+
try:
285+
retcode = self._run_process(test_name, tmp_dir, stdout_fh)
286+
finally:
287+
tmp_files = os.listdir(tmp_dir)
288+
os_helper.rmtree(tmp_dir)
289+
else:
290+
retcode = self._run_process(test_name, None, stdout_fh)
291+
tmp_files = ()
292+
stdout_fh.seek(0)
293+
stdout = stdout_fh.read().strip()
292294

293295
if retcode is None:
294296
return self.mp_result_error(Timeout(test_name), stdout)
@@ -343,9 +345,6 @@ def run(self) -> None:
343345
def _wait_completed(self) -> None:
344346
popen = self._popen
345347

346-
# stdout must be closed to ensure that communicate() does not hang
347-
popen.stdout.close()
348-
349348
try:
350349
popen.wait(JOIN_TIMEOUT)
351350
except (subprocess.TimeoutExpired, OSError) as exc:

0 commit comments

Comments
 (0)
0