8000 gh-109276: libregrtest: WASM use filename for JSON · python/cpython@2004fb6 · GitHub
[go: up one dir, main page]
Skip to content

Commit 2004fb6

Browse files
vstinnervstinner
committed
gh-109276: libregrtest: WASM use filename for JSON
On Emscripten and WASI platforms, libregrtest now uses a filename for the JSON file. Passing a file descriptor to a child process doesn't work on these platforms.
1 parent 66d1d7e commit 2004fb6

File tree

5 files changed

+61
-30
lines changed

5 files changed

+61
-30
lines changed

‎Lib/test/libregrtest/main.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -406,7 +406,7 @@ def create_run_tests(self, tests: TestTuple):
406406
python_cmd=self.python_cmd,
407407
randomize=self.randomize,
408408
random_seed=self.random_seed,
409-
json_fd=None,
409+
json_file=None,
410410
)
411411

412412
def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:

‎Lib/test/libregrtest/run_workers.py

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
from .logger import Logger
1818
from .result import TestResult, State
1919
from .results import TestResults
20-
from .runtests import RunTests
20+
from .runtests import RunTests, JsonFileType, JSON_FILE_USE_FILENAME
2121
from .single import PROGRESS_MIN_TIME
2222
from .utils import (
2323
StrPath, StrJSON, TestName, MS_WINDOWS,
@@ -155,10 +155,11 @@ def mp_result_error(
155155
) -> MultiprocessResult:
156156
return MultiprocessResult(test_result, stdout, err_msg)
157157

158-
def _run_process(self, runtests: RunTests, output_fd: int, json_fd: int,
158+
def _run_process(self, runtests: RunTests, output_fd: int,
159+
json_file: JsonFileType,
159160
tmp_dir: StrPath | None = None) -> int:
160161
try:
161-
popen = create_worker_process(runtests, output_fd, json_fd,
162+
popen = create_worker_process(runtests, output_fd, json_file,
162163
tmp_dir)
163164

164165
self._killed = False
@@ -226,21 +227,29 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
226227
match_tests = None
227228
err_msg = None
228229

230+
stdout_file = tempfile.TemporaryFile('w+', encoding=encoding)
231+
if JSON_FILE_USE_FILENAME:
232+
json_tmpfile = tempfile.NamedTemporaryFile('w+', encoding='utf8')
233+
else:
234+
json_tmpfile = tempfile.TemporaryFile('w+', encoding='utf8')
235+
229236
# gh-94026: Write stdout+stderr to a tempfile as workaround for
230237
# non-blocking pipes on Emscripten with NodeJS.
231-
with (tempfile.TemporaryFile('w+', encoding=encoding) as stdout_file,
232-
tempfile.TemporaryFile('w+', encoding='utf8') as json_file):
238+
with (stdout_file, json_tmpfile):
233239
stdout_fd = stdout_file.fileno()
234-
json_fd = json_file.fileno()
235-
if MS_WINDOWS:
236-
json_fd = msvcrt.get_osfhandle(json_fd)
240+
if JSON_FILE_USE_FILENAME:
241+
json_file = json_tmpfile.name
242+
else:
243+
json_file = json_tmpfile.fileno()
244+
if MS_WINDOWS:
245+
json_file = msvcrt.get_osfhandle(json_file)
237246

238247
kwargs = {}
239248
if match_tests:
240249
kwargs['match_tests'] = match_tests
241250
worker_runtests = self.runtests.copy(
242251
tests=tests,
243-
json_fd=json_fd,
252+
json_file=json_file,
244253
**kwargs)
245254

246255
# gh-93353: Check for leaked temporary files in the parent process,
@@ -254,13 +263,13 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
254263
tmp_dir = os.path.abspath(tmp_dir)
255264
try:
256265
retcode = self._run_process(worker_runtests,
257-
stdout_fd, json_fd, tmp_dir)
266+
stdout_fd, json_file, tmp_dir)
258267
finally:
259268
tmp_files = os.listdir(tmp_dir)
260269
os_helper.rmtree(tmp_dir)
261270
else:
262271
retcode = self._run_process(worker_runtests,
263-
stdout_fd, json_fd)
272+
stdout_fd, json_file)
264273
tmp_files = ()
265274
stdout_file.seek(0)
266275

@@ -275,8 +284,8 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
275284

276285
try:
277286
# deserialize run_tests_worker() output
278-
json_file.seek(0)
279-
worker_json: StrJSON = json_file.read()
287+
json_tmpfile.seek(0)
288+
worker_json: StrJSON = json_tmpfile.read()
280289
if worker_json:
281290
result = TestResult.from_json(worker_json)
282291
else:

‎Lib/test/libregrtest/runtests.py

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,25 @@
22
import json
33
from typing import Any
44

5+
from test import support
6+
57
from .utils import (
68
StrPath, StrJSON, TestTuple, FilterTuple, FilterDict)
79

810

11+
if support.is_emscripten or support.is_wasi:
12+
# On Emscripten/WASI, it's a filename. Passing a file descriptor to a
13+
# worker process fails with "OSError: [Errno 8] Bad file descriptor" in the
14+
# worker process.
15+
JsonFileType = StrPath
16+
JSON_FILE_USE_FILENAME = True
17+
else:
18+
# On Unix, it's a file descriptor.
19+
# On Windows, it's a handle.
20+
JsonFileType = int
21+
JSON_FILE_USE_FILENAME = False
22+
23+
924
@dataclasses.dataclass(slots=True, frozen=True)
1025
class HuntRefleak:
1126
warmups: int
@@ -38,9 +53,7 @@ class RunTests:
3853
python_cmd: tuple[str] | None
3954
randomize: bool
4055
random_seed: int | None
41-
# On Unix, it's a file descriptor.
42-
# On Windows, it's a handle.
43-
json_fd: int | None
56+
json_file: JsonFileType | None
4457

4558
def copy(self, **override):
4659
state = dataclasses.asdict(self)

‎Lib/test/libregrtest/utils.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -387,7 +387,7 @@ def get_work_dir(parent_dir: StrPath, worker: bool = False) -> StrPath:
387387
# testing (see the -j option).
388388
# Emscripten and WASI have stubbed getpid(), Emscripten has only
389389
# milisecond clock resolution. Use randint() instead.
390-
if sys.platform in {"emscripten", "wasi"}:
390+
if support.is_emscripten or support.is_wasi:
391391
nounce = random.randint(0, 1_000_000)
392392
else:
393393
nounce = os.getpid()

‎Lib/test/libregrtest/worker.py

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from test.support import os_helper
88

99
from .setup import setup_process, setup_test_dir
10-
from .runtests import RunTests
10+
from .runtests import RunTests, JsonFileType, JSON_FILE_USE_FILENAME
1111
from .single import run_single_test
1212
from .utils import (
1313
StrPath, StrJSON, FilterTuple, MS_WINDOWS,
@@ -18,7 +18,7 @@
1818

1919

2020
def create_worker_process(runtests: RunTests,
21-
output_fd: int, json_fd: int,
21+
output_fd: int, json_file: JsonFileType,
2222
tmp_dir: StrPath | None = None) -> subprocess.Popen:
2323
python_cmd = runtests.python_cmd
2424
worker_json = runtests.as_json()
@@ -55,33 +55,42 @@ def create_worker_process(runtests: RunTests,
5555
close_fds=True,
5656
cwd=work_dir,
5757
)
58-
if not MS_WINDOWS:
59-
kwargs['pass_fds'] = [json_fd]
60-
else:
58+
if JSON_FILE_USE_FILENAME:
59+
# Nothing to do to pass the JSON filename to the worker process
60+
pass
61+
elif MS_WINDOWS:
62+
# Pass the JSON handle to the worker process
6163
startupinfo = subprocess.STARTUPINFO()
62-
startupinfo.lpAttributeList = {"handle_list": [json_fd]}
64+
startupinfo.lpAttributeList = {"handle_list": [json_file]}
6365
kwargs['startupinfo'] = startupinfo
66+
else:
67+
# Pass the JSON file descriptor to the worker process
68+
kwargs['pass_fds'] = [json_file]
6469
if USE_PROCESS_GROUP:
6570
kwargs['start_new_session'] = True
6671

6772
if MS_WINDOWS:
68-
os.set_handle_inheritable(json_fd, True)
73+
os.set_handle_inheritable(json_file, True)
6974
try:
7075
return subprocess.Popen(cmd, **kwargs)
7176
finally:
7277
if MS_WINDOWS:
73-
os.set_handle_inheritable(json_fd, False)
78+
os.set_handle_inheritable(json_file, False)
7479

7580

7681
def worker_process(worker_json: StrJSON) -> NoReturn:
7782
runtests = RunTests.from_json(worker_json)
7883
test_name = runtests.tests[0]
7984
match_tests: FilterTuple | None = runtests.match_tests
80-
json_fd: int = runtests.json_fd
85+
# On Unix, it's a file descriptor.
86+
# On Windows, it's a handle.
87+
# On Emscripten/WASI, it's a filename.
88+
json_file: JsonFileType = runtests.json_file
8189

8290
if MS_WINDOWS:
8391
import msvcrt
84-
json_fd = msvcrt.open_osfhandle(json_fd, os.O_WRONLY)
92+
# Create a file descriptor from the handle
93+
json_file = msvcrt.open_osfhandle(json_file, os.O_WRONLY)
8594

8695

8796
setup_test_dir(runtests.test_dir)
@@ -96,8 +105,8 @@ def worker_process(worker_json: StrJSON) -> NoReturn:
96105

97106
result = run_single_test(test_name, runtests)
98107

99-
with open(json_fd, 'w', encoding='utf-8') as json_file:
100-
result.write_json_into(json_file)
108+
with open(json_file, 'w', encoding='utf-8') as fp:
109+
result.write_json_into(fp)
101110

102111
sys.exit(0)
103112

0 commit comments

Comments
 (0)
0