8000 gh-109162: libregrtest: remove WorkerJob class (#109204) · python/cpython@0c0f254 · GitHub
[go: up one dir, main page]
Skip to content

Commit 0c0f254

Browse files
vstinnervstinner
authored
gh-109162: libregrtest: remove WorkerJob class (#109204)
* Add attributes to Regrtest and RunTests: * gc_threshold * memory_limit * python_cmd * use_resources * Remove WorkerJob class. Add as_json() and from_json() methods to RunTests. A worker process now only uses RunTests for all parameters. * Add tests on support.set_memlimit() in test_support. Create _parse_memlimit() and also adds tests on it. * Remove 'ns' parameter from runtest.py.
1 parent 24fa8f2 commit 0c0f254

File tree

7 files changed

+126
-88
lines changed

7 files changed

+126
-88
lines changed

‎Lib/test/libregrtest/cmdline.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,8 @@ def __init__(self, **kwargs) -> None:
177177
self.worker_json = No 8000 ne
178178
self.start = None
179179
self.timeout = None
180+
self.memlimit = None
181+
self.threshold = None
180182

181183
super().__init__(**kwargs)
182184

‎Lib/test/libregrtest/main.py

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,10 @@ def __init__(self, ns: Namespace):
100100
self.hunt_refleak = None
101101
self.test_dir: str | None = ns.testdir
102102
self.junit_filename: str | None = ns.xmlpath
103+
self.memory_limit: str | None = ns.memlimit
104+
self.gc_threshold: int | None = ns.threshold
105+
self.use_resources: list[str] = ns.use_resources
106+
self.python_cmd: list[str] | None = ns.python
103107

104108
# tests
1051 6D40 09
self.tests = []
@@ -363,7 +367,7 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests):
363367
return runtests
364368

365369
def rerun_failed_tests(self, need_rerun, runtests: RunTests):
366-
if self.ns.python:
370+
if self.python_cmd:
367371
# Temp patch for https://github.com/python/cpython/issues/94052
368372
self.log(
369373
"Re-running failed tests is not supported with --python "
@@ -453,12 +457,12 @@ def run_test(self, test_name: str, runtests: RunTests, tracer):
453457
if tracer is not None:
454458
# If we're tracing code coverage, then we don't exit with status
455459
# if on a false return value from main.
456-
cmd = ('result = run_single_test(test_name, runtests, self.ns)')
460+
cmd = ('result = run_single_test(test_name, runtests)')
457461
ns = dict(locals())
458462
tracer.runctx(cmd, globals=globals(), locals=ns)
459463
result = ns['result']
460464
else:
461-
result = run_single_test(test_name, runtests, self.ns)
465+
result = run_single_test(test_name, runtests)
462466

463467
self.accumulate_result(result)
464468

@@ -876,9 +880,14 @@ def action_run_tests(self):
876880
quiet=self.quiet,
877881
hunt_refleak=self.hunt_refleak,
878882
test_dir=self.test_dir,
879-
junit_filename=self.junit_filename)
880-
881-
setup_tests(runtests, self.ns)
883+
junit_filename=self.junit_filename,
884+
memory_limit=self.memory_limit,
885+
gc_threshold=self.gc_threshold,
886+
use_resources=self.use_resources,
887+
python_cmd=self.python_cmd,
888+
)
889+
890+
setup_tests(runtests)
882891

883892
tracer = self.run_tests(runtests)
884893
self.display_result(runtests)

‎Lib/test/libregrtest/runtest.py

Lines changed: 40 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,18 @@
44
import gc
55
import importlib
66
import io
7+
import json
78
import os
89
import sys
910
import time
1011
import traceback
1112
import unittest
13+
from typing import Any
1214

1315
from test import support
1416
from test.support import TestStats
1517
from test.support import os_helper
1618
from test.support import threading_helper
17-
from test.libregrtest.cmdline import Namespace
1819
from test.libregrtest.save_env import saved_test_environment
1920
from test.libregrtest.utils import clear_caches, format_duration, print_warning
2021

@@ -230,6 +231,10 @@ class RunTests:
230231
hunt_refleak: HuntRefleak | None = None
231232
test_dir: str | None = None
232233
junit_filename: str | None = None
234+
memory_limit: str | None = None
235+
gc_threshold: int | None = None
236+
use_resources: list[str] = None
237+
python_cmd: list[str] | None = None
233238

234239
def copy(self, **override):
235240
state = dataclasses.asdict(self)
@@ -249,11 +254,32 @@ def iter_tests(self):
249254
else:
250255
yield from self.tests
251256

257+
def as_json(self):
258+
return json.dumps(self, cls=_EncodeRunTests)
259+
252260
@staticmethod
253-
def from_json_dict(json_dict):
254-
if json_dict['hunt_refleak']:
255-
json_dict['hunt_refleak'] = HuntRefleak(**json_dict['hunt_refleak'])
256-
return RunTests(**json_dict)
261+
def from_json(worker_json):
262+
return json.loads(worker_json, object_hook=_decode_runtests)
263+
264+
265+
class _EncodeRunTests(json.JSONEncoder):
266+
def default(self, o: Any) -> dict[str, Any]:
267+
if isinstance(o, RunTests):
268+
result = dataclasses.asdict(o)
269+
result["__runtests__"] = True
270+
return result
271+
else:
272+
return super().default(o)
273+
274+
275+
def _decode_runtests(data: dict[str, Any]) -> RunTests | dict[str, Any]:
276+
if "__runtests__" in data:
277+
data.pop('__runtests__')
278+
if data['hunt_refleak']:
279+
data['hunt_refleak'] = HuntRefleak(**data['hunt_refleak'])
280+
return RunTests(**data)
281+
else:
282+
return data
257283

258284

259285
# Minimum duration of a test to display its duration or to mention that
@@ -320,7 +346,7 @@ def abs_module_name(test_name: str, test_dir: str | None) -> str:
320346
return 'test.' + test_name
321347

322348

323-
def setup_support(runtests: RunTests, ns: Namespace):
349+
def setup_support(runtests: RunTests):
324350
support.PGO = runtests.pgo
325351
support.PGO_EXTENDED = runtests.pgo_extended
326352
support.set_match_tests(runtests.match_tests, runtests.ignore_tests)
@@ -332,7 +358,7 @@ def setup_support(runtests: RunTests, ns: Namespace):
332358
support.junit_xml_list = None
333359

334360

335-
def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None:
361+
def _runtest(result: TestResult, runtests: RunTests) -> None:
336362
# Capture stdout and stderr, set faulthandler timeout,
337363
# and create JUnit XML report.
338364
verbose = runtests.verbose
@@ -346,7 +372,7 @@ def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None:
346372
faulthandler.dump_traceback_later(timeout, exit=True)
347373

348374
try:
349-
setup_support(runtests, ns)
375+
setup_support(runtests)
350376

351377
if output_on_failure:
352378
support.verbose = True
@@ -366,7 +392,7 @@ def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None:
366392
# warnings will be written to sys.stderr below.
367393
print_warning.orig_stderr = stream
368394

369-
_runtest_env_changed_exc(result, runtests, ns, display_failure=False)
395+
_runtest_env_changed_exc(result, runtests, display_failure=False)
370396
# Ignore output if the test passed successfully
371397
if result.state != State.PASSED:
372398
output = stream.getvalue()
@@ -381,7 +407,7 @@ def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None:
381407
else:
382408
# Tell tests to be moderately quiet
383409
support.verbose = verbose
384-
_runtest_env_changed_exc(result, runtests, ns,
410+
_runtest_env_changed_exc(result, runtests,
385411
display_failure=not verbose)
386412

387413
xml_list = support.junit_xml_list
@@ -395,10 +421,9 @@ def _runtest(result: TestResult, runtests: RunTests, ns: Namespace) -> None:
395421
support.junit_xml_list = None
396422

397423

398-
def run_single_test(test_name: str, runtests: RunTests, ns: Namespace) -> TestResult:
424+
def run_single_test(test_name: str, runtests: RunTests) -> TestResult:
399425
"""Run a single test.
400426
401-
ns -- regrtest namespace of options
402427
test_name -- the name of the test
403428
404429
Returns a TestResult.
@@ -410,7 +435,7 @@ def run_single_test(test_name: str, runtests: RunTests, ns: Namespace) -> TestRe
410435
result = TestResult(test_name)
411436
pgo = runtests.pgo
412437
try:
413-
_runtest(result, runtests, ns)
438+
_runtest(result, runtests)
414439
except:
415440
if not pgo:
416441
msg = traceback.format_exc()
@@ -472,7 +497,7 @@ def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None:
472497
FOUND_GARBAGE = []
473498

474499

475-
def _load_run_test(result: TestResult, runtests: RunTests, ns: Namespace) -> None:
500+
def _load_run_test(result: TestResult, runtests: RunTests) -> None:
476501
# Load the test function, run the test function.
477502
module_name = abs_module_name(result.test_name, runtests.test_dir)
478503

@@ -513,7 +538,6 @@ def test_func():
513538

514539

515540
def _runtest_env_changed_exc(result: TestResult, runtests: RunTests,
516-
ns: Namespace,
517541
display_failure: bool = True) -> None:
518542
# Detect environment changes, handle exceptions.
519543

@@ -532,7 +556,7 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests,
532556
support.gc_collect()
533557

534558
with save_env(test_name, runtests):
535-
_load_run_test(result, runtests, ns)
559+
_load_run_test(result, runtests)
536560
except support.ResourceDenied as msg:
537561
if not quiet and not pgo:
538562
print(f"{test_name} skipped -- {msg}", flush=True)

‎Lib/test/libregrtest/runtest_mp.py

Lines changed: 10 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -47,49 +47,16 @@
4747
@dataclasses.dataclass(slots=True)
4848
class WorkerJob:
4949
runtests: RunTests
50-
namespace: Namespace
5150

5251

53-
class _EncodeWorkerJob(json.JSONEncoder):
54-
def default(self, o: Any) -> dict[str, Any]:
55-
match o:
56-
case WorkerJob():
57-
result = dataclasses.asdict(o)
58-
result["__worker_job__"] = True
59-
return result
60-
case Namespace():
61-
result = vars(o)
62-
result["__namespace__"] = True
63-
return result
64-
case _:
65-
return super().default(o)
66-
67-
68-
def _decode_worker_job(d: dict[str, Any]) -> WorkerJob | dict[str, Any]:
69-
if "__worker_job__" in d:
70-
d.pop('__worker_job__')
71-
d['runtests'] = RunTests.from_json_dict(d['runtests'])
72-
return WorkerJob(**d)
73-
if "__namespace__" in d:
74-
d.pop('__namespace__')
75-
return Namespace(**d)
76-
else:
77-
return d
78-
79-
80-
def _parse_worker_json(worker_json: str) -> tuple[Namespace, str]:
81-
return json.loads(worker_json, object_hook=_decode_worker_job)
82-
83-
84-
def create_worker_process(worker_job: WorkerJob,
52+
def create_worker_process(runtests: RunTests,
8553
output_file: TextIO,
8654
tmp_dir: str | None = None) -> subprocess.Popen:
87-
ns = worker_job.namespace
88-
python = ns.python
89-
worker_json = json.dumps(worker_job, cls=_EncodeWorkerJob)
55+
python_cmd = runtests.python_cmd
56+
worker_json = runtests.as_json()
9057

91-
if python is not None:
92-
executable = python
58+
if python_cmd is not None:
59+
executable = python_cmd
9360
else:
9461
executable = [sys.executable]
9562
cmd = [*executable, *support.args_from_interpreter_flags(),
@@ -121,14 +88,12 @@ def create_worker_process(worker_job: WorkerJob,
12188

12289

12390
def worker_process(worker_json: str) -> NoReturn:
124-
worker_job = _parse_worker_json(worker_json)
125-
runtests = worker_job.runtests
126-
ns = worker_job.namespace
91+
runtests = RunTests.from_json(worker_json)
12792
test_name = runtests.tests[0]
12893
match_tests: FilterTuple | None = runtests.match_tests
12994

13095
setup_test_dir(runtests.test_dir)
131-
setup_tests(runtests, ns)
96+
setup_tests(runtests)
13297

13398
if runtests.rerun:
13499
if match_tests:
@@ -137,7 +102,7 @@ def worker_process(worker_json: str) -> NoReturn:
137102
else:
138103
print(f"Re-running {test_name} in verbose mode", flush=True)
139104

140-
result = run_single_test(test_name, runtests, ns)
105+
result = run_single_test(test_name, runtests)
141106
print() # Force a newline (just in case)
142107

143108
# Serialize TestResult as dict in JSON
@@ -330,9 +295,6 @@ def _runtest(self, test_name: str) -> MultiprocessResult:
330295
if match_tests:
331296
kwargs['match_tests'] = match_tests
332297
worker_runtests = self.runtests.copy(tests=tests, **kwargs)
333-
worker_job = WorkerJob(
334-
worker_runtests,
335-
namespace=self.ns)
336298

337299
# gh-94026: Write stdout+stderr to a tempfile as workaround for
338300
# non-blocking pipes on Emscripten with NodeJS.
@@ -347,12 +309,12 @@ def _runtest(self, test_name: str) -> MultiprocessResult:
347309
tmp_dir = tempfile.mkdtemp(prefix="test_python_")
348310
tmp_dir = os.path.abspath(tmp_dir)
349311
try:
350-
retcode = self._run_process(worker_job, stdout_file, tmp_dir)
312+
retcode = self._run_process(worker_runtests, stdout_file, tmp_dir)
351313
finally:
352314
tmp_files = os.listdir(tmp_dir)
353315
os_helper.rmtree(tmp_dir)
354316
else:
355-
retcode = self._run_process(worker_job, stdout_file)
317+
retcode = self._run_process(worker_runtests, stdout_file)
356318
tmp_files = ()
357319
stdout_file.seek(0)
358320

‎Lib/test/libregrtest/setup.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ def setup_test_dir(testdir: str | None) -> None:
2525
sys.path.insert(0, os.path.abspath(testdir))
2626

2727

28-
def setup_tests(runtests, ns):
28+
def setup_tests(runtests):
2929
try:
3030
stderr_fd = sys.__stderr__.fileno()
3131
except (ValueError, AttributeError):
@@ -71,15 +71,15 @@ def setup_tests(runtests, ns):
7171
if runtests.hunt_refleak:
7272
unittest.BaseTestSuite._cleanup = False
7373

74-
if ns.memlimit is not None:
75-
support.set_memlimit(ns.memlimit)
74+
if runtests.memory_limit is not None:
75+
support.set_memlimit(runtests.memory_limit)
7676

77-
if ns.threshold is not None:
78-
gc.set_threshold(ns.threshold)
77+
if runtests.gc_threshold is not None:
78+
gc.set_threshold(runtests.gc_threshold)
7979

8080
support.suppress_msvcrt_asserts(runtests.verbose and runtests.verbose >= 2)
8181

82-
support.use_resources = ns.use_resources
82+
support.use_resources = runtests.use_resources
8383

8484
if hasattr(sys, 'addaudithook'):
8585
# Add an auditing hook for all tests to ensure PySys_Audit is tested

‎Lib/test/support/__init__.py

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -878,27 +878,31 @@ def inner(*args, **kwds):
878878

879879
MAX_Py_ssize_t = sys.maxsize
880880

881-
def set_memlimit(limit):
882-
global max_memuse
883-
global real_max_memuse
881+
def _parse_memlimit(limit: str) -> int:
884882
sizes = {
885883
'k': 1024,
886884
'm': _1M,
887885
'g': _1G,
888886
't': 1024*_1G,
889887
}
890-
m = re.match(r'(\d+(\.\d+)?) (K|M|G|T)b?$', limit,
888+
m = re.match(r'(\d+(?:\.\d+)?) (K|M|G|T)b?$', limit,
891889
re.IGNORECASE | re.VERBOSE)
892890
if m is None:
893-
raise ValueError('Invalid memory limit %r' % (limit,))
894-
memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()])
895-
real_max_memuse = memlimit
896-
if memlimit > MAX_Py_ssize_t:
897-
memlimit = MAX_Py_ssize_t
891+
raise ValueError(f'Invalid memory limit: {limit!r}')
892+
return int(float(m.group(1)) * sizes[m.group(2).lower()])
893+
894+
def set_memlimit(limit: str) -> None:
895+
global max_memuse
896+
global real_max_memuse
897+
memlimit = _parse_memlimit(limit)
898898
if memlimit < _2G - 1:
899-
raise ValueError('Memory limit %r too low to be useful' % (limit,))
899+
raise ValueError('Memory limit {limit!r} too low to be useful')
900+
901+
real_max_memuse = memlimit
902+
memlimit = min(memlimit, MAX_Py_ssize_t)
900903
max_memuse = memlimit
901904

905+
902906
class _MemoryWatchdog:
903907
"""An object which periodically watches the process' memory consumption
904908
and prints it out.

0 commit comments

Comments
 (0)
0