8000 Speed up testpythoneval (#2635) · python/mypy@996e3e8 · GitHub
[go: up one dir, main page]
Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 996e3e8

Browse files
ambvgvanrossum
ambv
authored and
gvanrossum
committed
Speed up testpythoneval (#2635)
Splits eval-test into simple buckets by first letter of test name, enabling parallel execution. This speeds up execution of the test suite by around 25% on my laptop. The split enables more consistent loading of all CPU cores during the entire run of ./runtests.py. To achieve this, I had to modify testpythoneval.py to not write all testcase inputs to the same temporary path. Before: SUMMARY all 204 tasks and 1811 tests passed *** OK *** total time in run: 554.571954 total time in check: 214.105742 total time in lint: 130.914682 total time in pytest: 92.031659 ./runtests.py -j4 -v 744.76s user 74.10s system 235% cpu 5:48.34 total After: SUMMARY all 225 tasks and 3823 tests passed *** OK *** total time in run: 640.698327 total time in check: 178.758370 total time in lint: 149.604402 total time in pytest: 78.356671 ./runtests.py -j4 -v 850.81s user 81.09s system 353% cpu 4:23.69 total Total wall clock time fell from 5:48 to 4:23. Note: the test sum is now over-reported. Looks like the driver counts also the filtered out tests in eval-test. I don't have cycles now to hunt this down.
1 parent 560f2ec commit 996e3e8

File tree

2 files changed

+88
-35
lines changed

2 files changed

+88
-35
lines changed

‎mypy/test/testpythoneval.py

Lines changed: 64 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,20 @@
1010
this suite would slow down the main suite too much.
1111
"""
1212

13+
from contextlib import contextmanager
14+
import errno
1315
import os
1416
import os.path
17+
import re
1518
import subprocess
1619
import sys
1720

1821
import typing
22+
from typing import Dict, List, Tuple
1923

2024
from mypy.myunit import Suite, SkipTestCaseException
2125
from mypy.test.config import test_data_prefix, test_temp_dir
22-
from mypy.test.data import parse_test_cases
26+
from mypy.test.data import DataDrivenTestCase, parse_test_cases
2327
from mypy.test.helpers import assert_string_arrays_equal
2428
from mypy.util import try_find_python2_interpreter
2529

@@ -33,6 +37,7 @@
3337

3438
# Path to Python 3 interpreter
3539
python3_path = sys.executable
40+
program_re = re.compile(r'\b_program.py\b')
3641

3742

3843
class PythonEvaluationSuite(Suite):
@@ -48,56 +53,83 @@ def cases(self):
4853
return c
4954

5055

51-
def test_python_evaluation(testcase):
52-
python2_interpreter = try_find_python2_interpreter()
53-
# Use Python 2 interpreter if running a Python 2 test case.
54-
if testcase.name.lower().endswith('python2'):
55-
if not python2_interpreter:
56+
def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
57+
"""Runs Mypy in a subprocess.
58+
59+
If this passes without errors, executes the script again with a given Python
60+
version.
61+
"""
62+
mypy_cmdline = [
63+
python3_path,
64+
os.path.join(testcase.old_cwd, 'scripts', 'mypy'),
65+
'--show-traceback',
66+
]
67+
py2 = testcase.name.lower().endswith('python2')
68+
if py2:
69+
mypy_cmdline.append('--py2')
70+
interpreter = try_find_python2_interpreter()
71+
if not interpreter:
5672
# Skip, can't find a Python 2 interpreter.
5773
raise SkipTestCaseException()
58-
interpreter = python2_interpreter
59-
args = ['--py2']
60-
py2 = True
6174
else:
6275
interpreter = python3_path
63-
args = []
64-
py2 = False
65-
args.append('--show-traceback')
76+
6677
# Write the program to a file.
67-
program = '_program.py'
78+
program = '_' + testcase.name + '.py'
79+
mypy_cmdline.append(program)
6880
program_path = os.path.join(test_temp_dir, program)
6981
with open(program_path, 'w') as file:
7082
for s in testcase.input:
7183
file.write('{}\n'.format(s))
7284
# Type check the program.
7385
# This uses the same PYTHONPATH as the current process.
74-
process = subprocess.Popen([python3_path,
75-
os.path.join(testcase.old_cwd, 'scripts', 'mypy')]
76-
+ args + [program],
77-
stdout=subprocess.PIPE,
78-
stderr=subprocess.STDOUT,
79-
cwd=test_temp_dir)
80-
outb = process.stdout.read()
81-
# Split output into lines.
82-
out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
83-
if not process.wait():
86+
returncode, out = run(mypy_cmdline)
87+
if returncode == 0:
8488
# Set up module path for the execution.
8589
# This needs the typing module but *not* the mypy module.
8690
vers_dir = '2.7' if py2 else '3.2'
8791
typing_path = os.path.join(testcase.old_cwd, 'lib-typing', vers_dir)
8892
assert os.path.isdir(typing_path)
8993
env = os.environ.copy()
9094
env['PYTHONPATH'] = typing_path
91-
process = subprocess.Popen([interpreter, program],
92-
stdout=subprocess.PIPE,
93-
stderr=subprocess.STDOUT,
94-
cwd=test_temp_dir,
95-
env=env)
96-
outb = process.stdout.read()
97-
# Split output into lines.
98-
out += [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
95+
returncode, interp_out = run([interpreter, program], env=env)
96+
out += interp_out
9997
# Remove temp file.
10098
os.remove(program_path)
101-
assert_string_arrays_equal(testcase.output, out,
99+
assert_string_arrays_equal(adapt_output(testcase), out,
102100
'Invalid output ({}, line {})'.format(
103101
testcase.file, testcase.line))
102+
103+
104+
def split_lines(*streams: bytes) -> List[str]:
105+
"""Returns a single list of string lines from the byte streams in args."""
106+
return [
107+
s.rstrip('\n\r')
108+
for stream in streams
109+
for s in str(stream, 'utf8').splitlines()
110+
]
111+
112+
113+
def adapt_output(testcase: DataDrivenTestCase) -> List[str]:
114+
"""Translates the generic _program.py into the actual filename."""
115+
program = '_' + testcase.name + '.py'
116+
return [program_re.sub(program, line) for line in testcase.output]
117+
118+
119+
def run(
120+
cmdline: List[str], *, env: Dict[str, str] = None, timeout: int = 30
121+
) -> Tuple[int, List[str]]:
122+
"""A poor man's subprocess.run() for 3.3 and 3.4 compatibility."""
123+
process = subprocess.Popen(
124+
cmdline,
125+
env=env,
126+
stdout=subprocess.PIPE,
127+
stderr=subprocess.PIPE,
128+
cwd=test_temp_dir,
129+
)
130+
try:
131+
out, err = process.communicate(timeout=timeout)
132+
except subprocess.TimeoutExpired:
133+
out = err = b''
134+
process.kill()
135+
return process.returncode, split_lines(out, err)

‎runtests.py

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,12 @@ def get_versions(): # type: () -> typing.List[str]
2828

2929
from mypy.waiter import Waiter, LazySubprocess
3030
from mypy import util
31+
from mypy.test.config import test_data_prefix
32+
from mypy.test.testpythoneval import python_eval_files, python_34_eval_files
3133

3234
import itertools
3335
import os
36+
import re
3437

3538

3639
# Ideally, all tests would be `discover`able so that they can be driven
@@ -233,9 +236,27 @@ def add_myunit(driver: Driver) -> None:
233236

234237

235238
def add_pythoneval(driver: Driver) -> None:
236-
driver.add_python_mod('eval-test', 'mypy.myunit',
237-
'-m', 'mypy.test.testpythoneval', *driver.arglist,
238-
coverage=True)
239+
cases = set()
240+
case_re = re.compile(r'^\[case ([^\]]+)\]$')
241+
for file in python_eval_files + python_34_eval_files:
242+
with open(os.path.join(test_data_prefix, file), 'r') as f:
243+
for line in f:
244+
m = case_re.match(line)
245+
if m:
246+
case_name = m.group(1)
247+
assert case_name[:4] == 'test'
248+
cases.add(case_name[4:5])
249+
250+
for prefix in sorted(cases):
251+
driver.add_python_mod(
252+
'eval-test-' + prefix,
253+
'mypy.myunit',
254+
'-m',
255+
'mypy.test.testpythoneval',
256+
'test_testpythoneval_PythonEvaluationSuite.test' + prefix + '*',
257+
*driver.arglist,
258+
coverage=True
259+
)
239260

240261

241262
def add_cmdline(driver: Driver) -> None:

0 commit comments

Comments
 (0)
0