Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 5bf9d18

Browse files
pkchJukkaL
pkch
authored and
JukkaL
committed
Optimize tests (parallel execution) (#3019)
I added a few features to runtests.py: * -p argument to pass arguments to pytest * move pytest to the front, run before anything else * combine all pytest tests into one shell command * schedule longer tasks first
1 parent 00359ad commit 5bf9d18

3 files changed

Lines changed: 147 additions & 23 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ docs/build/
1212
.mypy_cache/
1313
.incremental_checker_cache.json
1414
.cache
15+
.runtest_log.json
1516

1617
# Packages
1718
*.egg

‎mypy/waiter.py‎

Lines changed: 99 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,18 @@
33
This is used for running mypy tests.
44
"""
55

6-
from typing import Dict, List, Optional, Set, Tuple
6+
from typing import Dict, List, Optional, Set, Tuple, Any, Iterable
77

88
import os
9+
from multiprocessing import cpu_count
910
import pipes
1011
import re
1112
from subprocess import Popen, STDOUT
1213
import sys
1314
import tempfile
1415
import time
16+
import json
17+
from collections import defaultdict
1518

1619

1720
class WaiterError(Exception):
@@ -32,7 +35,7 @@ def __init__(self, name: str, args: List[str], *, cwd: str = None,
3235

3336
def start(self) -> None:
3437
self.outfile = tempfile.TemporaryFile()
35-
self.start_time = time.time()
38+
self.start_time = time.perf_counter()
3639
self.process = Popen(self.args, cwd=self.cwd, env=self.env,
3740
stdout=self.outfile, stderr=STDOUT)
3841
self.pid = self.process.pid
@@ -107,7 +110,11 @@ class Waiter:
107110
if not waiter.run():
108111
print('error')
109112
"""
110-
def __init__(self, limit: int = 0, *, verbosity: int = 0, xfail: List[str] = []) -> None:
113+
LOGSIZE = 50
114+
FULL_LOG_FILENAME = '.runtest_log.json'
115+
116+
def __init__(self, limit: int = 0, *, verbosity: int = 0, xfail: List[str] = [],
117+
lf: bool = False, ff: bool = False) -> None:
111118
self.verbosity = verbosity
112119
self.queue = [] # type: List[LazySubprocess]
113120
# Index of next task to run in the queue.
@@ -117,21 +124,42 @@ def __init__(self, limit: int = 0, *, verbosity: int = 0, xfail: List[str] = [])
117124
try:
118125
sched_getaffinity = os.sched_getaffinity
119126
except AttributeError:
120-
limit = 2
127+
# no support for affinity on OSX/Windows
128+
limit = cpu_count()
121129
else:
122130
# Note: only count CPUs we are allowed to use. It is a
123131
# major mistake to count *all* CPUs on the machine.
124132
limit = len(sched_getaffinity(0))
125133
self.limit = limit
134+
self.lf = lf
135+
self.ff = ff
126136
assert limit > 0
127137
self.xfail = set(xfail)
128138
self._note = None # type: Noter
129139
self.times1 = {} # type: Dict[str, float]
130140
self.times2 = {} # type: Dict[str, float]
131-
132-
def add(self, cmd: LazySubprocess) -> int:
141+
self.new_log = defaultdict(dict) # type: Dict[str, Dict[str, float]]
142+
self.sequential_tasks = set() # type: Set[str]
143+
144+
def load_log_file(self) -> Optional[List[Dict[str, Dict[str, Any]]]]:
145+
try:
146+
# get the last log
147+
with open(self.FULL_LOG_FILENAME) as fp:
148+
test_log = json.load(fp)
149+
except FileNotFoundError:
150+
test_log = []
151+
except json.JSONDecodeError:
152+
print('corrupt test log file {}'.format(self.FULL_LOG_FILENAME), file=sys.stderr)
153+
test_log = []
154+
return test_log
155+
156+
def add(self, cmd: LazySubprocess, sequential: bool = False) -> int:
133157
rv = len(self.queue)
158+
if cmd.name in (task.name for task in self.queue):
159+
sys.exit('Duplicate test name: {}'.format(cmd.name))
134160
self.queue.append(cmd)
161+
if sequential:
162+
self.sequential_tasks.add(cmd.name)
135163
return rv
136164

137165
def _start_next(self) -> None:
@@ -161,12 +189,14 @@ def _record_time(self, name: str, elapsed_time: float) -> None:
161189

162190
def _poll_current(self) -> Tuple[int, int]:
163191
while True:
164-
time.sleep(.05)
192+
time.sleep(.01)
165193
for pid in self.current:
166194
cmd = self.current[pid][1]
167195
code = cmd.process.poll()
168196
if code is not None:
169-
cmd.end_time = time.time()
197+
cmd.end_time = time.perf_counter()
198+
self.new_log['exit_code'][cmd.name] = code
199+
self.new_log['runtime'][cmd.name] = cmd.end_time - cmd.start_time
170200
return pid, code
171201

172202
def _wait_next(self) -> Tuple[List[str], int, int]:
@@ -239,22 +269,83 @@ def run(self) -> int:
239269
if self.verbosity == 0:
240270
self._note = Noter(len(self.queue))
241271
print('SUMMARY %d tasks selected' % len(self.queue))
272+
273+
def avg(lst: Iterable[float]) -> float:
274+
valid_items = [item for item in lst if item is not None]
275+
if not valid_items:
276+
# we don't know how long a new task takes
277+
# better err by putting it in front in case it is slow:
278+
# a fast task in front hurts performance less than a slow task in the back
279+
return float('inf')
280+
else:
281+
return sum(valid_items) / len(valid_items)
282+
283+
logs = self.load_log_file()
284+
if logs:
285+
times = {cmd.name: avg(log['runtime'].get(cmd.name, None) for log in logs)
286+
for cmd in self.queue}
287+
288+
def sort_function(cmd: LazySubprocess) -> Tuple[Any, int, float]:
289+
# longest tasks first
290+
runtime = -times[cmd.name]
291+
# sequential tasks go first by default
292+
sequential = -(cmd.name in self.sequential_tasks)
293+
if self.ff:
294+
# failed tasks first with -ff
295+
exit_code = -logs[-1]['exit_code'].get(cmd.name, 0)
296+
if not exit_code:
297+
# avoid interrupting parallel tasks with sequential in between
298+
# so either: seq failed, parallel failed, parallel passed, seq passed
299+
# or: parallel failed, seq failed, seq passed, parallel passed
300+
# I picked the first one arbitrarily, since no obvious pros/cons
301+
# in other words, among failed tasks, sequential should go before parallel,
302+
# and among successful tasks, sequential should go after parallel
303+
sequential = -sequential
304+
else:
305+
# ignore exit code without -ff
306+
exit_code = 0
307+
return exit_code, sequential, runtime
308+
self.queue = sorted(self.queue, key=sort_function)
309+
if self.lf:
310+
self.queue = [cmd for cmd in self.queue
311+
if logs[-1]['exit_code'].get(cmd.name, 0)]
312+
242313
sys.stdout.flush()
243314
# Failed tasks.
244315
all_failures = [] # type: List[str]
245316
# Number of test cases. Some tasks can involve multiple test cases.
246317
total_tests = 0
247318
# Number of failed test cases.
248319
total_failed_tests = 0
320+
running_sequential_task = False
249321
while self.current or self.next < len(self.queue):
250322
while len(self.current) < self.limit and self.next < len(self.queue):
323+
# only start next task if idle, or current and next tasks are both parallel
324+
if running_sequential_task:
325+
break
326+
if self.queue[self.next].name in self.sequential_tasks:
327+
if self.current:
328+
break
329+
else:
330+
running_sequential_task = True
251331
self._start_next()
252332
fails, tests, test_fails = self._wait_next()
333+
running_sequential_task = False
253334
all_failures += fails
254335
total_tests += tests
255336
total_failed_tests += test_fails
256337
if self.verbosity == 0:
257338
self._note.clear()
339+
340+
if self.new_log: # don't append empty log, it will corrupt the cache file
341+
# log only LOGSIZE most recent tests
342+
test_log = (self.load_log_file() + [self.new_log])[:self.LOGSIZE]
343+
try:
344+
with open(self.FULL_LOG_FILENAME, 'w') as fp:
345+
json.dump(test_log, fp, sort_keys=True, indent=4)
346+
except Exception as e:
347+
print('cannot save test log file:', e)
348+
258349
if all_failures:
259350
summary = 'SUMMARY %d/%d tasks and %d/%d tests failed' % (
260351
len(all_failures), len(self.queue), total_failed_tests, total_tests)
@@ -271,7 +362,6 @@ def run(self) -> int:
271362
len(self.queue), total_tests))
272363
print('*** OK ***')
273364
sys.stdout.flush()
274-
275365
return 0
276366

277367

‎runtests.py‎

Lines changed: 47 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -34,21 +34,25 @@ def get_versions(): # type: () -> typing.List[str]
3434
import itertools
3535
import os
3636
import re
37+
import json
3738

3839

3940
# Ideally, all tests would be `discover`able so that they can be driven
4041
# (and parallelized) by an external test driver.
4142

4243
class Driver:
4344

44-
def __init__(self, whitelist: List[str], blacklist: List[str],
45-
arglist: List[str], verbosity: int, parallel_limit: int,
45+
def __init__(self, *, whitelist: List[str], blacklist: List[str],
46+
lf: bool, ff: bool,
47+
arglist: List[str], pyt_arglist: List[str],
48+
verbosity: int, parallel_limit: int,
4649
xfail: List[str], coverage: bool) -> None:
4750
self.whitelist = whitelist
4851
self.blacklist = blacklist
4952
self.arglist = arglist
53+
self.pyt_arglist = pyt_arglist
5054
self.verbosity = verbosity
51-
self.waiter = Waiter(verbosity=verbosity, limit=parallel_limit, xfail=xfail)
55+
self.waiter = Waiter(verbosity=verbosity, limit=parallel_limit, xfail=xfail, lf=lf, ff=ff)
5256
self.versions = get_versions()
5357
self.cwd = os.getcwd()
5458
self.mypy = os.path.join(self.cwd, 'scripts', 'mypy')
@@ -107,7 +111,7 @@ def add_pytest(self, name: str, pytest_args: List[str], coverage: bool = False)
107111
else:
108112
args = [sys.executable, '-m', 'pytest'] + pytest_args
109113

110-
self.waiter.add(LazySubprocess(full_name, args, env=self.env))
114+
self.waiter.add(LazySubprocess(full_name, args, env=self.env), sequential=True)
111115

112116
def add_python(self, name: str, *args: str, cwd: Optional[str] = None) -> None:
113117
name = 'run %s' % name
@@ -155,7 +159,7 @@ def add_flake8(self, cwd: Optional[str] = None) -> None:
155159
name = 'lint'
156160
if not self.allow(name):
157161
return
158-
largs = ['flake8', '-j{}'.format(self.waiter.limit)]
162+
largs = ['flake8', '-j0']
159163
env = self.env
160164
self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env))
161165

@@ -175,7 +179,8 @@ def add_basic(driver: Driver) -> None:
175179

176180

177181
def add_selftypecheck(driver: Driver) -> None:
178-
driver.add_mypy_package('package mypy', 'mypy', '--config-file', 'mypy_self_check.ini')
182+
driver.add_mypy_package('package mypy nonstrict optional', 'mypy', '--config-file',
183+
'mypy_self_check.ini')
179184
driver.add_mypy_package('package mypy', 'mypy', '--config-file', 'mypy_strict_optional.ini')
180185

181186

@@ -209,8 +214,7 @@ def add_imports(driver: Driver) -> None:
209214

210215

211216
def add_pytest(driver: Driver) -> None:
212-
for f in PYTEST_FILES:
213-
driver.add_pytest(f, [f] + driver.arglist, True)
217+
driver.add_pytest('pytest', PYTEST_FILES + driver.arglist + driver.pyt_arglist, True)
214218

215219

216220
def add_myunit(driver: Driver) -> None:
@@ -297,7 +301,9 @@ def add_samples(driver: Driver) -> None:
297301

298302

299303
def usage(status: int) -> None:
300-
print('Usage: %s [-h | -v | -q | [-x] FILTER | -a ARG] ... [-- FILTER ...]' % sys.argv[0])
304+
print('Usage: %s [-h | -v | -q | --lf | --ff | [-x] FILTER | -a ARG | -p ARG]'
305+
'... [-- FILTER ...]'
306+
% sys.argv[0])
301307
print()
302308
print('Run mypy tests. If given no arguments, run all tests.')
303309
print()
@@ -309,9 +315,12 @@ def usage(status: int) -> None:
309315
print('Options:')
310316
print(' -h, --help show this help')
311317
print(' -v, --verbose increase driver verbosity')
318+
print(' --lf rerun only the tests that failed at the last run')
319+
print(' --ff run all tests but run the last failures first')
312320
print(' -q, --quiet decrease driver verbosity')
313321
print(' -jN run N tasks at once (default: one per CPU)')
314322
print(' -a, --argument ARG pass an argument to myunit tasks')
323+
print(' -p, --pytest_arg ARG pass an argument to pytest tasks')
315324
print(' (-v: verbose; glob pattern: filter by test name)')
316325
print(' -l, --list list included tasks (after filtering) and exit')
317326
print(' FILTER include tasks matching FILTER')
@@ -337,20 +346,25 @@ def sanity() -> None:
337346

338347

339348
def main() -> None:
349+
import time
350+
t0 = time.perf_counter()
340351
sanity()
341352

342353
verbosity = 0
343354
parallel_limit = 0
344355
whitelist = [] # type: List[str]
345356
blacklist = [] # type: List[str]
346357
arglist = [] # type: List[str]
358+
pyt_arglist = [] # type: List[str]
359+
lf = False
360+
ff = False
347361
list_only = False
348362
coverage = False
349363

350364
allow_opts = True
351365
curlist = whitelist
352366
for a in sys.argv[1:]:
353-
if curlist is not arglist and allow_opts and a.startswith('-'):
367+
if not (curlist is arglist or curlist is pyt_arglist) and allow_opts and a.startswith('-'):
354368
if curlist is not whitelist:
355369
break
356370
if a == '--':
@@ -368,6 +382,14 @@ def main() -> None:
368382
curlist = blacklist
369383
elif a == '-a' or a == '--argument':
370384
curlist = arglist
385+
elif a == '-p' or a == '--pytest_arg':
386+
curlist = pyt_arglist
387+
# will also pass this option to pytest
388+
elif a == '--lf':
389+
lf = True
390+
# will also pass this option to pytest
391+
elif a == '--ff':
392+
ff = True
371393
elif a == '-l' or a == '--list':
372394
list_only = True
373395
elif a == '-c' or a == '--coverage':
@@ -383,35 +405,46 @@ def main() -> None:
383405
sys.exit('-x must be followed by a filter')
384406
if curlist is arglist:
385407
sys.exit('-a must be followed by an argument')
408+
if curlist is pyt_arglist:
409+
sys.exit('-p must be followed by an argument')
410+
if lf and ff:
411+
sys.exit('use either --lf or --ff, not both')
386412
# empty string is a substring of all names
387413
if not whitelist:
388414
whitelist.append('')
415+
if lf:
416+
pyt_arglist.append('--lf')
417+
if ff:
418+
pyt_arglist.append('--ff')
389419

390-
driver = Driver(whitelist=whitelist, blacklist=blacklist, arglist=arglist,
391-
verbosity=verbosity, parallel_limit=parallel_limit, xfail=[], coverage=coverage)
420+
driver = Driver(whitelist=whitelist, blacklist=blacklist, lf=lf, ff=ff,
421+
arglist=arglist, pyt_arglist=pyt_arglist, verbosity=verbosity,
422+
parallel_limit=parallel_limit, xfail=[], coverage=coverage)
392423

393424
driver.prepend_path('PATH', [join(driver.cwd, 'scripts')])
394425
driver.prepend_path('MYPYPATH', [driver.cwd])
395426
driver.prepend_path('PYTHONPATH', [driver.cwd])
396427
driver.prepend_path('PYTHONPATH', [join(driver.cwd, 'lib-typing', v) for v in driver.versions])
397428

429+
driver.add_flake8()
430+
add_pytest(driver)
398431
add_pythoneval(driver)
399432
add_cmdline(driver)
400433
add_basic(driver)
401434
add_selftypecheck(driver)
402-
add_pytest(driver)
403435
add_myunit(driver)
404436
add_imports(driver)
405437
add_stubs(driver)
406438
add_stdlibsamples(driver)
407439
add_samples(driver)
408-
driver.add_flake8()
409440

410441
if list_only:
411442
driver.list_tasks()
412443
return
413444

414445
exit_code = driver.waiter.run()
446+
t1 = time.perf_counter()
447+
print('total runtime:', t1 - t0, 'sec')
415448

416449
if verbosity >= 1:
417450
times = driver.waiter.times2 if verbosity >= 2 else driver.waiter.times1

0 commit comments

Comments
 (0)