Source code

Revision control

Other Tools

1
#!/usr/bin/env python
2
# This Source Code Form is subject to the terms of the Mozilla Public
3
# License, v. 2.0. If a copy of the MPL was not distributed with this
4
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
5
6
# run-tests.py -- Python harness for GDB SpiderMonkey support
7
8
import os
9
import re
10
import subprocess
11
import sys
12
13
# From this directory:
14
import progressbar
15
from taskpool import TaskPool, get_cpu_count
16
17
18
def _relpath(path, start=None):
19
# Backported from Python 3.1 posixpath.py
20
21
"""Return a relative version of a path"""
22
23
if not path:
24
raise ValueError("no path specified")
25
26
if start is None:
27
start = os.curdir
28
29
start_list = os.path.abspath(start).split(os.sep)
30
path_list = os.path.abspath(path).split(os.sep)
31
32
# Work out how much of the filepath is shared by start and path.
33
i = len(os.path.commonprefix([start_list, path_list]))
34
35
rel_list = [os.pardir] * (len(start_list)-i) + path_list[i:]
36
if not rel_list:
37
return os.curdir
38
return os.path.join(*rel_list)
39
40
41
os.path.relpath = _relpath
42
43
# Characters that need to be escaped when used in shell words.
44
shell_need_escapes = re.compile('[^\w\d%+,-./:=@\'"]', re.DOTALL)
45
# Characters that need to be escaped within double-quoted strings.
46
shell_dquote_escapes = re.compile('[^\w\d%+,-./:=@"]', re.DOTALL)
47
48
49
def make_shell_cmd(l):
50
def quote(s):
51
if shell_need_escapes.search(s):
52
if s.find("'") < 0:
53
return "'" + s + "'"
54
return '"' + shell_dquote_escapes.sub('\\g<0>', s) + '"'
55
return s
56
57
return ' '.join([quote(_) for _ in l])
58
59
60
# An instance of this class collects the lists of passing, failing, and
61
# timing-out tests, runs the progress bar, and prints a summary at the end.
62
class Summary(object):
63
64
class SummaryBar(progressbar.ProgressBar):
65
def __init__(self, limit):
66
super(Summary.SummaryBar, self).__init__('', limit, 24)
67
68
def start(self):
69
self.label = '[starting ]'
70
self.update(0)
71
72
def counts(self, run, failures, timeouts):
73
self.label = '[%4d|%4d|%4d|%4d]' % (run - failures, failures, timeouts, run)
74
self.update(run)
75
76
def __init__(self, num_tests):
77
self.run = 0
78
self.failures = [] # kind of judgemental; "unexpecteds"?
79
self.timeouts = []
80
if not OPTIONS.hide_progress:
81
self.bar = Summary.SummaryBar(num_tests)
82
83
# Progress bar control.
84
def start(self):
85
if not OPTIONS.hide_progress:
86
self.bar.start()
87
88
def update(self):
89
if not OPTIONS.hide_progress:
90
self.bar.counts(self.run, len(self.failures), len(self.timeouts))
91
# Call 'thunk' to show some output, while getting the progress bar out of the way.
92
93
def interleave_output(self, thunk):
94
if not OPTIONS.hide_progress:
95
self.bar.clear()
96
thunk()
97
self.update()
98
99
def passed(self, test):
100
self.run += 1
101
self.update()
102
103
def failed(self, test):
104
self.run += 1
105
self.failures.append(test)
106
self.update()
107
108
def timeout(self, test):
109
self.run += 1
110
self.timeouts.append(test)
111
self.update()
112
113
def finish(self):
114
if not OPTIONS.hide_progress:
115
self.bar.finish()
116
117
if self.failures:
118
119
print("tests failed:")
120
for test in self.failures:
121
test.show(sys.stdout)
122
123
if OPTIONS.worklist:
124
try:
125
with open(OPTIONS.worklist) as out:
126
for test in self.failures:
127
out.write(test.name + '\n')
128
except IOError as err:
129
sys.stderr.write("Error writing worklist file '%s': %s"
130
% (OPTIONS.worklist, err))
131
sys.exit(1)
132
133
if OPTIONS.write_failures:
134
try:
135
with open(OPTIONS.write_failures, "w") as out:
136
for test in self.failures:
137
test.show(out)
138
except IOError as err:
139
sys.stderr.write("Error writing worklist file '%s': %s"
140
% (OPTIONS.write_failures, err))
141
sys.exit(1)
142
143
if self.timeouts:
144
print("tests timed out:")
145
for test in self.timeouts:
146
test.show(sys.stdout)
147
148
if self.failures or self.timeouts:
149
sys.exit(2)
150
151
152
class Test(TaskPool.Task):
153
def __init__(self, path, summary):
154
super(Test, self).__init__()
155
self.test_path = path # path to .py test file
156
self.summary = summary
157
158
# test.name is the name of the test relative to the top of the test
159
# directory. This is what we use to report failures and timeouts,
160
# and when writing test lists.
161
self.name = os.path.relpath(self.test_path, OPTIONS.testdir)
162
163
self.stdout = ''
164
self.stderr = ''
165
self.returncode = None
166
167
def cmd(self):
168
testlibdir = os.path.normpath(os.path.join(OPTIONS.testdir, '..', 'lib-for-tests'))
169
return [OPTIONS.gdb_executable,
170
'-nw', # Don't create a window (unnecessary?)
171
'-nx', # Don't read .gdbinit.
172
'--ex', 'add-auto-load-safe-path %s' % (OPTIONS.bindir,),
173
'--ex', 'set env LD_LIBRARY_PATH %s' % (OPTIONS.bindir,),
174
'--ex', 'file %s' % (os.path.join(OPTIONS.bindir, 'gdb-tests'),),
175
'--eval-command', 'python testlibdir=%r' % (testlibdir,),
176
'--eval-command', 'python testscript=%r' % (self.test_path,),
177
'--eval-command', 'python exec(open(%r).read())' % os.path.join(testlibdir,
178
'catcher.py')]
179
180
def start(self, pipe, deadline):
181
super(Test, self).start(pipe, deadline)
182
if OPTIONS.show_cmd:
183
self.summary.interleave_output(lambda: self.show_cmd(sys.stdout))
184
185
def onStdout(self, text):
186
self.stdout += text
187
188
def onStderr(self, text):
189
self.stderr += text
190
191
def onFinished(self, returncode):
192
self.returncode = returncode
193
if OPTIONS.show_output:
194
self.summary.interleave_output(lambda: self.show_output(sys.stdout))
195
if returncode != 0:
196
self.summary.failed(self)
197
else:
198
self.summary.passed(self)
199
200
def onTimeout(self):
201
self.summary.timeout(self)
202
203
def show_cmd(self, out):
204
out.write("Command: %s\n" % (make_shell_cmd(self.cmd()),))
205
206
def show_output(self, out):
207
if self.stdout:
208
out.write('Standard output:')
209
out.write('\n' + self.stdout + '\n')
210
if self.stderr:
211
out.write('Standard error:')
212
out.write('\n' + self.stderr + '\n')
213
214
def show(self, out):
215
out.write(self.name + '\n')
216
if OPTIONS.write_failure_output:
217
self.show_cmd(out)
218
self.show_output(out)
219
out.write('GDB exit code: %r\n' % (self.returncode,))
220
221
222
def find_tests(dir, substring=None):
223
ans = []
224
for dirpath, _, filenames in os.walk(dir):
225
if dirpath == '.':
226
continue
227
for filename in filenames:
228
if not filename.endswith('.py'):
229
continue
230
test = os.path.join(dirpath, filename)
231
if substring is None or substring in os.path.relpath(test, dir):
232
ans.append(test)
233
return ans
234
235
236
def build_test_exec(builddir):
237
subprocess.check_call(['make'], cwd=builddir)
238
239
240
def run_tests(tests, summary):
241
jobs = OPTIONS.workercount
242
# python 3.3 fixed a bug with concurrently writing .pyc files.
244
embedded_version = subprocess.check_output([
245
OPTIONS.gdb_executable,
246
'--batch',
247
'--ex', 'python import sys; print(sys.hexversion)'
248
]).decode('ascii').strip()
249
if hex(int(embedded_version)) < '0x3030000':
250
jobs = 1
251
252
pool = TaskPool(tests, job_limit=jobs, timeout=OPTIONS.timeout)
253
pool.run_all()
254
255
256
OPTIONS = None
257
258
259
def main(argv):
260
global OPTIONS
261
script_path = os.path.abspath(__file__)
262
script_dir = os.path.dirname(script_path)
263
264
# OBJDIR is a standalone SpiderMonkey build directory. This is where we
265
# find the SpiderMonkey shared library to link against.
266
#
267
# The [TESTS] optional arguments are paths of test files relative
268
# to the jit-test/tests directory.
269
from optparse import OptionParser
270
op = OptionParser(usage='%prog [options] OBJDIR [TESTS...]')
271
op.add_option('-s', '--show-cmd', dest='show_cmd', action='store_true',
272
help='show GDB shell command run')
273
op.add_option('-o', '--show-output', dest='show_output', action='store_true',
274
help='show output from GDB')
275
op.add_option('-x', '--exclude', dest='exclude', action='append',
276
help='exclude given test dir or path')
277
op.add_option('-t', '--timeout', dest='timeout', type=float, default=150.0,
278
help='set test timeout in seconds')
279
op.add_option('-j', '--worker-count', dest='workercount', type=int,
280
help='Run [WORKERCOUNT] tests at a time')
281
op.add_option('--no-progress', dest='hide_progress', action='store_true',
282
help='hide progress bar')
283
op.add_option('--worklist', dest='worklist', metavar='FILE',
284
help='Read tests to run from [FILE] (or run all if [FILE] not found);\n'
285
'write failures back to [FILE]')
286
op.add_option('-r', '--read-tests', dest='read_tests', metavar='FILE',
287
help='Run test files listed in [FILE]')
288
op.add_option('-w', '--write-failures', dest='write_failures', metavar='FILE',
289
help='Write failing tests to [FILE]')
290
op.add_option('--write-failure-output', dest='write_failure_output', action='store_true',
291
help='With --write-failures=FILE, additionally write the output of failed '
292
'tests to [FILE]')
293
op.add_option('--gdb', dest='gdb_executable', metavar='EXECUTABLE', default='gdb',
294
help='Run tests with [EXECUTABLE], rather than plain \'gdb\'.')
295
op.add_option('--srcdir', dest='srcdir',
296
default=os.path.abspath(os.path.join(script_dir, '..')),
297
help='Use SpiderMonkey sources in [SRCDIR].')
298
op.add_option('--testdir', dest='testdir', default=os.path.join(script_dir, 'tests'),
299
help='Find tests in [TESTDIR].')
300
op.add_option('--builddir', dest='builddir',
301
help='Build test executable from [BUILDDIR].')
302
op.add_option('--bindir', dest='bindir',
303
help='Run test executable from [BINDIR].')
304
(OPTIONS, args) = op.parse_args(argv)
305
if len(args) < 1:
306
op.error('missing OBJDIR argument')
307
OPTIONS.objdir = os.path.abspath(args[0])
308
309
test_args = args[1:]
310
311
if not OPTIONS.workercount:
312
OPTIONS.workercount = get_cpu_count()
313
314
# Compute defaults for OPTIONS.builddir and OPTIONS.bindir now, since we've
315
# computed OPTIONS.objdir.
316
if not OPTIONS.builddir:
317
OPTIONS.builddir = os.path.join(OPTIONS.objdir, 'js', 'src', 'gdb')
318
if not OPTIONS.bindir:
319
OPTIONS.bindir = os.path.join(OPTIONS.objdir, 'dist', 'bin')
320
321
test_set = set()
322
323
# All the various sources of test names accumulate.
324
if test_args:
325
for arg in test_args:
326
test_set.update(find_tests(OPTIONS.testdir, arg))
327
if OPTIONS.worklist:
328
try:
329
with open(OPTIONS.worklist) as f:
330
for line in f:
331
test_set.update(os.path.join(OPTIONS.testdir, line.strip('\n')))
332
except IOError:
333
# With worklist, a missing file means to start the process with
334
# the complete list of tests.
335
sys.stderr.write("Couldn't read worklist file '%s'; running all tests\n"
336
% (OPTIONS.worklist,))
337
test_set = set(find_tests(OPTIONS.testdir))
338
if OPTIONS.read_tests:
339
try:
340
with open(OPTIONS.read_tests) as f:
341
for line in f:
342
test_set.update(os.path.join(OPTIONS.testdir, line.strip('\n')))
343
except IOError as err:
344
sys.stderr.write("Error trying to read test file '%s': %s\n"
345
% (OPTIONS.read_tests, err))
346
sys.exit(1)
347
348
# If none of the above options were passed, and no tests were listed
349
# explicitly, use the complete set.
350
if not test_args and not OPTIONS.worklist and not OPTIONS.read_tests:
351
test_set = set(find_tests(OPTIONS.testdir))
352
353
if OPTIONS.exclude:
354
exclude_set = set()
355
for exclude in OPTIONS.exclude:
356
exclude_set.update(find_tests(OPTIONS.testdir, exclude))
357
test_set -= exclude_set
358
359
if not test_set:
360
sys.stderr.write("No tests found matching command line arguments.\n")
361
sys.exit(1)
362
363
summary = Summary(len(test_set))
364
test_list = [Test(_, summary) for _ in sorted(test_set)]
365
366
# Build the test executable from all the .cpp files found in the test
367
# directory tree.
368
try:
369
build_test_exec(OPTIONS.builddir)
370
except subprocess.CalledProcessError as err:
371
sys.stderr.write("Error building test executable: %s\n" % (err,))
372
sys.exit(1)
373
374
# Run the tests.
375
try:
376
summary.start()
377
run_tests(test_list, summary)
378
summary.finish()
379
except OSError as err:
380
sys.stderr.write("Error running tests: %s\n" % (err,))
381
sys.exit(1)
382
383
sys.exit(0)
384
385
386
if __name__ == '__main__':
387
main(sys.argv[1:])