Source code

Revision control

Other Tools

1
2
# This Source Code Form is subject to the terms of the Mozilla Public
3
# License, v. 2.0. If a copy of the MPL was not distributed with this
4
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
5
6
"""output formats for Talos"""
7
from __future__ import absolute_import
8
9
from talos import filter
10
# NOTE: we have a circular dependency with output.py when we import results
11
import simplejson as json
12
from talos import utils
13
from mozlog import get_proxy_logger
14
15
LOG = get_proxy_logger()
16
17
18
class Output(object):
19
"""abstract base class for Talos output"""
20
21
@classmethod
22
def check(cls, urls):
23
"""check to ensure that the urls are valid"""
24
25
def __init__(self, results, tsresult_class):
26
"""
27
- results : TalosResults instance
28
- tsresult_class : Results class
29
"""
30
self.results = results
31
self.tsresult_class = tsresult_class
32
33
def __call__(self):
34
suites = []
35
test_results = {
36
'framework': {
37
'name': self.results.results[0].framework,
38
},
39
'suites': suites,
40
}
41
42
for test in self.results.results:
43
# serialize test results
44
tsresult = None
45
if not test.using_xperf:
46
subtests = []
47
suite = {
48
'name': test.name(),
49
'extraOptions': self.results.extra_options or [],
50
'subtests': subtests
51
}
52
53
suites.append(suite)
54
vals = []
55
replicates = {}
56
57
# TODO: counters!!!! we don't have any, but they suffer the same
58
for result in test.results:
59
# XXX this will not work for manifests which list
60
# the same page name twice. It also ignores cycles
61
for page, val in result.raw_values():
62
if page == 'NULL':
63
page = test.name()
64
if tsresult is None:
65
tsresult = r = self.tsresult_class()
66
r.results = [{'index': 0, 'page': test.name(),
67
'runs': val}]
68
else:
69
r = tsresult.results[0]
70
if r['page'] == test.name():
71
r['runs'].extend(val)
72
replicates.setdefault(page, []).extend(val)
73
74
tresults = [tsresult] if tsresult else test.results
75
76
# Merge results for the same page when using cycle > 1
77
merged_results = {}
78
for result in tresults:
79
results = []
80
for r in result.results:
81
page = r['page']
82
if page in merged_results:
83
merged_results[page]['runs'].extend(r['runs'])
84
else:
85
merged_results[page] = r
86
results.append(r)
87
# override the list of page results for each run
88
result.results = results
89
90
for result in tresults:
91
filtered_results = \
92
result.values(suite['name'],
93
test.test_config['filters'])
94
vals.extend([[i['value'], j] for i, j in filtered_results])
95
subtest_index = 0
96
for val, page in filtered_results:
97
if page == 'NULL':
98
# no real subtests
99
page = test.name()
100
subtest = {
101
'name': page,
102
'value': val['filtered'],
103
'replicates': replicates[page],
104
}
105
# if results are from a comparison test i.e. perf-reftest, it will also
106
# contain replicates for 'base' and 'reference'; we wish to keep those
107
# to reference; actual results were calculated as the difference of those
108
base_runs = result.results[subtest_index].get('base_runs', None)
109
ref_runs = result.results[subtest_index].get('ref_runs', None)
110
if base_runs and ref_runs:
111
subtest['base_replicates'] = base_runs
112
subtest['ref_replicates'] = ref_runs
113
114
subtests.append(subtest)
115
subtest_index += 1
116
117
if test.test_config.get('lower_is_better') is not None:
118
subtest['lowerIsBetter'] = \
119
test.test_config['lower_is_better']
120
if test.test_config.get('alert_threshold') is not None:
121
subtest['alertThreshold'] = \
122
test.test_config['alert_threshold']
123
if test.test_config.get('subtest_alerts') is not None:
124
subtest['shouldAlert'] = \
125
test.test_config['subtest_alerts']
126
if test.test_config.get('alert_threshold') is not None:
127
subtest['alertThreshold'] = \
128
test.test_config['alert_threshold']
129
if test.test_config.get('unit'):
130
subtest['unit'] = test.test_config['unit']
131
132
# if there is more than one subtest, calculate a summary result
133
if len(subtests) > 1:
134
suite['value'] = self.construct_results(
135
vals, testname=test.name())
136
if test.test_config.get('lower_is_better') is not None:
137
suite['lowerIsBetter'] = \
138
test.test_config['lower_is_better']
139
if test.test_config.get('alert_threshold') is not None:
140
suite['alertThreshold'] = \
141
test.test_config['alert_threshold']
142
143
# counters results_aux data
144
counter_subtests = []
145
for cd in test.all_counter_results:
146
for name, vals in cd.items():
147
# We want to add the xperf data as talos_counters
148
# exclude counters whose values are tuples (bad for
149
# graphserver)
150
if len(vals) > 0 and isinstance(vals[0], list):
151
continue
152
153
# mainthread IO is a list of filenames and accesses, we do
154
# not report this as a counter
155
if 'mainthreadio' in name:
156
continue
157
158
# responsiveness has it's own metric, not the mean
159
# TODO: consider doing this for all counters
160
if 'responsiveness' is name:
161
subtest = {
162
'name': name,
163
'value': filter.responsiveness_Metric(vals)
164
}
165
counter_subtests.append(subtest)
166
continue
167
168
subtest = {
169
'name': name,
170
'value': 0.0,
171
}
172
counter_subtests.append(subtest)
173
174
if test.using_xperf:
175
if len(vals) > 0:
176
subtest['value'] = vals[0]
177
else:
178
# calculate mean value
179
if len(vals) > 0:
180
varray = [float(v) for v in vals]
181
subtest['value'] = filter.mean(varray)
182
if counter_subtests:
183
suites.append({'name': test.name(),
184
'extraOptions': self.results.extra_options or [],
185
'subtests': counter_subtests})
186
return test_results
187
188
def output(self, results, results_url):
189
"""output to the a file if results_url starts with file://
190
- results : json instance
191
- results_url : file:// URL
192
"""
193
194
# parse the results url
195
results_url_split = utils.urlsplit(results_url)
196
results_scheme, results_server, results_path, _, _ = results_url_split
197
198
if results_scheme in ('http', 'https'):
199
self.post(results, results_server, results_path, results_scheme)
200
elif results_scheme == 'file':
201
with open(results_path, 'w') as f:
202
for result in results:
203
f.write("%s\n" % result)
204
else:
205
raise NotImplementedError(
206
"%s: %s - only http://, https://, and file:// supported"
207
% (self.__class__.__name__, results_url)
208
)
209
210
# This is the output that treeherder expects to find when parsing the
211
# log file
212
if 'geckoProfile' not in self.results.extra_options:
213
LOG.info("PERFHERDER_DATA: %s" % json.dumps(results,
214
ignore_nan=True))
215
if results_scheme in ('file'):
216
json.dump(results, open(results_path, 'w'), indent=2,
217
sort_keys=True, ignore_nan=True)
218
219
def post(self, results, server, path, scheme):
220
raise NotImplementedError("Abstract base class")
221
222
@classmethod
223
def shortName(cls, name):
224
"""short name for counters"""
225
names = {"% Processor Time": "%cpu",
226
"XRes": "xres"}
227
return names.get(name, name)
228
229
@classmethod
230
def isMemoryMetric(cls, resultName):
231
"""returns if the result is a memory metric"""
232
memory_metric = ['xres'] # measured in bytes
233
return bool([i for i in memory_metric if i in resultName])
234
235
@classmethod
236
def v8_Metric(cls, val_list):
237
results = [i for i, j in val_list]
238
score = 100 * filter.geometric_mean(results)
239
return score
240
241
@classmethod
242
def JS_Metric(cls, val_list):
243
"""v8 benchmark score"""
244
results = [i for i, j in val_list]
245
return sum(results)
246
247
@classmethod
248
def benchmark_score(cls, val_list):
249
"""
250
benchmark_score: ares6/jetstream self reported as 'geomean'
251
"""
252
results = [i for i, j in val_list if j == 'geomean']
253
return filter.mean(results)
254
255
@classmethod
256
def stylebench_score(cls, val_list):
257
"""
259
"""
260
correctionFactor = 3
261
results = [i for i, j in val_list]
262
263
# stylebench has 5 tests, each of these are made of up 5 subtests
264
#
265
# * Adding classes.
266
# * Removing classes.
267
# * Mutating attributes.
268
# * Adding leaf elements.
269
# * Removing leaf elements.
270
#
271
# which are made of two subtests each (sync/async) and repeated 5 times
272
# each, thus, the list here looks like:
273
#
274
# [Test name/Adding classes - 0/ Sync; <x>]
275
# [Test name/Adding classes - 0/ Async; <y>]
276
# [Test name/Adding classes - 0; <x> + <y>]
277
# [Test name/Removing classes - 0/ Sync; <x>]
278
# [Test name/Removing classes - 0/ Async; <y>]
279
# [Test name/Removing classes - 0; <x> + <y>]
280
# ...
281
# [Test name/Adding classes - 1 / Sync; <x>]
282
# [Test name/Adding classes - 1 / Async; <y>]
283
# [Test name/Adding classes - 1 ; <x> + <y>]
284
# ...
285
# [Test name/Removing leaf elements - 4; <x> + <y>]
286
# [Test name; <sum>] <- This is what we want.
287
#
288
# So, 5 (subtests) *
289
# 5 (repetitions) *
290
# 3 (entries per repetition (sync/async/sum)) =
291
# 75 entries for test before the sum.
292
#
293
# We receive 76 entries per test, which ads up to 380. We want to use
294
# the 5 test entries, not the rest.
295
if len(results) != 380:
296
raise Exception("StyleBench has 380 entries, found: %s instead" % len(results))
297
298
results = results[75::76]
299
score = 60 * 1000 / filter.geometric_mean(results) / correctionFactor
300
return score
301
302
def construct_results(self, vals, testname):
303
if 'responsiveness' in testname:
304
return filter.responsiveness_Metric([val for (val, page) in vals])
305
elif testname.startswith('v8_7'):
306
return self.v8_Metric(vals)
307
elif testname.startswith('kraken'):
308
return self.JS_Metric(vals)
309
elif testname.startswith('ares6'):
310
return self.benchmark_score(vals)
311
elif testname.startswith('jetstream'):
312
return self.benchmark_score(vals)
313
elif testname.startswith('speedometer'):
314
return self.speedometer_score(vals)
315
elif testname.startswith('stylebench'):
316
return self.stylebench_score(vals)
317
elif len(vals) > 1:
318
return filter.geometric_mean([i for i, j in vals])
319
else:
320
return filter.mean([i for i, j in vals])