Source code

Revision control

Other Tools

1
2
# This Source Code Form is subject to the terms of the Mozilla Public
3
# License, v. 2.0. If a copy of the MPL was not distributed with this
4
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
5
6
"""output formats for Talos"""
7
from __future__ import absolute_import
8
9
from talos import filter
10
# NOTE: we have a circular dependency with output.py when we import results
11
import simplejson as json
12
from talos import utils
13
from mozlog import get_proxy_logger
14
15
LOG = get_proxy_logger()
16
17
18
class Output(object):
19
"""abstract base class for Talos output"""
20
21
@classmethod
22
def check(cls, urls):
23
"""check to ensure that the urls are valid"""
24
25
def __init__(self, results, tsresult_class):
26
"""
27
- results : TalosResults instance
28
- tsresult_class : Results class
29
"""
30
self.results = results
31
self.tsresult_class = tsresult_class
32
33
def __call__(self):
34
suites = []
35
test_results = {
36
'framework': {
37
'name': self.results.results[0].framework,
38
},
39
'suites': suites,
40
}
41
42
for test in self.results.results:
43
# serialize test results
44
tsresult = None
45
if not test.using_xperf:
46
subtests = []
47
suite = {
48
'name': test.name(),
49
'extraOptions': self.results.extra_options or [],
50
'subtests': subtests,
51
'shouldAlert': test.test_config.get('suite_should_alert', True)
52
}
53
54
suites.append(suite)
55
vals = []
56
replicates = {}
57
58
# TODO: counters!!!! we don't have any, but they suffer the same
59
for result in test.results:
60
# XXX this will not work for manifests which list
61
# the same page name twice. It also ignores cycles
62
for page, val in result.raw_values():
63
if page == 'NULL':
64
page = test.name()
65
if tsresult is None:
66
tsresult = r = self.tsresult_class()
67
r.results = [{'index': 0, 'page': test.name(),
68
'runs': val}]
69
else:
70
r = tsresult.results[0]
71
if r['page'] == test.name():
72
r['runs'].extend(val)
73
replicates.setdefault(page, []).extend(val)
74
75
tresults = [tsresult] if tsresult else test.results
76
77
# Merge results for the same page when using cycle > 1
78
merged_results = {}
79
for result in tresults:
80
results = []
81
for r in result.results:
82
page = r['page']
83
if page in merged_results:
84
merged_results[page]['runs'].extend(r['runs'])
85
else:
86
merged_results[page] = r
87
results.append(r)
88
# override the list of page results for each run
89
result.results = results
90
91
for result in tresults:
92
filtered_results = \
93
result.values(suite['name'],
94
test.test_config['filters'])
95
vals.extend([[i['value'], j] for i, j in filtered_results])
96
subtest_index = 0
97
for val, page in filtered_results:
98
if page == 'NULL':
99
# no real subtests
100
page = test.name()
101
subtest = {
102
'name': page,
103
'value': val['filtered'],
104
'replicates': replicates[page],
105
}
106
# if results are from a comparison test i.e. perf-reftest, it will also
107
# contain replicates for 'base' and 'reference'; we wish to keep those
108
# to reference; actual results were calculated as the difference of those
109
base_runs = result.results[subtest_index].get('base_runs', None)
110
ref_runs = result.results[subtest_index].get('ref_runs', None)
111
if base_runs and ref_runs:
112
subtest['base_replicates'] = base_runs
113
subtest['ref_replicates'] = ref_runs
114
115
subtests.append(subtest)
116
subtest_index += 1
117
118
if test.test_config.get('lower_is_better') is not None:
119
subtest['lowerIsBetter'] = \
120
test.test_config['lower_is_better']
121
if test.test_config.get('alert_threshold') is not None:
122
subtest['alertThreshold'] = \
123
test.test_config['alert_threshold']
124
if test.test_config.get('subtest_alerts') is not None:
125
subtest['shouldAlert'] = \
126
test.test_config['subtest_alerts']
127
if test.test_config.get('alert_threshold') is not None:
128
subtest['alertThreshold'] = \
129
test.test_config['alert_threshold']
130
if test.test_config.get('unit'):
131
subtest['unit'] = test.test_config['unit']
132
133
# if there is more than one subtest, calculate a summary result
134
if len(subtests) > 1:
135
suite['value'] = self.construct_results(
136
vals, testname=test.name())
137
if test.test_config.get('lower_is_better') is not None:
138
suite['lowerIsBetter'] = \
139
test.test_config['lower_is_better']
140
if test.test_config.get('alert_threshold') is not None:
141
suite['alertThreshold'] = \
142
test.test_config['alert_threshold']
143
144
# counters results_aux data
145
counter_subtests = []
146
for cd in test.all_counter_results:
147
for name, vals in cd.items():
148
# We want to add the xperf data as talos_counters
149
# exclude counters whose values are tuples (bad for
150
# graphserver)
151
if len(vals) > 0 and isinstance(vals[0], list):
152
continue
153
154
# mainthread IO is a list of filenames and accesses, we do
155
# not report this as a counter
156
if 'mainthreadio' in name:
157
continue
158
159
# responsiveness has it's own metric, not the mean
160
# TODO: consider doing this for all counters
161
if 'responsiveness' == name:
162
subtest = {
163
'name': name,
164
'value': filter.responsiveness_Metric(vals)
165
}
166
counter_subtests.append(subtest)
167
continue
168
169
subtest = {
170
'name': name,
171
'value': 0.0,
172
}
173
counter_subtests.append(subtest)
174
175
if test.using_xperf:
176
if len(vals) > 0:
177
subtest['value'] = vals[0]
178
else:
179
# calculate mean value
180
if len(vals) > 0:
181
varray = [float(v) for v in vals]
182
subtest['value'] = filter.mean(varray)
183
if counter_subtests:
184
suites.append({'name': test.name(),
185
'extraOptions': self.results.extra_options or [],
186
'subtests': counter_subtests,
187
'shouldAlert': test.test_config.get('suite_should_alert', True)})
188
return test_results
189
190
def output(self, results, results_url):
191
"""output to the a file if results_url starts with file://
192
- results : json instance
193
- results_url : file:// URL
194
"""
195
196
# parse the results url
197
results_url_split = utils.urlsplit(results_url)
198
results_scheme, results_server, results_path, _, _ = results_url_split
199
200
if results_scheme in ('http', 'https'):
201
self.post(results, results_server, results_path, results_scheme)
202
elif results_scheme == 'file':
203
with open(results_path, 'w') as f:
204
for result in results:
205
f.write("%s\n" % result)
206
else:
207
raise NotImplementedError(
208
"%s: %s - only http://, https://, and file:// supported"
209
% (self.__class__.__name__, results_url)
210
)
211
212
# This is the output that treeherder expects to find when parsing the
213
# log file
214
if 'geckoProfile' not in self.results.extra_options:
215
LOG.info("PERFHERDER_DATA: %s" % json.dumps(results,
216
ignore_nan=True))
217
if results_scheme in ('file'):
218
json.dump(results, open(results_path, 'w'), indent=2,
219
sort_keys=True, ignore_nan=True)
220
221
def post(self, results, server, path, scheme):
222
raise NotImplementedError("Abstract base class")
223
224
@classmethod
225
def shortName(cls, name):
226
"""short name for counters"""
227
names = {"% Processor Time": "%cpu",
228
"XRes": "xres"}
229
return names.get(name, name)
230
231
@classmethod
232
def isMemoryMetric(cls, resultName):
233
"""returns if the result is a memory metric"""
234
memory_metric = ['xres'] # measured in bytes
235
return bool([i for i in memory_metric if i in resultName])
236
237
@classmethod
238
def v8_Metric(cls, val_list):
239
results = [i for i, j in val_list]
240
score = 100 * filter.geometric_mean(results)
241
return score
242
243
@classmethod
244
def JS_Metric(cls, val_list):
245
"""v8 benchmark score"""
246
results = [i for i, j in val_list]
247
return sum(results)
248
249
@classmethod
250
def benchmark_score(cls, val_list):
251
"""
252
benchmark_score: ares6/jetstream self reported as 'geomean'
253
"""
254
results = [i for i, j in val_list if j == 'geomean']
255
return filter.mean(results)
256
257
@classmethod
258
def stylebench_score(cls, val_list):
259
"""
261
"""
262
correctionFactor = 3
263
results = [i for i, j in val_list]
264
265
# stylebench has 5 tests, each of these are made of up 5 subtests
266
#
267
# * Adding classes.
268
# * Removing classes.
269
# * Mutating attributes.
270
# * Adding leaf elements.
271
# * Removing leaf elements.
272
#
273
# which are made of two subtests each (sync/async) and repeated 5 times
274
# each, thus, the list here looks like:
275
#
276
# [Test name/Adding classes - 0/ Sync; <x>]
277
# [Test name/Adding classes - 0/ Async; <y>]
278
# [Test name/Adding classes - 0; <x> + <y>]
279
# [Test name/Removing classes - 0/ Sync; <x>]
280
# [Test name/Removing classes - 0/ Async; <y>]
281
# [Test name/Removing classes - 0; <x> + <y>]
282
# ...
283
# [Test name/Adding classes - 1 / Sync; <x>]
284
# [Test name/Adding classes - 1 / Async; <y>]
285
# [Test name/Adding classes - 1 ; <x> + <y>]
286
# ...
287
# [Test name/Removing leaf elements - 4; <x> + <y>]
288
# [Test name; <sum>] <- This is what we want.
289
#
290
# So, 5 (subtests) *
291
# 5 (repetitions) *
292
# 3 (entries per repetition (sync/async/sum)) =
293
# 75 entries for test before the sum.
294
#
295
# We receive 76 entries per test, which ads up to 380. We want to use
296
# the 5 test entries, not the rest.
297
if len(results) != 380:
298
raise Exception("StyleBench requires 380 entries, found: %s instead"
299
% len(results))
300
301
results = results[75::76]
302
score = 60 * 1000 / filter.geometric_mean(results) / correctionFactor
303
return score
304
305
def construct_results(self, vals, testname):
306
if 'responsiveness' in testname:
307
return filter.responsiveness_Metric([val for (val, page) in vals])
308
elif testname.startswith('v8_7'):
309
return self.v8_Metric(vals)
310
elif testname.startswith('kraken'):
311
return self.JS_Metric(vals)
312
elif testname.startswith('ares6'):
313
return self.benchmark_score(vals)
314
elif testname.startswith('jetstream'):
315
return self.benchmark_score(vals)
316
elif testname.startswith('speedometer'):
317
return self.speedometer_score(vals)
318
elif testname.startswith('stylebench'):
319
return self.stylebench_score(vals)
320
elif len(vals) > 1:
321
return filter.geometric_mean([i for i, j in vals])
322
else:
323
return filter.mean([i for i, j in vals])