Source code

Revision control

Other Tools

1
from __future__ import absolute_import
2
3
import os
4
5
from talos import filter
6
7
"""
8
test definitions for Talos
9
"""
10
11
_TESTS = {} # internal dict of Talos test classes
12
13
14
def register_test():
15
"""Decorator to register Talos test classes"""
16
def wrapper(klass):
17
assert issubclass(klass, Test)
18
assert klass.name() not in _TESTS
19
20
_TESTS[klass.name()] = klass
21
return klass
22
return wrapper
23
24
25
def test_dict():
26
"""Return the dict of the registered test classes"""
27
return _TESTS
28
29
30
class Test(object):
31
"""abstract base class for a Talos test case"""
32
__test__ = False # not pytest
33
34
cycles = None # number of cycles
35
keys = []
36
desktop = True
37
filters = filter.ignore_first.prepare(1) + filter.median.prepare()
38
lower_is_better = True
39
alert_threshold = 2.0
40
perfherder_framework = 'talos'
41
subtest_alerts = False
42
suite_should_alert = True
43
44
@classmethod
45
def name(cls):
46
return cls.__name__
47
48
@classmethod
49
def description(cls):
50
if cls.__doc__ is None:
51
return "No documentation available yet."
52
else:
53
doc = cls.__doc__
54
description_lines = [i.strip() for i in doc.strip().splitlines()]
55
return "\n".join(description_lines)
56
57
def __init__(self, **kw):
58
self.update(**kw)
59
60
def update(self, **kw):
61
self.__dict__.update(kw)
62
63
def items(self):
64
"""
65
returns a list of 2-tuples
66
"""
67
retval = [('name', self.name())]
68
for key in self.keys:
69
value = getattr(self, key, None)
70
if value is not None:
71
retval.append((key, value))
72
return retval
73
74
def __str__(self):
75
"""string form appropriate for YAML output"""
76
items = self.items()
77
78
key, value = items.pop(0)
79
lines = ["- %s: %s" % (key, value)]
80
for key, value in items:
81
lines.append(' %s: %s' % (key, value))
82
return '\n'.join(lines)
83
84
85
# ts-style startup tests (ts, twinopen, ts_cold, etc)
86
# The overall test number is calculated by excluding the max opening time
87
# and taking an average of the remaining numbers.
88
class TsBase(Test):
89
"""abstract base class for ts-style tests"""
90
keys = [
91
'url',
92
'url_timestamp',
93
'timeout',
94
'cycles',
95
'profile_path', # The path containing the template profile. This
96
# directory is copied to the temporary profile during
97
# initialization of the test. If some of the files may
98
# be overwritten by Firefox and need to be reinstalled
99
# before each pass, use key |reinstall|
100
'gecko_profile',
101
'gecko_profile_interval',
102
'gecko_profile_entries',
103
'gecko_profile_startup',
104
'preferences',
105
'xperf_counters',
106
'xperf_providers',
107
'xperf_user_providers',
108
'xperf_stackwalk',
109
'tpmozafterpaint',
110
'fnbpaint',
111
'tphero',
112
'tpmanifest',
113
'profile',
114
'firstpaint',
115
'userready',
116
'testeventmap',
117
'base_vs_ref',
118
'extensions',
119
'filters',
120
'setup',
121
'cleanup',
122
'webextensions',
123
'webextensions_folder',
124
'reinstall', # A list of files from the profile directory that
125
# should be copied to the temporary profile prior to
126
# running each cycle, to avoid one cycle overwriting
127
# the data used by the next another cycle (may be used
128
# e.g. for sessionstore.js to ensure that all cycles
129
# use the exact same sessionstore.js, rather than a
130
# more recent copy).
131
]
132
133
134
@register_test()
135
class ts_paint(TsBase):
136
"""
137
Launches tspaint_test.html with the current timestamp in the url,
138
waits for [MozAfterPaint and onLoad] to fire, then records the end
139
time and calculates the time to startup.
140
"""
141
cycles = 20
142
timeout = 150
143
gecko_profile_startup = True
144
gecko_profile_entries = 10000000
145
url = 'startup_test/tspaint_test.html'
146
xperf_counters = []
147
win7_counters = []
148
filters = filter.ignore_first.prepare(1) + filter.median.prepare()
149
tpmozafterpaint = True
150
mainthread = False
151
responsiveness = False
152
unit = 'ms'
153
154
155
@register_test()
156
class ts_paint_webext(ts_paint):
157
webextensions = '${talos}/webextensions/dummy/dummy.xpi'
158
preferences = {'xpinstall.signatures.required': False}
159
160
161
@register_test()
162
class ts_paint_heavy(ts_paint):
163
"""
164
ts_paint test ran against a heavy-user profile
165
"""
166
profile = 'simple'
167
168
169
@register_test()
170
class ts_paint_flex(ts_paint):
171
preferences = {'layout.css.emulate-moz-box-with-flex': True}
172
173
174
@register_test()
175
class startup_about_home_paint(ts_paint):
176
url = None
177
cycles = 20
178
extensions = ['${talos}/startup_test/startup_about_home_paint/addon']
179
tpmanifest = '${talos}/startup_test/startup_about_home_paint/startup_about_home_paint.manifest'
180
181
182
@register_test()
183
class startup_about_home_paint_realworld_webextensions(ts_paint):
184
url = None
185
cycles = 20
186
extensions = [
187
'${talos}/startup_test/startup_about_home_paint/addon',
188
'${talos}/getinfooffline'
189
]
190
tpmanifest = '${talos}/startup_test/startup_about_home_paint/startup_about_home_paint.manifest'
191
webextensions_folder = '${talos}/webextensions'
192
193
194
@register_test()
195
class sessionrestore(TsBase):
196
"""
197
A start up test measuring the time it takes to load a sessionstore.js file.
198
199
1. Set up Firefox to restore from a given sessionstore.js file.
200
2. Launch Firefox.
201
3. Measure the delta between firstPaint and sessionRestored.
202
"""
203
extensions = ['${talos}/startup_test/sessionrestore/addon']
204
cycles = 10
205
timeout = 900
206
gecko_profile_startup = True
207
gecko_profile_entries = 10000000
208
profile_path = '${talos}/startup_test/sessionrestore/profile'
209
reinstall = ['sessionstore.jsonlz4', 'sessionstore.js', 'sessionCheckpoints.json']
210
# Restore the session. We have to provide a URL, otherwise Talos
211
# asks for a manifest URL.
212
url = 'about:home'
213
preferences = {'browser.startup.page': 3}
214
unit = 'ms'
215
216
217
@register_test()
218
class sessionrestore_no_auto_restore(sessionrestore):
219
"""
220
A start up test measuring the time it takes to load a sessionstore.js file.
221
222
1. Set up Firefox to *not* restore automatically from sessionstore.js file.
223
2. Launch Firefox.
224
3. Measure the delta between firstPaint and sessionRestored.
225
"""
226
preferences = {
227
'browser.startup.page': 1,
228
'talos.sessionrestore.norestore': True,
229
}
230
231
232
@register_test()
233
class sessionrestore_many_windows(sessionrestore):
234
"""
235
A start up test measuring the time it takes to load a sessionstore.js file.
236
237
1. Set up Firefox to restore automatically from sessionstore.js file.
238
2. Launch Firefox.
239
3. Measure the delta between firstPaint and sessionRestored.
240
"""
241
profile_path = '${talos}/startup_test/sessionrestore/profile-manywindows'
242
243
244
# pageloader tests(tp5, etc)
245
246
# The overall test number is determined by first calculating the median
247
# page load time for each page in the set (excluding the max page load
248
# per individual page). The max median from that set is then excluded and
249
# the average is taken; that becomes the number reported to the tinderbox
250
# waterfall.
251
252
253
class PageloaderTest(Test):
254
"""abstract base class for a Talos Pageloader test"""
255
extensions = ['${talos}/pageloader']
256
tpmanifest = None # test manifest
257
tpcycles = 1 # number of time to run each page
258
cycles = None
259
timeout = None
260
261
keys = ['tpmanifest', 'tpcycles', 'tppagecycles', 'tprender', 'tpchrome',
262
'tpmozafterpaint', 'fnbpaint', 'tphero', 'tploadnocache', 'firstpaint',
263
'userready', 'testeventmap', 'base_vs_ref', 'mainthread', 'resolution',
264
'cycles', 'gecko_profile', 'gecko_profile_interval', 'gecko_profile_entries',
265
'tptimeout', 'win_counters', 'w7_counters', 'linux_counters', 'mac_counters',
266
'tpscrolltest', 'xperf_counters', 'timeout', 'responsiveness',
267
'profile_path', 'xperf_providers', 'xperf_user_providers', 'xperf_stackwalk',
268
'format_pagename', 'filters', 'preferences', 'extensions', 'setup', 'cleanup',
269
'lower_is_better', 'alert_threshold', 'unit', 'webextensions', 'profile',
270
'suite_should_alert', 'subtest_alerts', 'perfherder_framework', 'pdfpaint',
271
'webextensions_folder', 'a11y']
272
273
274
class QuantumPageloadTest(PageloaderTest):
275
"""
276
Base class for a Quantum Pageload test
277
"""
278
tpcycles = 1
279
tppagecycles = 25
280
gecko_profile_interval = 1
281
gecko_profile_entries = 2000000
282
filters = filter.ignore_first.prepare(5) + filter.median.prepare()
283
unit = 'ms'
284
lower_is_better = True
285
fnbpaint = True
286
287
288
@register_test()
289
class twinopen(PageloaderTest):
290
"""
291
Tests the amount of time it takes an open browser to open a new browser
292
window and paint the browser chrome. This test does not include startup
293
time. Multiple test windows are opened in succession.
294
(Measures ctrl-n performance.)
295
"""
296
extensions = ['${talos}/pageloader', '${talos}/tests/twinopen']
297
tpmanifest = '${talos}/tests/twinopen/twinopen.manifest'
298
tppagecycles = 20
299
timeout = 300
300
gecko_profile_interval = 1
301
gecko_profile_entries = 2000000
302
tpmozafterpaint = True
303
filters = filter.ignore_first.prepare(5) + filter.median.prepare()
304
unit = 'ms'
305
preferences = {
306
'browser.startup.homepage': 'about:blank'
307
}
308
309
310
@register_test()
311
class pdfpaint(PageloaderTest):
312
"""
313
Tests the amount of time it takes for the the first page of a PDF to
314
be rendered.
315
"""
316
tpmanifest = '${talos}/tests/pdfpaint/pdfpaint.manifest'
317
tppagecycles = 20
318
gecko_profile_entries = 1000000
319
pdfpaint = True
320
unit = 'ms'
321
preferences = {
322
'pdfjs.eventBusDispatchToDOM': True
323
}
324
325
326
@register_test()
327
class cpstartup(PageloaderTest):
328
"""
329
Tests the amount of time it takes to start up a new content process and
330
initialize it to the point where it can start processing incoming URLs
331
to load.
332
"""
333
extensions = ['${talos}/pageloader', '${talos}/tests/cpstartup/extension']
334
tpmanifest = '${talos}/tests/cpstartup/cpstartup.manifest'
335
tppagecycles = 20
336
gecko_profile_entries = 1000000
337
tploadnocache = True
338
unit = 'ms'
339
preferences = {
340
# By default, Talos is configured to open links from
341
# content in new windows. We're overriding them so that
342
# they open in new tabs instead.
345
'browser.link.open_newwindow': 3,
346
'browser.link.open_newwindow.restriction': 2,
347
}
348
349
350
@register_test()
351
class tabpaint(PageloaderTest):
352
"""
353
Tests the amount of time it takes to open new tabs, triggered from
354
both the parent process and the content process.
355
"""
356
extensions = ['${talos}/tests/tabpaint', '${talos}/pageloader']
357
tpmanifest = '${talos}/tests/tabpaint/tabpaint.manifest'
358
tppagecycles = 20
359
gecko_profile_entries = 1000000
360
tploadnocache = True
361
unit = 'ms'
362
preferences = {
363
# By default, Talos is configured to open links from
364
# content in new windows. We're overriding them so that
365
# they open in new tabs instead.
368
'browser.link.open_newwindow': 3,
369
'browser.link.open_newwindow.restriction': 2,
370
'browser.newtab.preload': False,
371
}
372
373
374
@register_test()
375
class tabswitch(PageloaderTest):
376
"""
377
Tests the amount of time it takes to switch between tabs
378
"""
379
extensions = ['${talos}/tests/tabswitch', '${talos}/pageloader']
380
tpmanifest = '${talos}/tests/tabswitch/tabswitch.manifest'
381
tppagecycles = 5
382
gecko_profile_entries = 5000000
383
tploadnocache = True
384
preferences = {
385
'addon.test.tabswitch.urlfile': os.path.join('${talos}',
386
'tests',
387
'tp5o.html'),
388
'addon.test.tabswitch.webserver': '${webserver}',
389
'addon.test.tabswitch.maxurls': -1,
390
}
391
unit = 'ms'
392
393
394
@register_test()
395
class tart(PageloaderTest):
396
"""
397
Tab Animation Regression Test
398
Tests tab animation on these cases:
399
1. Simple: single new tab of about:blank open/close without affecting
400
(shrinking/expanding) other tabs.
401
2. icon: same as above with favicons and long title instead of about:blank.
402
3. Newtab: newtab open with thumbnails preview - without affecting other
403
tabs, with and without preload.
404
4. Fade: opens a tab, then measures fadeout/fadein (tab animation without
405
the overhead of opening/closing a tab).
406
- Case 1 is tested with DPI scaling of 1.
407
- Case 2 is tested with DPI scaling of 1.0 and 2.0.
408
- Case 3 is tested with the default scaling of the test system.
409
- Case 4 is tested with DPI scaling of 2.0 with the "icon" tab
410
(favicon and long title).
411
- Each animation produces 3 test results:
412
- error: difference between the designated duration and the actual
413
completion duration from the trigger.
414
- half: average interval over the 2nd half of the animation.
415
- all: average interval over all recorded intervals.
416
"""
417
tpmanifest = '${talos}/tests/tart/tart.manifest'
418
extensions = ['${talos}/pageloader', '${talos}/tests/tart/addon']
419
tpcycles = 1
420
tppagecycles = 25
421
tploadnocache = True
422
tpmozafterpaint = False
423
gecko_profile_interval = 10
424
gecko_profile_entries = 1000000
425
win_counters = w7_counters = linux_counters = mac_counters = None
426
"""
427
ASAP mode
428
The recording API is broken with OMTC before ~2013-11-27
429
After ~2013-11-27, disabling OMTC will also implicitly disable
430
OGL HW composition to disable OMTC with older firefox builds, also
431
set 'layers.offmainthreadcomposition.enabled': False
432
"""
433
preferences = {'layout.frame_rate': 0,
434
'docshell.event_starvation_delay_hint': 1,
435
'dom.send_after_paint_to_content': False}
436
filters = filter.ignore_first.prepare(1) + filter.median.prepare()
437
unit = 'ms'
438
439
440
@register_test()
441
class tart_flex(tart):
442
preferences = {'layout.css.emulate-moz-box-with-flex': True}
443
444
445
@register_test()
446
class damp(PageloaderTest):
447
"""
448
Devtools At Maximum Performance
449
Tests the speed of DevTools toolbox open, close, and page reload
450
for each tool, across a very simple and very complicated page.
451
"""
452
tpmanifest = '${talos}/tests/devtools/damp.manifest'
453
extensions = ['${talos}/pageloader', '${talos}/tests/devtools/addon']
454
cycles = 5
455
tpcycles = 1
456
tppagecycles = 5
457
tploadnocache = True
458
tpmozafterpaint = False
459
gecko_profile_interval = 10
460
gecko_profile_entries = 10000000
461
win_counters = w7_counters = linux_counters = mac_counters = None
462
filters = filter.ignore_first.prepare(1) + filter.median.prepare()
463
preferences = {'devtools.memory.enabled': True}
464
unit = 'ms'
465
subtest_alerts = True
466
perfherder_framework = 'devtools'
467
468
469
@register_test()
470
class glterrain(PageloaderTest):
471
"""
472
Simple rotating WebGL scene with moving light source over a
473
textured terrain.
474
Measures average frame intervals.
475
The same sequence is measured 4 times for combinations of alpha and
476
antialias as canvas properties.
477
Each of these 4 runs is reported as a different test name.
478
"""
479
tpmanifest = '${talos}/tests/webgl/glterrain.manifest'
480
tpcycles = 1
481
tppagecycles = 25
482
tploadnocache = True
483
tpmozafterpaint = False
484
tpchrome = False
485
gecko_profile_interval = 10
486
gecko_profile_entries = 2000000
487
win_counters = w7_counters = linux_counters = mac_counters = None
488
""" ASAP mode """
489
preferences = {'layout.frame_rate': 0,
490
'docshell.event_starvation_delay_hint': 1,
491
'dom.send_after_paint_to_content': False}
492
filters = filter.ignore_first.prepare(1) + filter.median.prepare()
493
unit = 'frame interval'
494
495
496
@register_test()
497
class glvideo(PageloaderTest):
498
"""
499
WebGL video texture update with 1080p video.
500
Measures mean tick time across 100 ticks.
501
(each tick is texImage2D(<video>)+setTimeout(0))
502
"""
503
tpmanifest = '${talos}/tests/webgl/glvideo.manifest'
504
tpcycles = 1
505
tppagecycles = 5
506
tploadnocache = True
507
tpmozafterpaint = False
508
tpchrome = False
509
gecko_profile_interval = 2
510
gecko_profile_entries = 2000000
511
win_counters = w7_counters = linux_counters = mac_counters = None
512
filters = filter.ignore_first.prepare(1) + filter.median.prepare()
513
unit = 'ms'
514
515
516
@register_test()
517
class tp5n(PageloaderTest):
518
"""
519
Tests the time it takes Firefox to load the tp5 web page test set.
520
521
The tp5 is an updated web page test set to 100 pages from April 8th, 2011.
522
Effort was made for the pages to no longer be splash screens/login
523
pages/home pages but to be pages that better reflect the actual content
524
of the site in question.
525
"""
526
resolution = 20
527
tpmanifest = '${talos}/tests/tp5n/tp5n.manifest'
528
tpcycles = 1
529
tppagecycles = 1
530
cycles = 1
531
tpmozafterpaint = True
532
tptimeout = 10000
533
mainthread = True
534
w7_counters = []
535
win_counters = []
536
linux_counters = []
537
mac_counters = []
538
xperf_counters = ['main_startup_fileio', 'main_startup_netio',
539
'main_normal_fileio', 'main_normal_netio',
540
'nonmain_startup_fileio', 'nonmain_normal_fileio',
541
'nonmain_normal_netio', 'mainthread_readcount',
542
'mainthread_readbytes', 'mainthread_writecount',
543
'mainthread_writebytes',
544
'time_to_session_store_window_restored_ms',
545
]
546
xperf_providers = ['PROC_THREAD', 'LOADER', 'HARD_FAULTS', 'FILENAME',
547
'FILE_IO', 'FILE_IO_INIT']
548
xperf_user_providers = ['Mozilla Generic Provider',
549
'Microsoft-Windows-TCPIP']
550
xperf_stackwalk = ['FileCreate', 'FileRead', 'FileWrite', 'FileFlush',
551
'FileClose']
552
filters = filter.ignore_first.prepare(1) + filter.median.prepare()
553
timeout = 1800
554
setup = '${talos}/xtalos/start_xperf.py -c ${talos}/bcontroller.json'
555
cleanup = '${talos}/xtalos/parse_xperf.py -c ${talos}/bcontroller.json'
556
preferences = {'extensions.enabledScopes': '',
557
'talos.logfile': 'browser_output.txt'}
558
unit = 'ms'
559
560
561
@register_test()
562
class tp5o(PageloaderTest):
563
"""
564
Derived from the tp5n pageset, this is the 49 most reliable webpages.
565
"""
566
tpcycles = 1
567
tppagecycles = 25
568
cycles = 1
569
tpmozafterpaint = True
570
tptimeout = 5000
571
mainthread = False
572
tpmanifest = '${talos}/tests/tp5n/tp5o.manifest'
573
win_counters = ['% Processor Time']
574
w7_counters = ['% Processor Time']
575
linux_counters = ['XRes']
576
mac_counters = []
577
responsiveness = True
578
gecko_profile_interval = 2
579
gecko_profile_entries = 4000000
580
filters = filter.ignore_first.prepare(5) + filter.median.prepare()
581
timeout = 1800
582
unit = 'ms'
583
584
585
@register_test()
586
class tp5o_webext(tp5o):
587
webextensions = '${talos}/webextensions/dummy/dummy.xpi'
588
preferences = {'xpinstall.signatures.required': False}
589
590
591
@register_test()
592
class tp5o_scroll(PageloaderTest):
593
"""
594
Tests scroll (like tscrollx does, including ASAP) but on the tp5o pageset.
595
"""
596
tpmanifest = '${talos}/tests/tp5n/tp5o.manifest'
597
tpcycles = 1
598
tppagecycles = 12
599
gecko_profile_interval = 2
600
gecko_profile_entries = 2000000
601
tpscrolltest = True
602
"""ASAP mode"""
603
tpmozafterpaint = False
604
preferences = {'layout.frame_rate': 0,
605
'docshell.event_starvation_delay_hint': 1,
606
'dom.send_after_paint_to_content': True,
607
'apz.paint_skipping.enabled': False,
608
'layout.css.scroll-behavior.spring-constant': "'10'",
609
'toolkit.framesRecording.bufferSize': 10000}
610
filters = filter.ignore_first.prepare(1) + filter.median.prepare()
611
unit = '1/FPS'
612
613
614
@register_test()
615
class v8_7(PageloaderTest):
616
"""
617
This is the V8 (version 7) javascript benchmark taken verbatim and
618
slightly modified to fit into our pageloader extension and talos harness.
619
620
The previous version of this test is V8 version 5 which was run on
621
selective branches and operating systems.
622
"""
623
tpmanifest = '${talos}/tests/v8_7/v8.manifest'
624
gecko_profile_interval = 1
625
gecko_profile_entries = 1000000
626
tpcycles = 1
627
resolution = 20
628
tpmozafterpaint = False
629
preferences = {'dom.send_after_paint_to_content': False}
630
filters = filter.v8_subtest.prepare()
631
unit = 'score'
632
lower_is_better = False
633
634
635
@register_test()
636
class kraken(PageloaderTest):
637
"""
638
This is the Kraken javascript benchmark taken verbatim and slightly
639
modified to fit into our pageloader extension and talos harness.
640
"""
641
tpmanifest = '${talos}/tests/kraken/kraken.manifest'
642
tpcycles = 1
643
tppagecycles = 1
644
gecko_profile_interval = 1
645
gecko_profile_entries = 5000000
646
tpmozafterpaint = False
647
tpchrome = False
648
preferences = {'dom.send_after_paint_to_content': False}
649
filters = filter.mean.prepare()
650
unit = 'score'
651
652
653
@register_test()
654
class basic_compositor_video(PageloaderTest):
655
"""
656
Video test
657
"""
658
tpmanifest = '${talos}/tests/video/video.manifest'
659
tpcycles = 1
660
tppagecycles = 12
661
tpchrome = False
662
timeout = 10000
663
gecko_profile_interval = 1
664
gecko_profile_entries = 2000000
665
preferences = {'full-screen-api.allow-trusted-requests-only': False,
666
'layers.acceleration.force-enabled': False,
667
'layers.acceleration.disabled': True,
668
'layout.frame_rate': 0,
669
'docshell.event_starvation_delay_hint': 1,
670
'full-screen-api.warning.timeout': 500,
671
'media.ruin-av-sync.enabled': True}
672
filters = filter.ignore_first.prepare(1) + filter.median.prepare()
673
unit = 'ms/frame'
674
lower_is_better = True
675
676
677
class dromaeo(PageloaderTest):
678
"""abstract base class for dramaeo tests"""
679
filters = filter.dromaeo.prepare()
680
lower_is_better = False
681
alert_threshold = 5.0
682
tpchrome = False
683
684
685
@register_test()
686
class dromaeo_css(dromaeo):
687
"""
688
Dromaeo suite of tests for JavaScript performance testing.
689
See the Dromaeo wiki (https://wiki.mozilla.org/Dromaeo)
690
for more information.
691
692
Each page in the manifest is part of the dromaemo css benchmark.
693
"""
694
gecko_profile_interval = 2
695
gecko_profile_entries = 10000000
696
tpmanifest = '${talos}/tests/dromaeo/css.manifest'
697
unit = 'score'
698
699
700
@register_test()
701
class dromaeo_dom(dromaeo):
702
"""
703
Dromaeo suite of tests for JavaScript performance testing.
704
See the Dromaeo wiki (https://wiki.mozilla.org/Dromaeo)
705
for more information.
706
707
Each page in the manifest is part of the dromaemo dom benchmark.
708
"""
709
gecko_profile_interval = 2
710
gecko_profile_entries = 10000000
711
tpmanifest = '${talos}/tests/dromaeo/dom.manifest'
712
unit = 'score'
713
714
715
@register_test()
716
class tresize(PageloaderTest):
717
"""
718
This test does some resize thing.
719
"""
720
tpmanifest = '${talos}/tests/tresize/tresize.manifest'
721
extensions = ['${talos}/pageloader', '${talos}/tests/tresize/addon']
722
tppagecycles = 20
723
timeout = 900
724
gecko_profile_interval = 2
725
gecko_profile_entries = 1000000
726
tpmozafterpaint = True
727
filters = filter.ignore_first.prepare(5) + filter.median.prepare()
728
unit = 'ms'
729
730
731
@register_test()
732
class tsvgm(PageloaderTest):
733
"""
734
An svg-only number that measures SVG rendering performance
735
for dynamic content only.
736
"""
737
tpmanifest = '${talos}/tests/svgx/svgm.manifest'
738
tpcycles = 1
739
tppagecycles = 7
740
tpmozafterpaint = False
741
tpchrome = False
742
gecko_profile_interval = 10
743
gecko_profile_entries = 1000000
744
"""ASAP mode"""
745
preferences = {'layout.frame_rate': 0,
746
'docshell.event_starvation_delay_hint': 1,
747
'dom.send_after_paint_to_content': False}
748
filters = filter.ignore_first.prepare(2) + filter.median.prepare()
749
unit = 'ms'
750
751
752
@register_test()
753
class tsvgx(PageloaderTest):
754
"""
755
An svg-only number that measures SVG rendering performance
756
for dynamic content only.
757
"""
758
tpmanifest = '${talos}/tests/svgx/svgx.manifest'
759
tpcycles = 1
760
tppagecycles = 25
761
tpmozafterpaint = False
762
tpchrome = False
763
gecko_profile_interval = 10
764
gecko_profile_entries = 1000000
765
"""ASAP mode"""
766
preferences = {'layout.frame_rate': 0,
767
'docshell.event_starvation_delay_hint': 1,
768
'dom.send_after_paint_to_content': False}
769
filters = filter.ignore_first.prepare(5) + filter.median.prepare()
770
unit = 'ms'
771
772
773
@register_test()
774
class tsvg_static(PageloaderTest):
775
"""
776
An svg-only number that measures SVG rendering performance
777
for static content only.
778
"""
779
tpmanifest = '${talos}/tests/svg_static/svg_static.manifest'
780
tpcycles = 1
781
tppagecycles = 25
782
tpmozafterpaint = True
783
tpchrome = False
784
gecko_profile_interval = 1
785
gecko_profile_entries = 10000000
786
filters = filter.ignore_first.prepare(5) + filter.median.prepare()
787
unit = 'ms'
788
789
790
@register_test()
791
class tsvgr_opacity(PageloaderTest):
792
"""
793
An svg-only number that measures SVG rendering performance.
794
"""
795
tpmanifest = '${talos}/tests/svg_opacity/svg_opacity.manifest'
796
tpcycles = 1
797
tppagecycles = 25
798
tpmozafterpaint = True
799
tpchrome = False
800
gecko_profile_interval = 1
801
gecko_profile_entries = 10000000
802
filters = filter.ignore_first.prepare(5) + filter.median.prepare()
803
unit = 'ms'
804
805
806
@register_test()
807
class tscrollx(PageloaderTest):
808
"""
809
This test does some scrolly thing.
810
"""
811
tpmanifest = '${talos}/tests/scroll/scroll.manifest'
812
tpcycles = 1
813
tppagecycles = 25
814
tpmozafterpaint = False
815
tpchrome = False
816
gecko_profile_interval = 1
817
gecko_profile_entries = 1000000
818
""" ASAP mode """
819
preferences = {'layout.frame_rate': 0,
820
'docshell.event_starvation_delay_hint': 1,
821
'dom.send_after_paint_to_content': True,
822
'apz.paint_skipping.enabled': False,
823
'layout.css.scroll-behavior.spring-constant': "'10'",
824
'toolkit.framesRecording.bufferSize': 10000}
825
filters = filter.ignore_first.prepare(5) + filter.median.prepare()
826
unit = 'ms'
827
828
829
@register_test()
830
class a11yr(PageloaderTest):
831
"""
832
This test ensures basic a11y tables and permutations do not cause
833
performance regressions.
834
"""
835
tpmanifest = '${talos}/tests/a11y/a11y.manifest'
836
tpcycles = 1
837
tppagecycles = 25
838
tpmozafterpaint = True
839
tpchrome = False
840
preferences = {'dom.send_after_paint_to_content': False}
841
unit = 'ms'
842
alert_threshold = 5.0
843
a11y = True
844
845
846
class WebkitBenchmark(PageloaderTest):
847
tpcycles = 1
848
tppagecycles = 5
849
tpmozafterpaint = False
850
tpchrome = False
851
format_pagename = False
852
lower_is_better = False
853
unit = 'score'
854
855
856
@register_test()
857
class stylebench(WebkitBenchmark):
858
# StyleBench benchmark used by many browser vendors (from webkit)
859
tpmanifest = '${talos}/tests/stylebench/stylebench.manifest'
860
861
862
@register_test()
863
class motionmark_animometer(WebkitBenchmark):
864
# MotionMark benchmark used by many browser vendors (from webkit)
865
tpmanifest = '${talos}/tests/motionmark/animometer.manifest'
866
867
868
@register_test()
869
class ARES6(WebkitBenchmark):
870
# ARES-6 benchmark used by many browser vendors (from webkit)
871
tpmanifest = '${talos}/tests/ares6/ares6.manifest'
872
tppagecycles = 1
873
lower_is_better = True
874
875
876
@register_test()
877
class motionmark_htmlsuite(WebkitBenchmark):
878
# MotionMark benchmark used by many browser vendors (from webkit)
879
tpmanifest = '${talos}/tests/motionmark/htmlsuite.manifest'
880
881
882
@register_test()
883
class JetStream(WebkitBenchmark):
884
# JetStream benchmark used by many browser vendors (from webkit)
885
tpmanifest = '${talos}/tests/jetstream/jetstream.manifest'
886
tppagecycles = 1
887
888
889
@register_test()
890
class perf_reftest(PageloaderTest):
891
"""
892
Style perf-reftest a set of tests where the result is the difference of base vs ref pages
893
"""
894
base_vs_ref = True # compare the two test pages with eachother and report comparison
895
tpmanifest = '${talos}/tests/perf-reftest/perf_reftest.manifest'
896
tpcycles = 1
897
tppagecycles = 10
898
tptimeout = 30000
899
gecko_profile_interval = 1
900
gecko_profile_entries = 2000000
901
filters = filter.ignore_first.prepare(5) + filter.median.prepare()
902
unit = 'ms'
903
lower_is_better = True
904
alert_threshold = 5.0
905
subtest_alerts = True
906
907
908
@register_test()
909
class perf_reftest_singletons(PageloaderTest):
910
"""
911
Style perf-reftests run as individual tests
912
"""
913
tpmanifest = '${talos}/tests/perf-reftest-singletons/perf_reftest_singletons.manifest'
914
tpcycles = 1
915
tppagecycles = 15
916
tptimeout = 30000
917
gecko_profile_interval = 1
918
gecko_profile_entries = 2000000
919
filters = filter.ignore_first.prepare(5) + filter.median.prepare()
920
unit = 'ms'
921
lower_is_better = True
922
alert_threshold = 5.0
923
subtest_alerts = True
924
suite_should_alert = False
925
926
927
@register_test()
928
class displaylist_mutate(PageloaderTest):
929
"""
930
Test modifying single items in a large display list. Measure transaction speed
931
to the compositor.
932
"""
933
tpmanifest = '${talos}/tests/layout/displaylist_mutate.manifest'
934
tpcycles = 1
935
tppagecycles = 5
936
tploadnocache = True
937
tpmozafterpaint = False
938
tpchrome = False
939
gecko_profile_interval = 2
940
gecko_profile_entries = 2000000
941
win_counters = w7_counters = linux_counters = mac_counters = None
942
filters = filter.ignore_first.prepare(1) + filter.median.prepare()
943
"""ASAP mode"""
944
preferences = {'layout.frame_rate': 0,
945
'docshell.event_starvation_delay_hint': 1,
946
'dom.send_after_paint_to_content': False}
947
unit = 'ms'
948
949
950
@register_test()
951
class rasterflood_svg(PageloaderTest):
952
"""
953
Test modifying single items in a large display list. Measure transaction speed
954
to the compositor.
955
"""
956
tpmanifest = '${talos}/tests/gfx/rasterflood_svg.manifest'
957
tpcycles = 1
958
tppagecycles = 10
959
tploadnocache = True
960
tpmozafterpaint = False
961
tpchrome = False
962
gecko_profile_interval = 2
963
gecko_profile_entries = 2000000
964
win_counters = w7_counters = linux_counters = mac_counters = None
965
filters = filter.ignore_first.prepare(1) + filter.median.prepare()
966
"""ASAP mode"""
967
preferences = {'layout.frame_rate': 0,
968
'docshell.event_starvation_delay_hint': 1,
969
'dom.send_after_paint_to_content': False}
970
unit = 'ms'
971
972
973
@register_test()
974
class rasterflood_gradient(PageloaderTest):
975
"""
976
Test expensive rasterization while the main thread is busy.
977
"""
978
tpmanifest = '${talos}/tests/gfx/rasterflood_gradient.manifest'
979
tpcycles = 1
980
tppagecycles = 10
981
tploadnocache = True
982
tpmozafterpaint = False
983
tpchrome = False
984
gecko_profile_interval = 2
985
gecko_profile_entries = 2000000
986
win_counters = w7_counters = linux_counters = mac_counters = None
987
filters = filter.ignore_first.prepare(1) + filter.median.prepare()
988
"""ASAP mode"""
989
preferences = {'layout.frame_rate': 0,
990
'docshell.event_starvation_delay_hint': 1,
991
'dom.send_after_paint_to_content': False}
992
lower_is_better = False
993
unit = 'score'
994
995
996
@register_test()
997
class about_preferences_basic(PageloaderTest):
998
"""
999
Base class for about_preferences test
1000
"""
1001
tpmanifest = '${talos}/tests/about-preferences/about_preferences_basic.manifest'
1002
# this test uses 'about:blank' as a dummy page (see manifest) so that the pages
1003
# that just change url categories (i.e. about:preferences#search) will get a load event
1004
# also any of the url category pages cannot have more than one tppagecycle
1005
tpcycles = 25
1006
tppagecycles = 1
1007
gecko_profile_interval = 1
1008
gecko_profile_entries = 2000000
1009
filters = filter.ignore_first.prepare(5) + filter.median.prepare()
1010
unit = 'ms'
1011
lower_is_better = True
1012
fnbpaint = True
1013
1014
1015
@register_test()
1016
class about_newtab_with_snippets(PageloaderTest):
1017
"""
1018
Load about ActivityStream (about:home and about:newtab) with snippets enabled
1019
"""
1020
tpmanifest = '${talos}/tests/about-newtab/about_newtab.manifest'
1021
tpcycles = 25
1022
tppagecycles = 1
1023
responsiveness = True
1024
gecko_profile_interval = 1
1025
gecko_profile_entries = 2000000
1026
filters = filter.ignore_first.prepare(5) + filter.median.prepare()
1027
unit = 'ms'
1028
lower_is_better = True
1029
fnbpaint = True
1030
preferences = {
1031
# ensure that snippets are turned on and load the json messages
1032
'browser.newtabpage.activity-stream.asrouter.providers.snippets':\
1033
'{"id":"snippets","enabled":true,"type":"json","location":\
1035
"updateCycleInMs":14400000}',
1036
'browser.newtabpage.activity-stream.feeds.snippets': True
1037
}