Source code

Revision control

Copy as Markdown

Other Tools

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
job-defaults:
max-run-time:
by-test-platform:
.*-qr/.*: 2400
default: 1800
suite: raptor
run-on-projects:
by-test-platform:
linux.*shippable[^-qr].*: [mozilla-central]
default: []
limit-platforms:
by-app:
chrome:
- .*shippable.*
safari:
- .*macos.*shippable.*
custom-car:
- .*64.*shippable.*
default: []
test-manifest-loader: null # don't load tests in the taskgraph
tier: 3
virtualization: hardware
mozharness:
script: raptor_script.py
config:
by-test-platform:
macosx.*:
- raptor/mac_config.py
windows.*:
- raptor/windows_config.py
default:
- raptor/linux_config.py
extra-options:
- --conditioned-profile=settled
fetches:
toolchain:
by-app:
custom-car:
by-test-platform:
# this satisfied taskgraph for now, but once mac/win is implemented
# then linux will be the default
linux.*:
- linux64-custom-car
- linux64-minidump-stackwalk
- linux64-fix-stacks
windows10-64.*:
- win64-custom-car
- win64-minidump-stackwalk
- win64-fix-stacks
macosx1015.*:
- macosx64-custom-car
- macosx64-minidump-stackwalk
- macosx64-fix-stacks
macosx1400.*:
- macosx-arm64-custom-car
- macosx64-aarch64-minidump-stackwalk
- macosx64-aarch64-fix-stacks
default: []
default:
by-test-platform:
windows.*-32.*.:
- win32-minidump-stackwalk
- win32-fix-stacks
windows10-64.*:
- win64-minidump-stackwalk
- win64-fix-stacks
macos.*:
- macosx64-minidump-stackwalk
- macosx64-fix-stacks
default:
- linux64-minidump-stackwalk
- linux64-fix-stacks
browsertime-tp6:
<<: &tp6-defaults
description: Raptor (browsertime) tp6 page-load tests
limit-platforms:
by-app:
chrome:
- .*shippable.*
safari:
- .*macos.*shippable.*
custom-car:
- .*64.*shippable.*
default: []
max-run-time:
by-subtest:
imgur: 4800
default: 4000
mozharness:
extra-options:
by-test-platform:
windows10-64.*shippable.*-qr.*:
- --chimera
- --browsertime-no-ffwindowrecorder
- --conditioned-profile=settled
- --extra-profiler-run
default:
- --chimera
- --conditioned-profile=settled
- --extra-profiler-run
run-on-projects:
by-subtest:
netflix:
by-app:
firefox:
by-test-platform:
windows.*-32.*: []
(macosx1015|linux|windows10-64).*shippable-qr.*: [trunk]
default: []
default: []
default:
by-app:
firefox:
by-test-platform:
windows.*-32.*: []
.*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk]
default: []
default: []
treeherder-symbol: Btime(tp6)
raptor:
apps: [firefox, chrome]
test: tp6
run-visual-metrics: true
subtests:
- buzzfeed
- ebay
- espn
- expedia
- [facebook, fb]
- [google-docs, gdocs]
- [google-mail, gmail]
- [google-search, gsearch]
- imdb
- imgur
- linkedin
- microsoft
- netflix
- nytimes
- office
- outlook
- paypal
- pinterest
- reddit
- tumblr
- twitch
- wikia
- youtube
tier:
by-app:
firefox: 2
default: 3
browsertime-responsiveness:
<<: *tp6-defaults
raptor:
apps: [firefox, chrome]
test: tp6
run-visual-metrics: true
subtests:
- cnn-nav
- [facebook-nav, fbnav]
- [reddit-billgates-ama, rdt-ama]
- [reddit-billgates-post-1, rdt-post-1]
- [reddit-billgates-post-2, rdt-post-2]
mozharness:
extra-options:
by-test-platform:
windows10-.*:
- --cold
- --browsertime-no-ffwindowrecorder
- --conditioned-profile=settled
- --extra-profiler-run
default:
- --cold
- --conditioned-profile=settled
- --extra-profiler-run
run-on-projects:
by-app:
firefox:
by-test-platform:
windows.*-32.*: []
.*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk]
default: []
default: []
tier:
by-app:
firefox: 2
default: 3
browsertime-tp6-essential:
<<: *tp6-defaults
<<: &tp6-essential-defaults
mozharness:
extra-options:
by-test-platform:
windows10-64.*shippable.*-qr.*:
- --chimera
- --browsertime-no-ffwindowrecorder
- --conditioned-profile=settled
- --collect-perfstats
- --extra-profiler-run
default:
- --chimera
- --conditioned-profile=settled
- --extra-profiler-run
raptor:
apps: [firefox, chrome, custom-car]
test: tp6
run-visual-metrics:
by-test-platform:
macosx1400.*shippable.*: false
# Bug 1845014 Resolve osx 1300 vismet permafails
default: true
subtests:
- amazon
- [bing-search, bing]
- cnn
- fandom
- [google-slides, gslides]
- instagram
- twitter
- wikipedia
- yahoo-mail
run-on-projects:
by-app:
firefox:
by-test-platform:
windows.*-32.*: []
macosx1100.*shippable.*: [mozilla-central]
.*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk]
default: []
default: []
tier:
by-test-platform:
linux1804-64-clang-trunk-qr/opt: 2
default:
by-app:
firefox: 1
default: 3
browsertime-tp6-bytecode:
<<: *tp6-defaults
<<: *tp6-essential-defaults
raptor:
apps: [firefox]
test: tp6
run-visual-metrics: true
subtests:
- amazon
- [bing-search, bing]
- cnn
- fandom
- [google-slides, gslides]
- instagram
- twitter
- wikipedia
- yahoo-mail
run-on-projects:
by-app:
firefox:
by-test-platform:
windows.*-32.*: []
macosx1100.*shippable.*: [mozilla-central]
.*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk]
default: []
default: []
mozharness:
extra-options:
by-test-platform:
windows10-64.*shippable.*-qr.*:
- --chimera
- --browsertime
- --browsertime-no-ffwindowrecorder
- --conditioned-profile=settled
- --test-bytecode-cache
- --extra-profiler-run
default:
- --chimera
- --browsertime
- --conditioned-profile=settled
- --test-bytecode-cache
- --extra-profiler-run
tier:
by-app:
firefox: 2
default: 3
treeherder-symbol: Btime-cache(tp6)
browsertime-tp7:
<<: &tp7-defaults
description: Raptor (browsertime) tp7 page-load tests
limit-platforms:
by-app:
chrome:
- .*shippable.*
safari:
- .*macos.*shippable.*
custom-car:
- .*64.*shippable.*
default: []
max-run-time:
by-subtest:
imgur: 4800
default: 4000
mozharness:
extra-options:
by-test-platform:
windows10-64.*shippable.*-qr.*:
- --chimera
- --browsertime-no-ffwindowrecorder
- --conditioned-profile=settled
- --extra-profiler-run
default:
- --chimera
- --conditioned-profile=settled
- --extra-profiler-run
run-on-projects: []
treeherder-symbol: Btime(tp7)
raptor:
apps: [firefox, chrome]
test: tp7
run-visual-metrics: true
subtests:
- tiktok
- openai
- samsung
- weather
- docomo
tier:
by-app:
firefox: 3
default: 3
browsertime-benchmark:
description: Raptor (browsertime) Benchmark tests
raptor:
apps: [firefox, chrome, safari, custom-car]
subtests:
by-test-platform:
macosx1400.*shippable.*:
- [speedometer, sp]
- [speedometer3, sp3]
- [motionmark-1-3, mm-1.3]
- [motionmark-htmlsuite-1-3, mm-h-1.3]
default:
by-app:
firefox:
- ares6
- [assorted-dom, dom]
- [jetstream2, js2]
- [matrix-react-bench, mrb]
- [motionmark-animometer, mm-a]
- [motionmark-animometer-ramp, mm-ar]
- [motionmark-htmlsuite-ramp, mm-hr]
- [motionmark-htmlsuite, mm-h]
- [motionmark-1-3, mm-1.3]
- [motionmark-htmlsuite-1-3, mm-h-1.3]
- [speedometer, sp]
- [speedometer3, sp3]
- [stylebench, sb]
- [sunspider, ss]
- [twitch-animation, tw-a]
- [unity-webgl, ugl]
- [webaudio, wa]
safari:
- ares6
- [assorted-dom, dom]
- [jetstream2, js2]
# - [matrix-react-bench, mrb] Perma fails on osx 10.15.x machines (Bug 1807754)
- [motionmark-animometer, mm-a]
- [motionmark-htmlsuite, mm-h]
- [motionmark-animometer-ramp, mm-ar]
- [motionmark-htmlsuite-ramp, mm-hr]
- [motionmark-1-3, mm-1.3]
- [motionmark-htmlsuite-1-3, mm-h-1.3]
- [speedometer, sp]
# - [speedometer3, sp3] Perma fails on osx 10.15.x machines (Bug 1809535)
- [stylebench, sb]
- [sunspider, ss]
# - [unity-webgl, ugl] Almost Perma failing on osx 10.15.x machines (Bug 1888398)
- [webaudio, wa]
custom-car:
- [speedometer, sp]
- [speedometer3, sp3]
- [motionmark-animometer-ramp, mm-ar]
- [motionmark-htmlsuite-ramp, mm-hr]
- [motionmark-1-3, mm-1.3]
- [motionmark-htmlsuite-1-3, mm-h-1.3]
chrome:
by-test-platform:
macosx1015.*shippable.*:
- ares6
- [assorted-dom, dom]
- [jetstream2, js2]
- [matrix-react-bench, mrb]
- [motionmark-animometer, mm-a]
- [motionmark-htmlsuite, mm-h]
- [motionmark-1-3, mm-1.3]
- [motionmark-htmlsuite-1-3, mm-h-1.3]
- [speedometer, sp]
- [speedometer3, sp3]
- [stylebench, sb]
- [sunspider, ss]
- [unity-webgl, ugl]
- [webaudio, wa]
default:
- ares6
- [assorted-dom, dom]
- [jetstream2, js2]
- [matrix-react-bench, mrb]
- [motionmark-animometer, mm-a]
- [motionmark-animometer-ramp, mm-ar]
- [motionmark-htmlsuite, mm-h]
- [motionmark-htmlsuite-ramp, mm-hr]
- [motionmark-1-3, mm-1.3]
- [motionmark-htmlsuite-1-3, mm-h-1.3]
- [speedometer, sp]
- [speedometer3, sp3]
- [stylebench, sb]
- [sunspider, ss]
- [unity-webgl, ugl]
- [webaudio, wa]
default:
- ares6
- [assorted-dom, dom]
- [jetstream2, js2]
- [matrix-react-bench, mrb]
- [motionmark-animometer, mm-a]
- [motionmark-animometer-ramp, mm-ar]
- [motionmark-htmlsuite, mm-h]
- [motionmark-htmlsuite-ramp, mm-hr]
- [motionmark-1-3, mm-1.3]
- [motionmark-htmlsuite-1-3, mm-h-1.3]
- [speedometer, sp]
- [speedometer3, sp3]
- [stylebench, sb]
- [sunspider, ss]
- [unity-webgl, ugl]
- [webaudio, wa]
treeherder-symbol: Btime()
tier:
by-app:
firefox:
by-subtest:
motionmark-animometer:
by-test-platform:
linux1804-64-clang-trunk-qr/opt: 2
default: 1
default:
by-test-platform:
windows10-64-ccov.*/.*: 3
linux64-ccov.*/.*: 3
default: 2
default:
by-subtest:
unity-webgl: 3
default: 2
run-on-projects:
by-subtest:
speedometer3:
by-app:
firefox:
by-test-platform:
windows.*-32.*: []
.*64.*shippable-qr/.*: [trunk]
.*nightlyasrelease.*: [autoland, mozilla-central]
default: []
custom-car:
by-test-platform:
(linux|windows10-64).*shippable-qr.*: [mozilla-central]
default: []
default: []
unity-webgl:
by-app:
firefox:
by-test-platform:
windows.*-32.*: []
# Disabled on autoland due to bug 1720063, and bug 1795826
.*64.*shippable-qr/.*: [mozilla-central]
default: []
default: []
speedometer:
by-app:
firefox:
by-test-platform:
windows.*-32.*: []
.*64.*shippable-qr/.*: [trunk]
.*nightlyasrelease.*: [autoland]
default: []
custom-car:
by-test-platform:
(linux|windows10-64).*shippable-qr.*: [mozilla-central]
default: []
default: []
default:
by-app:
firefox:
by-test-platform:
windows.*-32.*: []
.*64.*shippable-qr/.*: [trunk]
default: []
default: []
max-run-time:
by-subtest:
ares6: 1500
assorted-dom:
by-test-platform:
.*-qr/.*: 2100
default: 1500
jetstream2: 8000
matrix-react-bench: 1500
motionmark-htmlsuite: 1500
motionmark-htmlsuite-ramp: 3000
motionmark-animometer-ramp: 2100
motionmark-1-3: 3000
motionmark-htmlsuite-1-3: 3000
unity-webgl: 2400
speedometer3:
by-test-platform:
macosx1400.*shippable.*: 4200
default: 2100
speedometer: 2100
default: 900
mozharness:
extra-options:
by-test-platform:
macosx1400.*shippable.*:
- --extra-profiler-run
- --page-timeout=900000
default:
- --extra-profiler-run
browsertime-benchmark-wasm:
description: Raptor (browsertime) WASM benchmarks
raptor:
apps: [firefox, chrome, safari]
subtests:
by-app:
firefox:
- [wasm-godot, godot]
- [wasm-godot-baseline, godot-b]
- [wasm-godot-optimizing, godot-i]
- [wasm-misc, wm]
- [wasm-misc-baseline, wm-b]
- [wasm-misc-optimizing, wm-i]
safari:
# - [wasm-misc, wm] Perma fails (Bug 1809533)
- [wasm-godot, godot]
default:
- [wasm-misc, wm]
- [wasm-godot, godot]
treeherder-symbol: Btime()
tier: 2
run-on-projects:
by-app:
firefox:
by-test-platform:
windows.*-32.*: []
.*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central]
default: []
default: []
max-run-time:
by-subtest:
wasm-godot: 1500
wasm-godot-baseline: 1500
wasm-misc-baseline: 1500
default: 900
mozharness:
extra-options:
- --extra-profiler-run
browsertime-youtube-playback:
description: Browsertime YouTube Playback on Firefox
raptor:
apps: [firefox]
subtests:
by-app:
firefox:
- [youtube-playback-av1-sfr, ytp-av1-sfr]
- [youtube-playback-h264-sfr, ytp-h264-sfr]
- [youtube-playback-hfr, ytp-hfr]
- [youtube-playback-vp9-sfr, ytp-vp9-sfr]
- [youtube-playback-widevine-hfr, ytp-widevine-hfr]
- [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr]
- [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr]
treeherder-symbol: Btime()
run-on-projects:
by-test-platform:
windows.*-32.*: []
.*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central]
default: []
tier: 2
limit-platforms:
by-subtest:
youtube-playback-av1-sfr:
- macosx1015-64-shippable-qr/opt
default: []
max-run-time:
by-subtest:
youtube-playback-hfr: 3600
youtube-playback-widevine.*: 3600
default: 2700
browsertime-youtube-playback-power:
description: Browsertime YouTube Playback on Firefox with power usage tracking
raptor:
apps: [firefox]
subtests:
by-app:
firefox:
- [youtube-playback-h264-1080p30, ytp-h264-1080p30]
- [youtube-playback-h264-1080p60, ytp-h264-1080p60]
- [youtube-playback-h264-full-1080p30, ytp-h264-full-1080p30]
- [youtube-playback-h264-full-1080p60, ytp-h264-full-1080p60]
- [youtube-playback-v9-1080p30, ytp-v9-1080p30]
- [youtube-playback-v9-1080p60, ytp-v9-1080p60]
- [youtube-playback-v9-full-1080p30, ytp-v9-full-1080p30]
- [youtube-playback-v9-full-1080p60, ytp-v9-full-1080p60]
treeherder-symbol: Btime()
run-on-projects:
by-test-platform:
macosx.*shippable-qr.*: [mozilla-central]
default: []
tier: 2
max-run-time: 2700
browsertime-tp6-live:
<<: *tp6-defaults
description: Raptor (browsertime) tp6 on live-sites
mozharness:
extra-options:
- --browser-cycles=15
- --chimera
- --live-sites
- --extra-profiler-run
raptor:
apps: [firefox, chrome, safari]
test: tp6
run-visual-metrics: true
subtests:
by-app:
safari:
- amazon
- [bing-search, bing]
- ebay
# - [facebook, fb] Perma fails on osx 10.15.7
- fandom
- [google-docs, gdocs]
- [google-mail, gmail]
- [google-search, gsearch]
- [google-slides, gslides]
- imdb
- imgur
# - instagram # (see bug 1736999)
- linkedin
- microsoft
# - netflix # Perma fails on osx 10.15.7 (Bug 1802922)
# - office (site loads blank page if not signed in)
# - outlook Perma fails on osx 10.15.7
- paypal
- pinterest
- reddit
- tumblr
- twitch
# - twitter Perma fails on osx 10.15.7
- wikipedia
- yahoo-mail
- youtube
chrome:
- amazon
- [bing-search, bing]
- ebay
- [facebook, fb]
- fandom
- [google-docs, gdocs]
- [google-mail, gmail]
- [google-search, gsearch]
- [google-slides, gslides]
- imdb
# - imgur (see bugs 1825337 & 1817420)
- instagram
- linkedin
- microsoft
- netflix
# - office (site loads blank page if not signed in)
- outlook
- paypal
- pinterest
- reddit
- tumblr
- twitch
- twitter
- wikipedia
- yahoo-mail
- youtube
default:
- amazon
- [bing-search, bing]
- ebay
- [facebook, fb]
- fandom
- [google-docs, gdocs]
- [google-mail, gmail]
- [google-search, gsearch]
- [google-slides, gslides]
- imdb
- imgur
- instagram
- linkedin
- microsoft
- netflix
# - office (site loads blank page if not signed in)
- outlook
- paypal
- pinterest
- reddit
- tumblr
- twitch
- twitter
- wikipedia
- yahoo-mail
- youtube
run-on-projects: []
tier: 3
treeherder-symbol: Btime-live(tp6)
browsertime-tp6-live-sheriffed:
<<: *tp6-defaults
raptor:
apps: [firefox]
test: tp6
run-visual-metrics: true
subtests:
- cnn
description: Raptor (browsertime) tp6 on live-sites
mozharness:
extra-options:
- --browser-cycles=5
- --chimera
- --live-sites
- --extra-profiler-run
- --browsertime-arg=maxLoadTime=240000
run-on-projects:
by-test-platform:
(linux|windows10-64|macos)(?!.*shippable).*: []
(linux|windows|macosx1015)(?!.*-qr).*: []
windows.*-32.*: []
default: []
tier: 2
treeherder-symbol: Btime-live(tp6)
browsertime-tp6-profiling:
<<: *tp6-defaults
raptor:
apps: [firefox]
test: tp6
run-visual-metrics: true
subtests: [amazon]
description: Raptor (browsertime) tp6 page-load tests with Gecko Profiling
mozharness:
extra-options:
- --chimera
- --gecko-profile
run-on-projects:
by-test-platform:
(linux|windows|macosx1015).*shippable-qr.*: [mozilla-central]
default: []
tier: 2
treeherder-symbol: Btime-Prof(tp6)
browsertime-custom:
raptor:
apps: [firefox]
run-visual-metrics: true
subtests: [[process-switch, ps]]
description: Raptor (browsertime) custom pageload tests
max-run-time: 2700
mozharness:
extra-options:
- --chimera
- --extra-profiler-run
run-on-projects:
by-test-platform:
windows.*-32.*: []
.*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central]
default: []
tier: 1
treeherder-symbol: Btime(ps)
browsertime-first-install:
raptor:
apps: [firefox]
run-visual-metrics: true
subtests: [welcome]
description: Raptor (browsertime) custom pageload tests
max-run-time: 2700
mozharness:
extra-options:
- --chimera
- --extra-profiler-run
run-on-projects:
by-test-platform:
windows.*-32.*: []
.*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk]
default: []
tier: 1
treeherder-symbol: Btime(welcome)
browsertime-regression-tests:
raptor:
apps: [firefox]
subtests: [[constant-regression, regress-c]]
description: Raptor (browsertime) tests for our regression detection system
max-run-time: 700
run-on-projects: [autoland]
limit-platforms:
- .*linux(?!.*shippable)(?!.*clang).*
mozharness:
extra-options:
by-test-platform:
windows10-64.*shippable.*-qr.*: []
default: []
tier: 2
treeherder-symbol: Btime(regressor)
browsertime-upload:
raptor:
apps: [firefox, chrome, safari]
subtests:
- [upload, upload-h2]
- upload-h3
description: Raptor (browsertime) http2, http3 file upload performance test
max-run-time: 2700
run-on-projects:
by-app:
firefox:
by-test-platform:
windows.*-32.*: []
.*64.*shippable-qr/.*: [autoland, mozilla-central]
default: []
default: []
tier: 3
treeherder-symbol: Btime(upload)
fetches:
fetch:
- upload-speed-test-file
browsertime-pageload-benchmark:
raptor:
apps: [firefox]
subtests: [tp6-bench]
description: Raptor (browsertime) TP6 pageload benchmark test
max-run-time: 4500
mozharness:
extra-options:
- --browser-cycles=5
run-on-projects: []
tier: 2
treeherder-symbol: Btime(tp6-bench)
browsertime-speculative:
raptor:
apps: [firefox]
run-visual-metrics: true
subtests: [connect]
description: Raptor (browsertime) speculative connect pageload tests
max-run-time: 420
run-on-projects:
by-test-platform:
windows.*-32.*: []
.*64.*shippable-qr/.*: [autoland]
default: []
tier: 2
treeherder-symbol: Btime(speculative)
browsertime-throttled:
raptor:
apps: [firefox]
run-visual-metrics: true
subtests: [throttled]
description: Raptor (browsertime) throttled network pageload tests
max-run-time: 420
run-on-projects: []
tier: 3
treeherder-symbol: Btime(throttled)
browsertime-tp6-webextensions:
<<: *tp6-defaults
<<: *tp6-essential-defaults
description: Raptor (browsertime) tp6 page-load tests with webextensions installed on Firefox Desktop
raptor:
apps: [firefox]
test: tp6
run-visual-metrics: true
subtests:
- amazon
fetches:
fetch:
- firefox-addons
run-on-projects:
by-app:
firefox:
by-test-platform:
windows.*-32.*: []
macosx1100.*shippable.*: [mozilla-central]
.*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk-only]
default: []
default: []
mozharness:
extra-options:
by-test-platform:
windows10-64.*shippable.*-qr.*:
- --chimera
- --browsertime
- --browsertime-no-ffwindowrecorder
- --conditioned-profile=settled-webext
default:
- --chimera
- --browsertime
- --conditioned-profile=settled-webext
# TODO(Bug 1867781): promote to tier 2 after confirming no issue have been caught while runing as tier 3.
tier: 3
treeherder-symbol: Btime-webext(tp6)
browsertime-indexeddb:
description: Raptor (browsertime) tests for IndexedDB
max-run-time: 2700
treeherder-symbol: Btime(idb)
limit-platforms:
by-app:
chrome:
- .*shippable.*
custom-car:
- .*64.*shippable.*
default: []
raptor:
apps: [firefox, chrome]
run-visual-metrics: false
subtests:
- addMab1
- addMabN
- addMar1
- addMarN
- addMbl1
- addMblN
- addkAB1
- addkABN
- addkAR1
- addkARN
- addkBL1
- addkBLN
- getkeyrng
run-on-projects:
by-app:
firefox:
by-test-platform:
windows.*-32.*: []
.*64.*shippable-qr/.*: [autoland, mozilla-central]
default: []
default: []
tier:
by-app:
firefox: 2
default: 3
mozharness:
extra-options:
- --extra-profiler-run