Source code

Revision control

Other Tools

1
#!/usr/bin/env python
2
3
# This Source Code Form is subject to the terms of the Mozilla Public
4
# License, v. 2.0. If a copy of the MPL was not distributed with this
5
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
6
7
from __future__ import absolute_import
8
9
from abc import ABCMeta, abstractmethod
10
import json
11
import os
12
import posixpath
13
import shutil
14
import signal
15
import sys
16
import tempfile
17
import time
18
19
import requests
20
21
import mozcrash
22
import mozinfo
23
import mozprocess
24
from logger.logger import RaptorLogger
25
from mozdevice import ADBDevice
26
from mozlog import commandline
27
from mozpower import MozPower
28
from mozprofile import create_profile
29
from mozproxy import get_playback
30
from mozrunner import runners
31
32
# need this so raptor imports work both from /raptor and via mach
33
here = os.path.abspath(os.path.dirname(__file__))
34
paths = [here]
35
36
webext_dir = os.path.join(here, '..', 'webext')
37
paths.append(webext_dir)
38
39
for path in paths:
40
if not os.path.exists(path):
41
raise IOError("%s does not exist. " % path)
42
sys.path.insert(0, path)
43
44
try:
45
from mozbuild.base import MozbuildObject
46
47
build = MozbuildObject.from_environment(cwd=here)
48
except ImportError:
49
build = None
50
51
from benchmark import Benchmark
52
from cmdline import (parse_args,
53
FIREFOX_ANDROID_APPS,
54
CHROMIUM_DISTROS)
55
from control_server import RaptorControlServer
56
from gecko_profile import GeckoProfile
57
from gen_test_config import gen_test_config
58
from outputhandler import OutputHandler
59
from manifest import get_raptor_test_list
60
from memory import generate_android_memory_profile
61
from performance_tuning import tune_performance
62
from power import init_android_power_test, finish_android_power_test
63
from results import RaptorResultsHandler
64
from utils import view_gecko_profile, write_yml_file
65
from cpu import start_android_cpu_profiler
66
67
LOG = RaptorLogger(component='raptor-main')
68
69
70
class SignalHandler:
71
72
def __init__(self):
73
signal.signal(signal.SIGINT, self.handle_signal)
74
signal.signal(signal.SIGTERM, self.handle_signal)
75
76
def handle_signal(self, signum, frame):
77
raise SignalHandlerException("Program aborted due to signal %s" % signum)
78
79
80
class SignalHandlerException(Exception):
81
pass
82
83
84
class Perftest(object):
85
"""Abstract base class for perftests that execute via a subharness,
86
either Raptor or browsertime."""
87
88
__metaclass__ = ABCMeta
89
90
def __init__(self, app, binary, run_local=False, noinstall=False,
91
obj_path=None, profile_class=None,
92
gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None,
93
symbols_path=None, host=None, power_test=False, cpu_test=False, memory_test=False,
94
is_release_build=False, debug_mode=False, post_startup_delay=None,
95
interrupt_handler=None, e10s=True, enable_webrender=False, **kwargs):
96
97
# Override the magic --host HOST_IP with the value of the environment variable.
98
if host == 'HOST_IP':
99
host = os.environ['HOST_IP']
100
101
self.config = {
102
'app': app,
103
'binary': binary,
104
'platform': mozinfo.os,
105
'processor': mozinfo.processor,
106
'run_local': run_local,
107
'obj_path': obj_path,
108
'gecko_profile': gecko_profile,
109
'gecko_profile_interval': gecko_profile_interval,
110
'gecko_profile_entries': gecko_profile_entries,
111
'symbols_path': symbols_path,
112
'host': host,
113
'power_test': power_test,
114
'memory_test': memory_test,
115
'cpu_test': cpu_test,
116
'is_release_build': is_release_build,
117
'enable_control_server_wait': memory_test or cpu_test,
118
'e10s': e10s,
119
'enable_webrender': enable_webrender,
120
}
121
# We can never use e10s on fennec
122
if self.config['app'] == 'fennec':
123
self.config['e10s'] = False
124
125
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
126
self.playback = None
127
self.benchmark = None
128
self.benchmark_port = 0
129
self.gecko_profiler = None
130
self.post_startup_delay = post_startup_delay
131
self.device = None
132
self.profile_class = profile_class or app
133
self.firefox_android_apps = FIREFOX_ANDROID_APPS
134
self.interrupt_handler = interrupt_handler
135
136
# debug mode is currently only supported when running locally
137
self.debug_mode = debug_mode if self.config['run_local'] else False
138
139
# if running debug-mode reduce the pause after browser startup
140
if self.debug_mode:
141
self.post_startup_delay = min(self.post_startup_delay, 3000)
142
LOG.info("debug-mode enabled, reducing post-browser startup pause to %d ms"
143
% self.post_startup_delay)
144
145
LOG.info("main raptor init, config is: %s" % str(self.config))
146
147
# setup the control server
148
self.results_handler = RaptorResultsHandler(self.config)
149
150
self.build_browser_profile()
151
152
def build_browser_profile(self):
153
self.profile = create_profile(self.profile_class)
154
155
# Merge extra profile data from testing/profiles
156
with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh:
157
base_profiles = json.load(fh)['raptor']
158
159
for profile in base_profiles:
160
path = os.path.join(self.profile_data_dir, profile)
161
LOG.info("Merging profile: {}".format(path))
162
self.profile.merge(path)
163
164
# share the profile dir with the config and the control server
165
self.config['local_profile_dir'] = self.profile.profile
166
167
@property
168
def profile_data_dir(self):
169
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
170
return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles')
171
if build:
172
return os.path.join(build.topsrcdir, 'testing', 'profiles')
173
return os.path.join(here, 'profile_data')
174
175
@property
176
def artifact_dir(self):
177
artifact_dir = os.getcwd()
178
if self.config.get('run_local', False):
179
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
180
artifact_dir = os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
181
'testing', 'mozharness', 'build')
182
else:
183
artifact_dir = here
184
elif os.getenv('MOZ_UPLOAD_DIR'):
185
artifact_dir = os.getenv('MOZ_UPLOAD_DIR')
186
return artifact_dir
187
188
@abstractmethod
189
def run_test_setup(self, test):
190
LOG.info("starting test: %s" % test['name'])
191
192
def run_tests(self, tests, test_names):
193
try:
194
for test in tests:
195
try:
196
self.run_test(test, timeout=int(test.get('page_timeout')))
197
except RuntimeError as e:
198
LOG.critical("Tests failed to finish! Application timed out.")
199
LOG.error(e)
200
finally:
201
self.run_test_teardown(test)
202
return self.process_results(test_names)
203
finally:
204
self.clean_up()
205
206
@abstractmethod
207
def run_test(self, test, timeout):
208
raise NotImplementedError()
209
210
@abstractmethod
211
def run_test_teardown(self, test):
212
self.check_for_crashes()
213
214
# gecko profiling symbolication
215
if self.config['gecko_profile'] is True:
216
self.gecko_profiler.symbolicate()
217
# clean up the temp gecko profiling folders
218
LOG.info("cleaning up after gecko profiling")
219
self.gecko_profiler.clean()
220
221
def process_results(self, test_names):
222
# when running locally output results in build/raptor.json; when running
223
# in production output to a local.json to be turned into tc job artifact
224
raptor_json_path = os.path.join(self.artifact_dir, 'raptor.json')
225
if not self.config.get('run_local', False):
226
raptor_json_path = os.path.join(os.getcwd(), 'local.json')
227
228
self.config['raptor_json_path'] = raptor_json_path
229
return self.results_handler.summarize_and_output(self.config, test_names)
230
231
@abstractmethod
232
def check_for_crashes(self):
233
pass
234
235
@abstractmethod
236
def clean_up(self):
237
pass
238
239
def get_page_timeout_list(self):
240
return self.results_handler.page_timeout_list
241
242
def get_recording_paths(self, test):
243
recordings = test.get("playback_recordings")
244
245
if recordings:
246
recording_paths = []
247
proxy_dir = self.playback.mozproxy_dir
248
249
for recording in recordings.split():
250
if not recording:
251
continue
252
recording_paths.append(os.path.join(proxy_dir, recording))
253
254
return recording_paths
255
256
def log_recording_dates(self, test):
257
_recording_paths = self.get_recording_paths(test)
258
if _recording_paths is None:
259
LOG.info("No playback recordings specified in the test; so not getting recording info")
260
return
261
262
for r in _recording_paths:
263
json_path = '{}.json'.format(r.split('.')[0])
264
265
if os.path.exists(json_path):
266
with open(json_path) as f:
267
recording_date = json.loads(f.read()).get('recording_date')
268
269
if recording_date is not None:
270
LOG.info('Playback recording date: {} '.
271
format(recording_date.split(' ')[0]))
272
else:
273
LOG.info('Playback recording date not available')
274
else:
275
LOG.info('Playback recording information not available')
276
277
def get_playback_config(self, test):
278
platform = self.config['platform']
279
playback_dir = os.path.join(here, 'playback')
280
281
self.config.update({
282
'playback_tool': test.get('playback'),
283
'playback_version': test.get('playback_version', "4.0.4"),
284
'playback_binary_zip': test.get('playback_binary_zip_%s' % platform),
285
'playback_pageset_zip': test.get('playback_pageset_zip_%s' % platform),
286
'playback_binary_manifest': test.get('playback_binary_manifest'),
287
'playback_pageset_manifest': test.get('playback_pageset_manifest'),
288
})
289
290
for key in ('playback_pageset_manifest', 'playback_pageset_zip'):
291
if self.config.get(key) is None:
292
continue
293
self.config[key] = os.path.join(playback_dir, self.config[key])
294
295
LOG.info("test uses playback tool: %s " % self.config['playback_tool'])
296
297
def delete_proxy_settings_from_profile(self):
298
# Must delete the proxy settings from the profile if running
299
# the test with a host different from localhost.
300
userjspath = os.path.join(self.profile.profile, 'user.js')
301
with open(userjspath) as userjsfile:
302
prefs = userjsfile.readlines()
303
prefs = [pref for pref in prefs if 'network.proxy' not in pref]
304
with open(userjspath, 'w') as userjsfile:
305
userjsfile.writelines(prefs)
306
307
def start_playback(self, test):
308
# creating the playback tool
309
self.get_playback_config(test)
310
self.playback = get_playback(self.config, self.device)
311
312
self.playback.config['playback_files'] = self.get_recording_paths(test)
313
314
# let's start it!
315
self.playback.start()
316
317
self.log_recording_dates(test)
318
319
320
class Browsertime(Perftest):
321
"""Container class for Browsertime"""
322
323
def __init__(self, *args, **kwargs):
324
for key in kwargs.keys():
325
if key.startswith('browsertime_'):
326
value = kwargs.pop(key)
327
setattr(self, key, value)
328
329
super(Browsertime, self).__init__(*args, **kwargs)
330
331
LOG.info("cwd: '{}'".format(os.getcwd()))
332
333
# For debugging.
334
for k in ("browsertime_node",
335
"browsertime_browsertimejs",
336
"browsertime_ffmpeg",
337
"browsertime_geckodriver",
338
"browsertime_chromedriver"):
339
LOG.info("{}: {}".format(k, getattr(self, k)))
340
try:
341
LOG.info("{}: {}".format(k, os.stat(getattr(self, k))))
342
except Exception as e:
343
LOG.info("{}: {}".format(k, e))
344
345
def run_test_setup(self, test):
346
super(Browsertime, self).run_test_setup(test)
347
348
if test.get('playback') is not None:
349
self.start_playback(test)
350
351
# TODO: geckodriver/chromedriver from tasks.
352
self.driver_paths = []
353
if self.browsertime_geckodriver:
354
self.driver_paths.extend(['--firefox.geckodriverPath', self.browsertime_geckodriver])
355
if self.browsertime_chromedriver:
356
self.driver_paths.extend(['--chrome.chromedriverPath', self.browsertime_chromedriver])
357
358
self.resultdir = [
359
'--resultDir',
360
os.path.join(os.environ.get('MOZ_UPLOAD_DIR', os.getcwd()),
361
'browsertime-results', test['name']),
362
]
363
364
LOG.info('test: {}'.format(test))
365
366
def run_test_teardown(self, test):
367
super(Browsertime, self).run_test_teardown(test)
368
369
# if we were using a playback tool, stop it
370
if self.playback is not None:
371
self.playback.stop()
372
373
def check_for_crashes(self):
374
super(Browsertime, self).check_for_crashes()
375
376
def clean_up(self):
377
super(Browsertime, self).clean_up()
378
379
def run_test(self, test, timeout):
380
381
self.run_test_setup(test)
382
383
cmd = [self.browsertime_node, self.browsertime_browsertimejs, '--browser', 'firefox'] + \
384
self.driver_paths + \
385
['--firefox.binaryPath', self.config['binary'],
386
'--skipHar',
387
'--video', 'true',
388
'--visualMetrics', 'false',
389
'-vv'] + \
390
self.resultdir + \
391
['-n', str(test.get('browser_cycles', 1)),
392
test['test_url']]
393
394
# timeout is a single page-load timeout value in ms from the test INI
395
# convert timeout to seconds and account for browser cycles
396
timeout = int(timeout / 1000) * int(test.get('browser_cycles', 1))
397
398
# add some time for browser startup, time for the browsertime measurement code
399
# to be injected/invoked, and for exceptions to bubble up; be generous
400
timeout += (20 * int(test.get('browser_cycles', 1)))
401
402
# if geckoProfile enabled, give browser more time for profiling
403
if self.config['gecko_profile'] is True:
404
timeout += 5 * 60
405
406
LOG.info('timeout (s): {}'.format(timeout))
407
LOG.info('browsertime cwd: {}'.format(os.getcwd()))
408
LOG.info('browsertime cmd: {}'.format(cmd))
409
410
# browsertime requires ffmpeg on the PATH for `--video=true`.
411
# It's easier to configure the PATH here than at the TC level.
412
env = dict(os.environ)
413
if self.browsertime_ffmpeg:
414
ffmpeg_dir = os.path.dirname(os.path.abspath(self.browsertime_ffmpeg))
415
old_path = env.setdefault('PATH', '')
416
new_path = os.pathsep.join([ffmpeg_dir, old_path])
417
if isinstance(new_path, unicode):
418
# Python 2 doesn't like unicode in the environment.
419
new_path = new_path.encode('utf-8', 'strict')
420
env['PATH'] = new_path
421
422
try:
423
proc = mozprocess.ProcessHandler(cmd, env=env)
424
proc.run(timeout=timeout,
425
outputTimeout=2*60)
426
proc.wait()
427
428
except Exception as e:
429
raise Exception("Error while attempting to run browsertime: %s" % str(e))
430
431
def process_results(self, test_names):
432
# TODO - Bug 1565316 - Process browsertime results and dump out for perfherder
433
LOG.info("TODO: Bug 1565316 - Process browsertime results and dump out for perfherder")
434
435
436
class Raptor(Perftest):
437
"""Container class for Raptor"""
438
439
def __init__(self, *args, **kwargs):
440
self.raptor_webext = None
441
self.control_server = None
442
self.cpu_profiler = None
443
444
super(Raptor, self).__init__(*args, **kwargs)
445
446
self.start_control_server()
447
448
def run_test_setup(self, test):
449
super(Raptor, self).run_test_setup(test)
450
451
LOG.info("starting raptor test: %s" % test['name'])
452
LOG.info("test settings: %s" % str(test))
453
LOG.info("raptor config: %s" % str(self.config))
454
455
if test.get('type') == "benchmark":
456
self.serve_benchmark_source(test)
457
458
gen_test_config(
459
self.config['app'],
460
test['name'],
461
self.control_server.port,
462
self.post_startup_delay,
463
host=self.config['host'],
464
b_port=self.benchmark_port,
465
debug_mode=1 if self.debug_mode else 0,
466
browser_cycle=test.get('browser_cycle', 1),
467
)
468
469
self.install_raptor_webext()
470
471
if test.get("preferences") is not None:
472
self.set_browser_test_prefs(test['preferences'])
473
474
# if 'alert_on' was provided in the test INI, add to our config for results/output
475
self.config['subtest_alert_on'] = test.get('alert_on')
476
477
def wait_for_test_finish(self, test, timeout):
478
# this is a 'back-stop' i.e. if for some reason Raptor doesn't finish for some
479
# serious problem; i.e. the test was unable to send a 'page-timeout' to the control
480
# server, etc. Therefore since this is a 'back-stop' we want to be generous here;
481
# we don't want this timeout occurring unless abosultely necessary
482
483
# convert timeout to seconds and account for page cycles
484
timeout = int(timeout / 1000) * int(test.get('page_cycles', 1))
485
# account for the pause the raptor webext runner takes after browser startup
486
# and the time an exception is propagated through the framework
487
timeout += (int(self.post_startup_delay / 1000) + 10)
488
489
# for page-load tests we don't start the page-timeout timer until the pageload.js content
490
# is successfully injected and invoked; which differs per site being tested; therefore we
491
# need to be generous here - let's add 10 seconds extra per page-cycle
492
if test.get('type') == "pageload":
493
timeout += (10 * int(test.get('page_cycles', 1)))
494
495
# if geckoProfile enabled, give browser more time for profiling
496
if self.config['gecko_profile'] is True:
497
timeout += 5 * 60
498
499
# we also need to give time for results processing, not just page/browser cycles!
500
timeout += 60
501
502
elapsed_time = 0
503
while not self.control_server._finished:
504
if self.config['enable_control_server_wait']:
505
response = self.control_server_wait_get()
506
if response == 'webext_shutdownBrowser':
507
if self.config['memory_test']:
508
generate_android_memory_profile(self, test['name'])
509
if self.cpu_profiler:
510
self.cpu_profiler.generate_android_cpu_profile(test['name'])
511
512
self.control_server_wait_continue()
513
time.sleep(1)
514
# we only want to force browser-shutdown on timeout if not in debug mode;
515
# in debug-mode we leave the browser running (require manual shutdown)
516
if not self.debug_mode:
517
elapsed_time += 1
518
if elapsed_time > (timeout) - 5: # stop 5 seconds early
519
self.control_server.wait_for_quit()
520
raise RuntimeError("Test failed to finish. "
521
"Application timed out after {} seconds".format(timeout))
522
523
def run_test_teardown(self, test):
524
super(Raptor, self).run_test_teardown(test)
525
526
if self.playback is not None:
527
self.playback.stop()
528
529
self.remove_raptor_webext()
530
531
def set_browser_test_prefs(self, raw_prefs):
532
# add test specific preferences
533
LOG.info("setting test-specific Firefox preferences")
534
self.profile.set_preferences(json.loads(raw_prefs))
535
536
def build_browser_profile(self):
537
super(Raptor, self).build_browser_profile()
538
539
if self.control_server:
540
# The control server and the browser profile are not well factored
541
# at this time, so the start-up process overlaps. Accommodate.
542
self.control_server.user_profile = self.profile
543
544
def start_control_server(self):
545
self.control_server = RaptorControlServer(self.results_handler, self.debug_mode)
546
self.control_server.user_profile = self.profile
547
self.control_server.start()
548
549
if self.config['enable_control_server_wait']:
550
self.control_server_wait_set('webext_shutdownBrowser')
551
552
def serve_benchmark_source(self, test):
553
# benchmark-type tests require the benchmark test to be served out
554
self.benchmark = Benchmark(self.config, test)
555
self.benchmark_port = int(self.benchmark.port)
556
557
def install_raptor_webext(self):
558
# must intall raptor addon each time because we dynamically update some content
559
# the webext is installed into the browser profile
560
# note: for chrome the addon is just a list of paths that ultimately are added
561
# to the chromium command line '--load-extension' argument
562
self.raptor_webext = os.path.join(webext_dir, 'raptor')
563
LOG.info("installing webext %s" % self.raptor_webext)
564
self.profile.addons.install(self.raptor_webext)
565
566
# on firefox we can get an addon id; chrome addon actually is just cmd line arg
567
try:
568
self.webext_id = self.profile.addons.addon_details(self.raptor_webext)['id']
569
except AttributeError:
570
self.webext_id = None
571
572
def remove_raptor_webext(self):
573
# remove the raptor webext; as it must be reloaded with each subtest anyway
574
if not self.raptor_webext:
575
LOG.info("raptor webext not installed - not attempting removal")
576
return
577
578
LOG.info("removing webext %s" % self.raptor_webext)
579
if self.config['app'] in ['firefox', 'geckoview', 'fennec', 'refbrow', 'fenix']:
580
self.profile.addons.remove_addon(self.webext_id)
581
582
# for chrome the addon is just a list (appended to cmd line)
583
chrome_apps = CHROMIUM_DISTROS + ["chrome-android", "chromium-android"]
584
if self.config['app'] in chrome_apps:
585
self.profile.addons.remove(self.raptor_webext)
586
587
def _init_gecko_profiling(self, test):
588
LOG.info("initializing gecko profiler")
589
upload_dir = os.getenv('MOZ_UPLOAD_DIR')
590
if not upload_dir:
591
LOG.critical("Profiling ignored because MOZ_UPLOAD_DIR was not set")
592
else:
593
self.gecko_profiler = GeckoProfile(upload_dir,
594
self.config,
595
test)
596
597
def clean_up(self):
598
super(Raptor, self).clean_up()
599
600
if self.config['enable_control_server_wait']:
601
self.control_server_wait_clear('all')
602
603
self.control_server.stop()
604
LOG.info("finished")
605
606
def control_server_wait_set(self, state):
607
response = requests.post("http://127.0.0.1:%s/" % self.control_server.port,
608
json={"type": "wait-set", "data": state})
609
return response.content
610
611
def control_server_wait_timeout(self, timeout):
612
response = requests.post("http://127.0.0.1:%s/" % self.control_server.port,
613
json={"type": "wait-timeout", "data": timeout})
614
return response.content
615
616
def control_server_wait_get(self):
617
response = requests.post("http://127.0.0.1:%s/" % self.control_server.port,
618
json={"type": "wait-get", "data": ""})
619
return response.content
620
621
def control_server_wait_continue(self):
622
response = requests.post("http://127.0.0.1:%s/" % self.control_server.port,
623
json={"type": "wait-continue", "data": ""})
624
return response.content
625
626
def control_server_wait_clear(self, state):
627
response = requests.post("http://127.0.0.1:%s/" % self.control_server.port,
628
json={"type": "wait-clear", "data": state})
629
return response.content
630
631
632
class RaptorDesktop(Raptor):
633
634
def __init__(self, *args, **kwargs):
635
super(RaptorDesktop, self).__init__(*args, **kwargs)
636
637
# create the desktop browser runner
638
LOG.info("creating browser runner using mozrunner")
639
self.output_handler = OutputHandler()
640
process_args = {
641
'processOutputLine': [self.output_handler],
642
}
643
runner_cls = runners[self.config['app']]
644
self.runner = runner_cls(
645
self.config['binary'], profile=self.profile, process_args=process_args,
646
symbols_path=self.config['symbols_path'])
647
648
if self.config['enable_webrender']:
649
self.runner.env['MOZ_WEBRENDER'] = '1'
650
self.runner.env['MOZ_ACCELERATED'] = '1'
651
else:
652
self.runner.env['MOZ_WEBRENDER'] = '0'
653
654
def launch_desktop_browser(self, test):
655
raise NotImplementedError
656
657
def start_runner_proc(self):
658
# launch the browser via our previously-created runner
659
self.runner.start()
660
661
proc = self.runner.process_handler
662
self.output_handler.proc = proc
663
664
# give our control server the browser process so it can shut it down later
665
self.control_server.browser_proc = proc
666
667
def run_test(self, test, timeout):
668
# tests will be run warm (i.e. NO browser restart between page-cycles)
669
# unless otheriwse specified in the test INI by using 'cold = true'
670
mozpower_measurer = None
671
if self.config.get('power_test', False):
672
output_dir = os.path.join(self.artifact_dir, 'power-measurements')
673
test_dir = os.path.join(output_dir, test['name'].replace('/', '-').replace('\\', '-'))
674
675
try:
676
if not os.path.exists(output_dir):
677
os.mkdir(output_dir)
678
if not os.path.exists(test_dir):
679
os.mkdir(test_dir)
680
except Exception as e:
681
LOG.critical("Could not create directories to store power testing data.")
682
raise e
683
684
# Start power measurements with IPG creating a power usage log
685
# every 30 seconds with 1 data point per second (or a 1000 milli-
686
# second sampling rate).
687
mozpower_measurer = MozPower(
688
ipg_measure_duration=30,
689
sampling_rate=1000,
690
output_file_path=os.path.join(test_dir, 'power-usage')
691
)
692
mozpower_measurer.initialize_power_measurements()
693
694
if test.get('cold', False) is True:
695
self.__run_test_cold(test, timeout)
696
else:
697
self.__run_test_warm(test, timeout)
698
699
if mozpower_measurer:
700
mozpower_measurer.finalize_power_measurements(test_name=test['name'])
701
perfherder_data = mozpower_measurer.get_perfherder_data()
702
703
if not self.config.get('run_local', False):
704
# when not running locally, zip the data and delete the folder which
705
# was placed in the zip
706
power_data_path = os.path.join(self.artifact_dir, 'power-measurements')
707
shutil.make_archive(power_data_path + '.zip', 'zip', power_data_path)
708
shutil.rmtree(power_data_path)
709
710
self.control_server.submit_supporting_data(perfherder_data['utilization'])
711
self.control_server.submit_supporting_data(perfherder_data['power-usage'])
712
713
def __run_test_cold(self, test, timeout):
714
'''
715
Run the Raptor test but restart the entire browser app between page-cycles.
716
717
Note: For page-load tests, playback will only be started once - at the beginning of all
718
browser cycles, and then stopped after all cycles are finished. That includes the import
719
of the mozproxy ssl cert and turning on the browser proxy.
720
721
Since we're running in cold-mode, before this point (in manifest.py) the
722
'expected-browser-cycles' value was already set to the initial 'page-cycles' value;
723
and the 'page-cycles' value was set to 1 as we want to perform one page-cycle per
724
browser restart.
725
726
The 'browser-cycle' value is the current overall browser start iteration. The control
727
server will receive the current 'browser-cycle' and the 'expected-browser-cycles' in
728
each results set received; and will pass that on as part of the results so that the
729
results processing will know results for multiple browser cycles are being received.
730
731
The default will be to run in warm mode; unless 'cold = true' is set in the test INI.
732
'''
733
LOG.info("test %s is running in cold mode; browser WILL be restarted between "
734
"page cycles" % test['name'])
735
736
for test['browser_cycle'] in range(1, test['expected_browser_cycles'] + 1):
737
738
LOG.info("begin browser cycle %d of %d for test %s"
739
% (test['browser_cycle'], test['expected_browser_cycles'], test['name']))
740
741
self.run_test_setup(test)
742
743
if test['browser_cycle'] == 1:
744
745
if test.get('playback') is not None:
746
self.start_playback(test)
747
748
if self.config['host'] not in ('localhost', '127.0.0.1'):
749
self.delete_proxy_settings_from_profile()
750
751
else:
752
# initial browser profile was already created before run_test was called;
753
# now additional browser cycles we want to create a new one each time
754
self.build_browser_profile()
755
756
self.run_test_setup(test)
757
758
# now start the browser/app under test
759
self.launch_desktop_browser(test)
760
761
# set our control server flag to indicate we are running the browser/app
762
self.control_server._finished = False
763
764
self.wait_for_test_finish(test, timeout)
765
766
def __run_test_warm(self, test, timeout):
767
self.run_test_setup(test)
768
769
if test.get('playback') is not None:
770
self.start_playback(test)
771
772
if self.config['host'] not in ('localhost', '127.0.0.1'):
773
self.delete_proxy_settings_from_profile()
774
775
# start the browser/app under test
776
self.launch_desktop_browser(test)
777
778
# set our control server flag to indicate we are running the browser/app
779
self.control_server._finished = False
780
781
self.wait_for_test_finish(test, timeout)
782
783
def run_test_teardown(self, test):
784
# browser should be closed by now but this is a backup-shutdown (if not in debug-mode)
785
if not self.debug_mode:
786
if self.runner.is_running():
787
self.runner.stop()
788
else:
789
# in debug mode, and running locally, leave the browser running
790
if self.config['run_local']:
791
LOG.info("* debug-mode enabled - please shutdown the browser manually...")
792
self.runner.wait(timeout=None)
793
794
super(RaptorDesktop, self).run_test_teardown(test)
795
796
def check_for_crashes(self):
797
super(RaptorDesktop, self).check_for_crashes()
798
799
try:
800
self.runner.check_for_crashes()
801
except NotImplementedError: # not implemented for Chrome
802
pass
803
804
def clean_up(self):
805
self.runner.stop()
806
807
super(RaptorDesktop, self).clean_up()
808
809
810
class RaptorDesktopFirefox(RaptorDesktop):
811
812
def disable_non_local_connections(self):
813
# For Firefox we need to set MOZ_DISABLE_NONLOCAL_CONNECTIONS=1 env var before startup
814
# when testing release builds from mozilla-beta/release. This is because of restrictions
815
# on release builds that require webextensions to be signed unless this env var is set
816
LOG.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=1")
817
os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "1"
818
819
def enable_non_local_connections(self):
820
# pageload tests need to be able to access non-local connections via mitmproxy
821
LOG.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=0")
822
os.environ['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = "0"
823
824
def launch_desktop_browser(self, test):
825
LOG.info("starting %s" % self.config['app'])
826
if self.config['is_release_build']:
827
self.disable_non_local_connections()
828
829
# if running debug-mode, tell Firefox to open the browser console on startup
830
if self.debug_mode:
831
self.runner.cmdargs.extend(['-jsconsole'])
832
833
self.start_runner_proc()
834
835
if self.config['is_release_build'] and test.get('playback') is not None:
836
self.enable_non_local_connections()
837
838
# if geckoProfile is enabled, initialize it
839
if self.config['gecko_profile'] is True:
840
self._init_gecko_profiling(test)
841
# tell the control server the gecko_profile dir; the control server
842
# will receive the filename of the stored gecko profile from the web
843
# extension, and will move it out of the browser user profile to
844
# this directory; where it is picked-up by gecko_profile.symbolicate
845
self.control_server.gecko_profile_dir = self.gecko_profiler.gecko_profile_dir
846
847
848
class RaptorDesktopChrome(RaptorDesktop):
849
850
def setup_chrome_desktop_for_playback(self):
851
# if running a pageload test on google chrome, add the cmd line options
852
# to turn on the proxy and ignore security certificate errors
853
# if using host localhost, 127.0.0.1.
854
chrome_args = [
855
'--proxy-server=127.0.0.1:8080',
856
'--proxy-bypass-list=localhost;127.0.0.1',
857
'--ignore-certificate-errors',
858
]
859
if self.config['host'] not in ('localhost', '127.0.0.1'):
860
chrome_args[0] = chrome_args[0].replace('127.0.0.1', self.config['host'])
861
if ' '.join(chrome_args) not in ' '.join(self.runner.cmdargs):
862
self.runner.cmdargs.extend(chrome_args)
863
864
def launch_desktop_browser(self, test):
865
LOG.info("starting %s" % self.config['app'])
866
# some chromium-specfic cmd line opts required
867
self.runner.cmdargs.extend(['--use-mock-keychain', '--no-default-browser-check'])
868
869
# if running in debug-mode, open the devtools on the raptor test tab
870
if self.debug_mode:
871
self.runner.cmdargs.extend(['--auto-open-devtools-for-tabs'])
872
873
if test.get('playback') is not None:
874
self.setup_chrome_desktop_for_playback()
875
876
self.start_runner_proc()
877
878
def set_browser_test_prefs(self, raw_prefs):
879
# add test-specific preferences
880
LOG.info("preferences were configured for the test, however \
881
we currently do not install them on non-Firefox browsers.")
882
883
884
class RaptorAndroid(Raptor):
885
886
def __init__(self, app, binary, activity=None, intent=None, **kwargs):
887
super(RaptorAndroid, self).__init__(app, binary, profile_class="firefox", **kwargs)
888
889
self.config.update({
890
'activity': activity,
891
'intent': intent,
892
})
893
894
self.remote_test_root = os.path.abspath(os.path.join(os.sep, 'sdcard', 'raptor'))
895
self.remote_profile = os.path.join(self.remote_test_root, "profile")
896
self.os_baseline_data = None
897
self.power_test_time = None
898
self.screen_off_timeout = 0
899
self.screen_brightness = 127
900
self.app_launched = False
901
902
def set_reverse_port(self, port):
903
tcp_port = "tcp:{}".format(port)
904
self.device.create_socket_connection('reverse', tcp_port, tcp_port)
905
906
def set_reverse_ports(self, is_benchmark=False):
907
# Make services running on the host available to the device
908
if self.config['host'] in ('localhost', '127.0.0.1'):
909
LOG.info("making the raptor control server port available to device")
910
self.set_reverse_port(self.control_server.port)
911
912
if self.config['host'] in ('localhost', '127.0.0.1'):
913
LOG.info("making the raptor playback server port available to device")
914
self.set_reverse_port(8080)
915
916
if is_benchmark and self.config['host'] in ('localhost', '127.0.0.1'):
917
LOG.info("making the raptor benchmarks server port available to device")
918
self.set_reverse_port(self.benchmark_port)
919
920
def setup_adb_device(self):
921
if self.device is None:
922
self.device = ADBDevice(verbose=True)
923
tune_performance(self.device, log=LOG)
924
925
LOG.info("creating remote root folder for raptor: %s" % self.remote_test_root)
926
self.device.rm(self.remote_test_root, force=True, recursive=True)
927
self.device.mkdir(self.remote_test_root)
928
self.device.chmod(self.remote_test_root, recursive=True, root=True)
929
930
self.clear_app_data()
931
self.set_debug_app_flag()
932
933
def build_browser_profile(self):
934
super(RaptorAndroid, self).build_browser_profile()
935
936
# Merge in the android profile
937
path = os.path.join(self.profile_data_dir, 'raptor-android')
938
LOG.info("Merging profile: {}".format(path))
939
self.profile.merge(path)
940
self.profile.set_preferences({'browser.tabs.remote.autostart': self.config['e10s']})
941
942
def clear_app_data(self):
943
LOG.info("clearing %s app data" % self.config['binary'])
944
self.device.shell("pm clear %s" % self.config['binary'])
945
946
def set_debug_app_flag(self):
947
# required so release apks will read the android config.yml file
948
LOG.info("setting debug-app flag for %s" % self.config['binary'])
949
self.device.shell("am set-debug-app --persistent %s" % self.config['binary'])
950
951
def copy_profile_to_device(self):
952
"""Copy the profile to the device, and update permissions of all files."""
953
if not self.device.is_app_installed(self.config['binary']):
954
raise Exception('%s is not installed' % self.config['binary'])
955
956
try:
957
LOG.info("copying profile to device: %s" % self.remote_profile)
958
self.device.rm(self.remote_profile, force=True, recursive=True)
959
# self.device.mkdir(self.remote_profile)
960
self.device.push(self.profile.profile, self.remote_profile)
961
self.device.chmod(self.remote_profile, recursive=True, root=True)
962
963
except Exception:
964
LOG.error("Unable to copy profile to device.")
965
raise
966
967
def turn_on_android_app_proxy(self):
968
# for geckoview/android pageload playback we can't use a policy to turn on the
969
# proxy; we need to set prefs instead; note that the 'host' may be different
970
# than '127.0.0.1' so we must set the prefs accordingly
971
LOG.info("setting profile prefs to turn on the android app proxy")
972
proxy_prefs = {}
973
proxy_prefs["network.proxy.type"] = 1
974
proxy_prefs["network.proxy.http"] = self.config['host']
975
proxy_prefs["network.proxy.http_port"] = 8080
976
proxy_prefs["network.proxy.ssl"] = self.config['host']
977
proxy_prefs["network.proxy.ssl_port"] = 8080
978
proxy_prefs["network.proxy.no_proxies_on"] = self.config['host']
979
self.profile.set_preferences(proxy_prefs)
980
981
def log_android_device_temperature(self):
982
try:
983
# retrieve and log the android device temperature
984
thermal_zone0 = self.device.shell_output('cat sys/class/thermal/thermal_zone0/temp')
985
thermal_zone0 = float(thermal_zone0)
986
zone_type = self.device.shell_output('cat sys/class/thermal/thermal_zone0/type')
987
LOG.info("(thermal_zone0) device temperature: %.3f zone type: %s"
988
% (thermal_zone0 / 1000, zone_type))
989
except Exception as exc:
990
LOG.warning("Unexpected error: {} - {}"
991
.format(exc.__class__.__name__, exc))
992
993
def write_android_app_config(self):
994
# geckoview supports having a local on-device config file; use this file
995
# to tell the app to use the specified browser profile, as well as other opts
996
# on-device: /data/local/tmp/com.yourcompany.yourapp-geckoview-config.yaml
998
999
# only supported for geckoview apps
1000
if self.config['app'] == "fennec":
1001
return
1002
LOG.info("creating android app config.yml")
1003
1004
yml_config_data = dict(
1005
args=['--profile', self.remote_profile, 'use_multiprocess', self.config['e10s']],
1006
env=dict(
1007
LOG_VERBOSE=1,
1008
R_LOG_LEVEL=6,
1009
MOZ_WEBRENDER=int(self.config['enable_webrender']),
1010
)
1011
)
1012
1013
yml_name = '%s-geckoview-config.yaml' % self.config['binary']
1014
yml_on_host = os.path.join(tempfile.mkdtemp(), yml_name)
1015
write_yml_file(yml_on_host, yml_config_data)
1016
yml_on_device = os.path.join('/data', 'local', 'tmp', yml_name)
1017
1018
try:
1019
LOG.info("copying %s to device: %s" % (yml_on_host, yml_on_device))
1020
self.device.rm(yml_on_device, force=True, recursive=True)
1021
self.device.push(yml_on_host, yml_on_device)
1022
1023
except Exception:
1024
LOG.critical("failed to push %s to device!" % yml_on_device)
1025
raise
1026
1027
def launch_firefox_android_app(self, test_name):
1028
LOG.info("starting %s" % self.config['app'])
1029
1030
extra_args = ["-profile", self.remote_profile,
1031
"--es", "env0", "LOG_VERBOSE=1",
1032
"--es", "env1", "R_LOG_LEVEL=6",
1033
"--es", "env2", "MOZ_WEBRENDER=%d" % self.config['enable_webrender']]
1034
1035
try:
1036
# make sure the android app is not already running
1037
self.device.stop_application(self.config['binary'])
1038
1039
if self.config['app'] == "fennec":
1040
self.device.launch_fennec(self.config['binary'],
1041
extra_args=extra_args,
1042
url='about:blank',
1043
fail_if_running=False)
1044
else:
1045
1046
# command line 'extra' args not used with geckoview apps; instead we use
1047
# an on-device config.yml file (see write_android_app_config)
1048
1049
self.device.launch_application(self.config['binary'],
1050
self.config['activity'],
1051
self.config['intent'],
1052
extras=None,
1053
url='about:blank',
1054
fail_if_running=False)
1055
1056
# Check if app has started and it's running
1057
if not self.device.process_exist(self.config['binary']):
1058
raise Exception("Error launching %s. App did not start properly!" %
1059
self.config['binary'])
1060
self.app_launched = True
1061
except Exception as e:
1062
LOG.error("Exception launching %s" % self.config['binary'])
1063
LOG.error("Exception: %s %s" % (type(e).__name__, str(e)))
1064
if self.config['power_test']:
1065
finish_android_power_test(self, test_name)
1066
raise
1067
1068
# give our control server the device and app info
1069
self.control_server.device = self.device
1070
self.control_server.app_name = self.config['binary']
1071
1072
def copy_cert_db(self, source_dir, target_dir):
1073
# copy browser cert db (that was previously created via certutil) from source to target
1074
cert_db_files = ['pkcs11.txt', 'key4.db', 'cert9.db']
1075
for next_file in cert_db_files:
1076
_source = os.path.join(source_dir, next_file)
1077
_dest = os.path.join(target_dir, next_file)
1078
if os.path.exists(_source):
1079
LOG.info("copying %s to %s" % (_source, _dest))
1080
shutil.copyfile(_source, _dest)
1081
else:
1082
LOG.critical("unable to find ssl cert db file: %s" % _source)
1083
1084
def run_tests(self, tests, test_names):
1085
self.setup_adb_device()
1086
1087
return super(RaptorAndroid, self).run_tests(tests, test_names)
1088
1089
def run_test_setup(self, test):
1090
super(RaptorAndroid, self).run_test_setup(test)
1091
1092
is_benchmark = test.get('type') == "benchmark"
1093
self.set_reverse_ports(is_benchmark=is_benchmark)
1094
1095
def run_test_teardown(self, test):
1096
LOG.info('removing reverse socket connections')
1097
self.device.remove_socket_connections('reverse')
1098
1099
super(RaptorAndroid, self).run_test_teardown(test)
1100
1101
def run_test(self, test, timeout):
1102
# tests will be run warm (i.e. NO browser restart between page-cycles)
1103
# unless otheriwse specified in the test INI by using 'cold = true'
1104
try:
1105
1106
if self.config['power_test']:
1107
# gather OS baseline data
1108
init_android_power_test(self)
1109
LOG.info("Running OS baseline, pausing for 1 minute...")
1110
time.sleep(60)
1111
finish_android_power_test(self, 'os-baseline', os_baseline=True)
1112
1113
# initialize for the test
1114
init_android_power_test(self)
1115
1116
if test.get('cold', False) is True:
1117
self.__run_test_cold(test, timeout)
1118
else:
1119
self.__run_test_warm(test, timeout)
1120
1121
except SignalHandlerException:
1122
self.device.stop_application(self.config['binary'])
1123
1124
finally:
1125
if self.config['power_test']:
1126
finish_android_power_test(self, test['name'])
1127
1128
def __run_test_cold(self, test, timeout):
1129
'''
1130
Run the Raptor test but restart the entire browser app between page-cycles.
1131
1132
Note: For page-load tests, playback will only be started once - at the beginning of all
1133
browser cycles, and then stopped after all cycles are finished. The proxy is set via prefs
1134
in the browser profile so those will need to be set again in each new profile/cycle.
1135
Note that instead of using the certutil tool each time to create a db and import the
1136
mitmproxy SSL cert (it's done in mozbase/mozproxy) we will simply copy the existing
1137
cert db from the first cycle's browser profile into the new clean profile; this way
1138
we don't have to re-create the cert db on each browser cycle.
1139
1140
Since we're running in cold-mode, before this point (in manifest.py) the
1141
'expected-browser-cycles' value was already set to the initial 'page-cycles' value;
1142
and the 'page-cycles' value was set to 1 as we want to perform one page-cycle per
1143
browser restart.
1144
1145
The 'browser-cycle' value is the current overall browser start iteration. The control
1146
server will receive the current 'browser-cycle' and the 'expected-browser-cycles' in
1147
each results set received; and will pass that on as part of the results so that the
1148
results processing will know results for multiple browser cycles are being received.
1149
1150
The default will be to run in warm mode; unless 'cold = true' is set in the test INI.
1151
'''
1152
LOG.info("test %s is running in cold mode; browser WILL be restarted between "
1153
"page cycles" % test['name'])
1154
1155
for test['browser_cycle'] in range(1, test['expected_browser_cycles'] + 1):
1156
1157
LOG.info("begin browser cycle %d of %d for test %s"
1158
% (test['browser_cycle'], test['expected_browser_cycles'], test['name']))
1159
1160
self.run_test_setup(test)
1161
1162
self.clear_app_data()
1163
self.set_debug_app_flag()
1164
1165
if test['browser_cycle'] == 1:
1166
if test.get('playback') is not None:
1167
self.start_playback(test)
1168
1169
# an ssl cert db has now been created in the profile; copy it out so we
1170
# can use the same cert db in future test cycles / browser restarts
1171
local_cert_db_dir = tempfile.mkdtemp()
1172
LOG.info("backing up browser ssl cert db that was created via certutil")
1173
self.copy_cert_db(self.config['local_profile_dir'], local_cert_db_dir)
1174
1175
if self.config['host'] not in ('localhost', '127.0.0.1'):
1176
self.delete_proxy_settings_from_profile()
1177
1178
else:
1179
# double-check to ensure app has been shutdown
1180
self.device.stop_application(self.config['binary'])
1181
1182
# initial browser profile was already created before run_test was called;
1183
# now additional browser cycles we want to create a new one each time
1184
self.build_browser_profile()
1185
1186
if test.get('playback') is not None:
1187
# get cert db from previous cycle profile and copy into new clean profile
1188
# this saves us from having to start playback again / recreate cert db etc.
1189
LOG.info("copying existing ssl cert db into new browser profile")
1190
self.copy_cert_db(local_cert_db_dir, self.config['local_profile_dir'])
1191
1192
self.run_test_setup(test)
1193
1194
if test.get('playback') is not None:
1195
self.turn_on_android_app_proxy()
1196
1197
self.copy_profile_to_device()
1198
self.log_android_device_temperature()
1199
1200
# write android app config.yml
1201
self.write_android_app_config()
1202
1203
# now start the browser/app under test
1204
self.launch_firefox_android_app(test['name'])
1205
1206
# set our control server flag to indicate we are running the browser/app
1207
self.control_server._finished = False
1208
1209
if self.config['cpu_test']:
1210
# start measuring CPU usage
1211
self.cpu_profiler = start_android_cpu_profiler(self)
1212
1213
self.wait_for_test_finish(test, timeout)
1214
1215
# in debug mode, and running locally, leave the browser running
1216
if self.debug_mode and self.config['run_local']:
1217
LOG.info("* debug-mode enabled - please shutdown the browser manually...")
1218
self.runner.wait(timeout=None)
1219
1220
# break test execution if a exception is present
1221
if len(self.results_handler.page_timeout_list) > 0:
1222
break
1223
1224
def __run_test_warm(self, test, timeout):
1225
LOG.info("test %s is running in warm mode; browser will NOT be restarted between "
1226
"page cycles" % test['name'])
1227
1228
self.run_test_setup(test)
1229
1230
if test.get('playback') is not None:
1231
self.start_playback(test)
1232
1233
if self.config['host'] not in ('localhost', '127.0.0.1'):
1234
self.delete_proxy_settings_from_profile()
1235
1236
if test.get('playback') is not None:
1237
self.turn_on_android_app_proxy()
1238
1239
self.clear_app_data()
1240
self.set_debug_app_flag()
1241
self.copy_profile_to_device()
1242
self.log_android_device_temperature()
1243
1244
# write android app config.yml
1245
self.write_android_app_config()
1246
1247
# now start the browser/app under test
1248
self.launch_firefox_android_app(test['name'])
1249
1250
# set our control server flag to indicate we are running the browser/app
1251
self.control_server._finished = False
1252
1253
if self.config['cpu_test']:
1254
# start measuring CPU usage
1255
self.cpu_profiler = start_android_cpu_profiler(self)
1256
1257
self.wait_for_test_finish(test, timeout)
1258
1259
# in debug mode, and running locally, leave the browser running
1260
if self.debug_mode and self.config['run_local']:
1261
LOG.info("* debug-mode enabled - please shutdown the browser manually...")
1262
self.runner.wait(timeout=None)
1263
1264
def check_for_crashes(self):
1265
super(RaptorAndroid, self).check_for_crashes()
1266
1267
if not self.app_launched:
1268
LOG.info("skipping check_for_crashes: application has not been launched")
1269
return
1270
self.app_launched = False
1271
1272
# Turn off verbose to prevent logcat from being inserted into the main log.
1273
verbose = self.device._verbose
1274
self.device._verbose = False
1275
logcat = self.device.get_logcat()
1276
self.device._verbose = verbose
1277
if logcat:
1278
if mozcrash.check_for_java_exception(logcat, "raptor"):
1279
return
1280
try:
1281
dump_dir = tempfile.mkdtemp()
1282
remote_dir = posixpath.join(self.remote_profile, 'minidumps')
1283
if not self.device.is_dir(remote_dir):
1284
LOG.error("No crash directory (%s) found on remote device" % remote_dir)
1285
return
1286
self.device.pull(remote_dir, dump_dir)
1287
mozcrash.log_crashes(LOG, dump_dir, self.config['symbols_path'])
1288
finally:
1289
try:
1290
shutil.rmtree(dump_dir)
1291
except Exception:
1292
LOG.warning("unable to remove directory: %s" % dump_dir)
1293
1294
def clean_up(self):
1295
LOG.info("removing test folder for raptor: %s" % self.remote_test_root)
1296
self.device.rm(self.remote_test_root, force=True, recursive=True)
1297
1298
super(RaptorAndroid, self).clean_up()
1299
1300
1301
def main(args=sys.argv[1:]):
1302
args = parse_args()
1303
commandline.setup_logging('raptor', args, {'tbpl': sys.stdout})
1304
1305
LOG.info("raptor-start")
1306
1307
if args.debug_mode:
1308
LOG.info("debug-mode enabled")
1309
1310
LOG.info("received command line arguments: %s" % str(args))
1311
1312
# if a test name specified on command line, and it exists, just run that one
1313
# otherwise run all available raptor tests that are found for this browser
1314
raptor_test_list = get_raptor_test_list(args, mozinfo.os)
1315
raptor_test_names = [raptor_test['name'] for raptor_test in raptor_test_list]
1316
1317
# ensure we have at least one valid test to run
1318
if len(raptor_test_list) == 0:
1319
LOG.critical("this test is not targeted for {}".format(args.app))
1320
sys.exit(1)
1321
1322
LOG.info("raptor tests scheduled to run:")
1323
for next_test in raptor_test_list:
1324
LOG.info(next_test['name'])
1325
1326
if not args.browsertime:
1327
if args.app == "firefox":
1328
raptor_class = RaptorDesktopFirefox
1329
elif args.app in CHROMIUM_DISTROS:
1330
raptor_class = RaptorDesktopChrome
1331
else:
1332
raptor_class = RaptorAndroid
1333
else:
1334
def raptor_class(*inner_args, **inner_kwargs):
1335
outer_kwargs = vars(args)
1336
# peel off arguments that are specific to browsertime
1337
for key in outer_kwargs.keys():
1338
if key.startswith('browsertime_'):
1339
value = outer_kwargs.pop(key)
1340
inner_kwargs[key] = value
1341
1342
return Browsertime(*inner_args, **inner_kwargs)
1343
1344
raptor = raptor_class(args.app,
1345
args.binary,
1346
run_local=args.run_local,
1347
noinstall=args.noinstall,
1348
obj_path=args.obj_path,
1349
gecko_profile=args.gecko_profile,
1350
gecko_profile_interval=args.gecko_profile_interval,
1351
gecko_profile_entries=args.gecko_profile_entries,
1352
symbols_path=args.symbols_path,
1353
host=args.host,
1354
power_test=args.power_test,
1355
cpu_test=args.cpu_test,
1356
memory_test=args.memory_test,
1357
is_release_build=args.is_release_build,
1358
debug_mode=args.debug_mode,
1359
post_startup_delay=args.post_startup_delay,
1360
activity=args.activity,
1361
intent=args.intent,
1362
interrupt_handler=SignalHandler(),
1363
enable_webrender=args.enable_webrender,
1364
)
1365
1366
success = raptor.run_tests(raptor_test_list, raptor_test_names)
1367
1368
if not success:
1369
# didn't get test results; test timed out or crashed, etc. we want job to fail
1370
LOG.critical("TEST-UNEXPECTED-FAIL: no raptor test results were found for %s" %
1371
', '.join(raptor_test_names))
1372
os.sys.exit(1)
1373
1374
# if we have results but one test page timed out (i.e. one tp6 test page didn't load
1375
# but others did) we still dumped PERFHERDER_DATA for the successfull pages but we
1376
# want the overall test job to marked as a failure
1377
pages_that_timed_out = raptor.get_page_timeout_list()
1378
if len(pages_that_timed_out) > 0:
1379
for _page in pages_that_timed_out:
1380
message = [("TEST-UNEXPECTED-FAIL", "test '%s'" % _page['test_name']),
1381
("timed out loading test page", _page['url'])]
1382
if raptor_test.get("type") == 'pageload':
1383
message.append(("pending metrics", _page['pending_metrics']))
1384
1385
LOG.critical(" ".join("%s: %s" % (subject, msg) for subject, msg in message))
1386
os.sys.exit(1)
1387
1388
# when running raptor locally with gecko profiling on, use the view-gecko-profile
1389
# tool to automatically load the latest gecko profile in profiler.firefox.com
1390
if args.gecko_profile and args.run_local:
1391
if os.environ.get('DISABLE_PROFILE_LAUNCH', '0') == '1':
1392
LOG.info("Not launching profiler.firefox.com because DISABLE_PROFILE_LAUNCH=1")
1393
else:
1394
view_gecko_profile(args.binary)
1395
1396
1397
if __name__ == "__main__":
1398
main()