Source code

Revision control

Other Tools

1
# This Source Code Form is subject to the terms of the Mozilla Public
2
# License, v. 2.0. If a copy of the MPL was not distributed with this
3
# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
4
5
from __future__ import absolute_import, print_function, unicode_literals
6
7
import argparse
8
import itertools
9
import json
10
import logging
11
import operator
12
import os
13
import re
14
import subprocess
15
import sys
16
import tempfile
17
18
import mozpack.path as mozpath
19
20
from mach.decorators import (
21
CommandArgument,
22
CommandArgumentGroup,
23
CommandProvider,
24
Command,
25
SettingsProvider,
26
SubCommand,
27
)
28
29
from mozbuild.base import (
30
BuildEnvironmentNotFoundException,
31
MachCommandBase,
32
MachCommandConditions as conditions,
33
MozbuildObject,
34
)
35
36
here = os.path.abspath(os.path.dirname(__file__))
37
38
EXCESSIVE_SWAP_MESSAGE = '''
39
===================
40
PERFORMANCE WARNING
41
42
Your machine experienced a lot of swap activity during the build. This is
43
possibly a sign that your machine doesn't have enough physical memory or
44
not enough available memory to perform the build. It's also possible some
45
other system activity during the build is to blame.
46
47
If you feel this message is not appropriate for your machine configuration,
48
please file a Firefox Build System :: General bug at
50
and tell us about your machine and build configuration so we can adjust the
51
warning heuristic.
52
===================
53
'''
54
55
56
class StoreDebugParamsAndWarnAction(argparse.Action):
57
def __call__(self, parser, namespace, values, option_string=None):
58
sys.stderr.write('The --debugparams argument is deprecated. Please ' +
59
'use --debugger-args instead.\n\n')
60
setattr(namespace, self.dest, values)
61
62
63
@CommandProvider
64
class Watch(MachCommandBase):
65
"""Interface to watch and re-build the tree."""
66
67
@Command('watch', category='post-build', description='Watch and re-build the tree.',
68
conditions=[conditions.is_firefox])
69
@CommandArgument('-v', '--verbose', action='store_true',
70
help='Verbose output for what commands the watcher is running.')
71
def watch(self, verbose=False):
72
"""Watch and re-build the source tree."""
73
74
if not conditions.is_artifact_build(self):
75
print('mach watch requires an artifact build. See '
77
return 1
78
79
if not self.substs.get('WATCHMAN', None):
80
print('mach watch requires watchman to be installed. See '
82
return 1
83
84
self._activate_virtualenv()
85
try:
86
self.virtualenv_manager.install_pip_package('pywatchman==1.3.0')
87
except Exception:
88
print('Could not install pywatchman from pip. See '
90
return 1
91
92
from mozbuild.faster_daemon import Daemon
93
daemon = Daemon(self.config_environment)
94
95
try:
96
return daemon.watch()
97
except KeyboardInterrupt:
98
# Suppress ugly stack trace when user hits Ctrl-C.
99
sys.exit(3)
100
101
102
@CommandProvider
103
class CargoProvider(MachCommandBase):
104
"""Invoke cargo in useful ways."""
105
106
@Command('cargo', category='build',
107
description='Invoke cargo in useful ways.')
108
def cargo(self):
109
self.parser.print_usage()
110
return 1
111
112
@SubCommand('cargo', 'check',
113
description='Run `cargo check` on a given crate. Defaults to gkrust.')
114
@CommandArgument('--all-crates', default=None, action='store_true',
115
help='Check all of the crates in the tree.')
116
@CommandArgument('crates', default=None, nargs='*', help='The crate name(s) to check.')
117
def check(self, all_crates=None, crates=None):
118
# XXX duplication with `mach vendor rust`
119
crates_and_roots = {
120
'gkrust': 'toolkit/library/rust',
121
'gkrust-gtest': 'toolkit/library/gtest/rust',
122
'js': 'js/rust',
123
'mozjs_sys': 'js/src',
124
'baldrdash': 'js/src/wasm/cranelift',
125
'geckodriver': 'testing/geckodriver',
126
}
127
128
if all_crates:
129
crates = crates_and_roots.keys()
130
elif crates is None or crates == []:
131
crates = ['gkrust']
132
133
for crate in crates:
134
root = crates_and_roots.get(crate, None)
135
if not root:
136
print('Cannot locate crate %s. Please check your spelling or '
137
'add the crate information to the list.' % crate)
138
return 1
139
140
check_targets = [
141
'force-cargo-library-check',
142
'force-cargo-host-library-check',
143
'force-cargo-program-check',
144
'force-cargo-host-program-check',
145
]
146
147
ret = self._run_make(srcdir=False, directory=root,
148
ensure_exit_code=0, silent=True,
149
print_directory=False, target=check_targets)
150
if ret != 0:
151
return ret
152
153
return 0
154
155
156
@CommandProvider
157
class Doctor(MachCommandBase):
158
"""Provide commands for diagnosing common build environment problems"""
159
@Command('doctor', category='devenv',
160
description='')
161
@CommandArgument('--fix', default=None, action='store_true',
162
help='Attempt to fix found problems.')
163
def doctor(self, fix=None):
164
self._activate_virtualenv()
165
from mozbuild.doctor import Doctor
166
doctor = Doctor(self.topsrcdir, self.topobjdir, fix)
167
return doctor.check_all()
168
169
170
@CommandProvider
171
class Clobber(MachCommandBase):
172
NO_AUTO_LOG = True
173
CLOBBER_CHOICES = ['objdir', 'python']
174
175
@Command('clobber', category='build',
176
description='Clobber the tree (delete the object directory).')
177
@CommandArgument('what', default=['objdir'], nargs='*',
178
help='Target to clobber, must be one of {{{}}} (default objdir).'.format(
179
', '.join(CLOBBER_CHOICES)))
180
@CommandArgument('--full', action='store_true',
181
help='Perform a full clobber')
182
def clobber(self, what, full=False):
183
"""Clean up the source and object directories.
184
185
Performing builds and running various commands generate various files.
186
187
Sometimes it is necessary to clean up these files in order to make
188
things work again. This command can be used to perform that cleanup.
189
190
By default, this command removes most files in the current object
191
directory (where build output is stored). Some files (like Visual
192
Studio project files) are not removed by default. If you would like
193
to remove the object directory in its entirety, run with `--full`.
194
195
The `python` target will clean up various generated Python files from
196
the source directory and will remove untracked files from well-known
197
directories containing Python packages. Run this to remove .pyc files,
198
compiled C extensions, etc. Note: all files not tracked or ignored by
199
version control in well-known Python package directories will be
200
deleted. Run the `status` command of your VCS to see if any untracked
201
files you haven't committed yet will be deleted.
202
"""
203
invalid = set(what) - set(self.CLOBBER_CHOICES)
204
if invalid:
205
print('Unknown clobber target(s): {}'.format(', '.join(invalid)))
206
return 1
207
208
ret = 0
209
if 'objdir' in what:
210
from mozbuild.controller.clobber import Clobberer
211
try:
212
Clobberer(self.topsrcdir, self.topobjdir, self.substs).remove_objdir(full)
213
except OSError as e:
214
if sys.platform.startswith('win'):
215
if isinstance(e, WindowsError) and e.winerror in (5, 32):
216
self.log(logging.ERROR, 'file_access_error', {'error': e},
217
"Could not clobber because a file was in use. If the "
218
"application is running, try closing it. {error}")
219
return 1
220
raise
221
222
if 'python' in what:
223
if conditions.is_hg(self):
224
cmd = ['hg', 'purge', '--all', '-I', 'glob:**.py[cdo]',
225
'-I', 'path:python/', '-I', 'path:third_party/python/']
226
elif conditions.is_git(self):
227
cmd = ['git', 'clean', '-f', '-x', '*.py[cdo]', 'python/',
228
'third_party/python/']
229
else:
230
# We don't know what is tracked/untracked if we don't have VCS.
231
# So we can't clean python/ and third_party/python/.
232
cmd = ['find', '.', '-type', 'f', '-name', '*.py[cdo]',
233
'-delete']
234
ret = subprocess.call(cmd, cwd=self.topsrcdir)
235
return ret
236
237
@property
238
def substs(self):
239
try:
240
return super(Clobber, self).substs
241
except BuildEnvironmentNotFoundException:
242
return {}
243
244
245
@CommandProvider
246
class Logs(MachCommandBase):
247
"""Provide commands to read mach logs."""
248
NO_AUTO_LOG = True
249
250
@Command('show-log', category='post-build',
251
description='Display mach logs')
252
@CommandArgument('log_file', nargs='?', type=argparse.FileType('rb'),
253
help='Filename to read log data from. Defaults to the log of the last '
254
'mach command.')
255
def show_log(self, log_file=None):
256
if not log_file:
257
path = self._get_state_filename('last_log.json')
258
log_file = open(path, 'rb')
259
260
if os.isatty(sys.stdout.fileno()):
261
env = dict(os.environ)
262
if 'LESS' not in env:
263
# Sensible default flags if none have been set in the user
264
# environment.
265
env[b'LESS'] = b'FRX'
266
less = subprocess.Popen(['less'], stdin=subprocess.PIPE, env=env)
267
# Various objects already have a reference to sys.stdout, so we
268
# can't just change it, we need to change the file descriptor under
269
# it to redirect to less's input.
270
# First keep a copy of the sys.stdout file descriptor.
271
output_fd = os.dup(sys.stdout.fileno())
272
os.dup2(less.stdin.fileno(), sys.stdout.fileno())
273
274
startTime = 0
275
for line in log_file:
276
created, action, params = json.loads(line)
277
if not startTime:
278
startTime = created
279
self.log_manager.terminal_handler.formatter.start_time = \
280
created
281
if 'line' in params:
282
record = logging.makeLogRecord({
283
'created': created,
284
'name': self._logger.name,
285
'levelno': logging.INFO,
286
'msg': '{line}',
287
'params': params,
288
'action': action,
289
})
290
self._logger.handle(record)
291
292
if self.log_manager.terminal:
293
# Close less's input so that it knows that we're done sending data.
294
less.stdin.close()
295
# Since the less's input file descriptor is now also the stdout
296
# file descriptor, we still actually have a non-closed system file
297
# descriptor for less's input. Replacing sys.stdout's file
298
# descriptor with what it was before we replaced it will properly
299
# close less's input.
300
os.dup2(output_fd, sys.stdout.fileno())
301
less.wait()
302
303
304
@CommandProvider
305
class Warnings(MachCommandBase):
306
"""Provide commands for inspecting warnings."""
307
308
@property
309
def database_path(self):
310
return self._get_state_filename('warnings.json')
311
312
@property
313
def database(self):
314
from mozbuild.compilation.warnings import WarningsDatabase
315
316
path = self.database_path
317
318
database = WarningsDatabase()
319
320
if os.path.exists(path):
321
database.load_from_file(path)
322
323
return database
324
325
@Command('warnings-summary', category='post-build',
326
description='Show a summary of compiler warnings.')
327
@CommandArgument('-C', '--directory', default=None,
328
help='Change to a subdirectory of the build directory first.')
329
@CommandArgument('report', default=None, nargs='?',
330
help='Warnings report to display. If not defined, show the most '
331
'recent report.')
332
def summary(self, directory=None, report=None):
333
database = self.database
334
335
if directory:
336
dirpath = self.join_ensure_dir(self.topsrcdir, directory)
337
if not dirpath:
338
return 1
339
else:
340
dirpath = None
341
342
type_counts = database.type_counts(dirpath)
343
sorted_counts = sorted(type_counts.iteritems(),
344
key=operator.itemgetter(1))
345
346
total = 0
347
for k, v in sorted_counts:
348
print('%d\t%s' % (v, k))
349
total += v
350
351
print('%d\tTotal' % total)
352
353
@Command('warnings-list', category='post-build',
354
description='Show a list of compiler warnings.')
355
@CommandArgument('-C', '--directory', default=None,
356
help='Change to a subdirectory of the build directory first.')
357
@CommandArgument('--flags', default=None, nargs='+',
358
help='Which warnings flags to match.')
359
@CommandArgument('report', default=None, nargs='?',
360
help='Warnings report to display. If not defined, show the most '
361
'recent report.')
362
def list(self, directory=None, flags=None, report=None):
363
database = self.database
364
365
by_name = sorted(database.warnings)
366
367
topsrcdir = mozpath.normpath(self.topsrcdir)
368
369
if directory:
370
directory = mozpath.normsep(directory)
371
dirpath = self.join_ensure_dir(topsrcdir, directory)
372
if not dirpath:
373
return 1
374
375
if flags:
376
# Flatten lists of flags.
377
flags = set(itertools.chain(*[flaglist.split(',') for flaglist in flags]))
378
379
for warning in by_name:
380
filename = mozpath.normsep(warning['filename'])
381
382
if filename.startswith(topsrcdir):
383
filename = filename[len(topsrcdir) + 1:]
384
385
if directory and not filename.startswith(directory):
386
continue
387
388
if flags and warning['flag'] not in flags:
389
continue
390
391
if warning['column'] is not None:
392
print('%s:%d:%d [%s] %s' % (
393
filename, warning['line'], warning['column'],
394
warning['flag'], warning['message']))
395
else:
396
print('%s:%d [%s] %s' % (filename, warning['line'],
397
warning['flag'], warning['message']))
398
399
def join_ensure_dir(self, dir1, dir2):
400
dir1 = mozpath.normpath(dir1)
401
dir2 = mozpath.normsep(dir2)
402
joined_path = mozpath.join(dir1, dir2)
403
if os.path.isdir(joined_path):
404
return joined_path
405
print('Specified directory not found.')
406
return None
407
408
409
@CommandProvider
410
class GTestCommands(MachCommandBase):
411
@Command('gtest', category='testing',
412
description='Run GTest unit tests (C++ tests).')
413
@CommandArgument('gtest_filter', default=b"*", nargs='?', metavar='gtest_filter',
414
help="test_filter is a ':'-separated list of wildcard patterns "
415
"(called the positive patterns), optionally followed by a '-' "
416
"and another ':'-separated pattern list (called the negative patterns).")
417
@CommandArgument('--jobs', '-j', default='1', nargs='?', metavar='jobs', type=int,
418
help='Run the tests in parallel using multiple processes.')
419
@CommandArgument('--tbpl-parser', '-t', action='store_true',
420
help='Output test results in a format that can be parsed by TBPL.')
421
@CommandArgument('--shuffle', '-s', action='store_true',
422
help='Randomize the execution order of tests.')
423
@CommandArgument('--enable-webrender', action='store_true',
424
default=False, dest='enable_webrender',
425
help='Enable the WebRender compositor in Gecko.')
426
@CommandArgumentGroup('Android')
427
@CommandArgument('--package',
428
default='org.mozilla.geckoview.test',
429
group='Android',
430
help='Package name of test app.')
431
@CommandArgument('--adbpath',
432
dest='adb_path',
433
group='Android',
434
help='Path to adb binary.')
435
@CommandArgument('--deviceSerial',
436
dest='device_serial',
437
group='Android',
438
help="adb serial number of remote device. "
439
"Required when more than one device is connected to the host. "
440
"Use 'adb devices' to see connected devices.")
441
@CommandArgument('--remoteTestRoot',
442
dest='remote_test_root',
443
group='Android',
444
help='Remote directory to use as test root '
445
'(eg. /mnt/sdcard/tests or /data/local/tests).')
446
@CommandArgument('--libxul',
447
dest='libxul_path',
448
group='Android',
449
help='Path to gtest libxul.so.')
450
@CommandArgument('--no-install', action='store_true',
451
default=False,
452
group='Android',
453
help='Skip the installation of the APK.')
454
@CommandArgumentGroup('debugging')
455
@CommandArgument('--debug', action='store_true', group='debugging',
456
help='Enable the debugger. Not specifying a --debugger option will result in '
457
'the default debugger being used.')
458
@CommandArgument('--debugger', default=None, type=str, group='debugging',
459
help='Name of debugger to use.')
460
@CommandArgument('--debugger-args', default=None, metavar='params', type=str,
461
group='debugging',
462
help='Command-line arguments to pass to the debugger itself; '
463
'split as the Bourne shell would.')
464
def gtest(self, shuffle, jobs, gtest_filter, tbpl_parser, enable_webrender,
465
package, adb_path, device_serial, remote_test_root, libxul_path, no_install,
466
debug, debugger, debugger_args):
467
468
# We lazy build gtest because it's slow to link
469
try:
470
config = self.config_environment
471
except Exception:
472
print("Please run |./mach build| before |./mach gtest|.")
473
return 1
474
475
active_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
476
if 'Tup' in active_backend:
477
gtest_build_target = mozpath.join(self.topobjdir, '<gtest>')
478
else:
479
gtest_build_target = 'recurse_gtest'
480
481
res = self._mach_context.commands.dispatch('build', self._mach_context,
482
what=[gtest_build_target])
483
if res:
484
print("Could not build xul-gtest")
485
return res
486
487
if self.substs.get('MOZ_WIDGET_TOOLKIT') == 'cocoa':
488
self._run_make(directory='browser/app', target='repackage',
489
ensure_exit_code=True)
490
491
cwd = os.path.join(self.topobjdir, '_tests', 'gtest')
492
493
if not os.path.isdir(cwd):
494
os.makedirs(cwd)
495
496
if conditions.is_android(self):
497
if jobs != 1:
498
print("--jobs is not supported on Android and will be ignored")
499
if debug or debugger or debugger_args:
500
print("--debug options are not supported on Android and will be ignored")
501
return self.android_gtest(cwd, shuffle, gtest_filter,
502
package, adb_path, device_serial,
503
remote_test_root, libxul_path,
504
enable_webrender, not no_install)
505
506
if package or adb_path or device_serial or remote_test_root or libxul_path or no_install:
507
print("One or more Android-only options will be ignored")
508
509
app_path = self.get_binary_path('app')
510
args = [app_path, '-unittest', '--gtest_death_test_style=threadsafe']
511
512
if sys.platform.startswith('win') and \
513
'MOZ_LAUNCHER_PROCESS' in self.defines:
514
args.append('--wait-for-browser')
515
516
if debug or debugger or debugger_args:
517
args = self.prepend_debugger_args(args, debugger, debugger_args)
518
519
# Use GTest environment variable to control test execution
520
# For details see:
522
gtest_env = {b'GTEST_FILTER': gtest_filter}
523
524
# Note: we must normalize the path here so that gtest on Windows sees
525
# a MOZ_GMP_PATH which has only Windows dir seperators, because
526
# nsIFile cannot open the paths with non-Windows dir seperators.
527
xre_path = os.path.join(os.path.normpath(self.topobjdir), "dist", "bin")
528
gtest_env["MOZ_XRE_DIR"] = xre_path
529
gtest_env["MOZ_GMP_PATH"] = os.pathsep.join(
530
os.path.join(xre_path, p, "1.0")
531
for p in ('gmp-fake', 'gmp-fakeopenh264')
532
)
533
534
gtest_env[b"MOZ_RUN_GTEST"] = b"True"
535
536
if shuffle:
537
gtest_env[b"GTEST_SHUFFLE"] = b"True"
538
539
if tbpl_parser:
540
gtest_env[b"MOZ_TBPL_PARSER"] = b"True"
541
542
if enable_webrender:
543
gtest_env[b"MOZ_WEBRENDER"] = b"1"
544
gtest_env[b"MOZ_ACCELERATED"] = b"1"
545
else:
546
gtest_env[b"MOZ_WEBRENDER"] = b"0"
547
548
if jobs == 1:
549
return self.run_process(args=args,
550
append_env=gtest_env,
551
cwd=cwd,
552
ensure_exit_code=False,
553
pass_thru=True)
554
555
from mozprocess import ProcessHandlerMixin
556
import functools
557
558
def handle_line(job_id, line):
559
# Prepend the jobId
560
line = '[%d] %s' % (job_id + 1, line.strip())
561
self.log(logging.INFO, "GTest", {'line': line}, '{line}')
562
563
gtest_env["GTEST_TOTAL_SHARDS"] = str(jobs)
564
processes = {}
565
for i in range(0, jobs):
566
gtest_env["GTEST_SHARD_INDEX"] = str(i)
567
processes[i] = ProcessHandlerMixin([app_path, "-unittest"],
568
cwd=cwd,
569
env=gtest_env,
570
processOutputLine=[
571
functools.partial(handle_line, i)],
572
universal_newlines=True)
573
processes[i].run()
574
575
exit_code = 0
576
for process in processes.values():
577
status = process.wait()
578
if status:
579
exit_code = status
580
581
# Clamp error code to 255 to prevent overflowing multiple of
582
# 256 into 0
583
if exit_code > 255:
584
exit_code = 255
585
586
return exit_code
587
588
def android_gtest(self, test_dir, shuffle, gtest_filter,
589
package, adb_path, device_serial, remote_test_root, libxul_path,
590
enable_webrender, install):
591
# setup logging for mozrunner
592
from mozlog.commandline import setup_logging
593
format_args = {'level': self._mach_context.settings['test']['level']}
594
default_format = self._mach_context.settings['test']['format']
595
setup_logging('mach-gtest', {}, {default_format: sys.stdout}, format_args)
596
597
# ensure that a device is available and test app is installed
598
from mozrunner.devices.android_device import (verify_android_device, get_adb_path)
599
verify_android_device(self, install=install, app=package, device_serial=device_serial)
600
601
if not adb_path:
602
adb_path = get_adb_path(self)
603
if not libxul_path:
604
libxul_path = os.path.join(self.topobjdir, "dist", "bin", "gtest", "libxul.so")
605
606
# run gtest via remotegtests.py
607
import imp
608
path = os.path.join('testing', 'gtest', 'remotegtests.py')
609
with open(path, 'r') as fh:
610
imp.load_module('remotegtests', fh, path,
611
('.py', 'r', imp.PY_SOURCE))
612
import remotegtests
613
tester = remotegtests.RemoteGTests()
614
tester.run_gtest(test_dir, shuffle, gtest_filter, package, adb_path, device_serial,
615
remote_test_root, libxul_path, None, enable_webrender)
616
tester.cleanup()
617
618
return 0
619
620
def prepend_debugger_args(self, args, debugger, debugger_args):
621
'''
622
Given an array with program arguments, prepend arguments to run it under a
623
debugger.
624
625
:param args: The executable and arguments used to run the process normally.
626
:param debugger: The debugger to use, or empty to use the default debugger.
627
:param debugger_args: Any additional parameters to pass to the debugger.
628
'''
629
630
import mozdebug
631
632
if not debugger:
633
# No debugger name was provided. Look for the default ones on
634
# current OS.
635
debugger = mozdebug.get_default_debugger_name(mozdebug.DebuggerSearch.KeepLooking)
636
637
if debugger:
638
debuggerInfo = mozdebug.get_debugger_info(debugger, debugger_args)
639
if not debuggerInfo:
640
print("Could not find a suitable debugger in your PATH.")
641
return 1
642
643
# Parameters come from the CLI. We need to convert them before
644
# their use.
645
if debugger_args:
646
from mozbuild import shellutil
647
try:
648
debugger_args = shellutil.split(debugger_args)
649
except shellutil.MetaCharacterException as e:
650
print("The --debugger_args you passed require a real shell to parse them.")
651
print("(We can't handle the %r character.)" % e.char)
652
return 1
653
654
# Prepend the debugger args.
655
args = [debuggerInfo.path] + debuggerInfo.args + args
656
return args
657
658
659
@CommandProvider
660
class Package(MachCommandBase):
661
"""Package the built product for distribution."""
662
663
@Command('package', category='post-build',
664
description='Package the built product for distribution as an APK, DMG, etc.')
665
@CommandArgument('-v', '--verbose', action='store_true',
666
help='Verbose output for what commands the packaging process is running.')
667
def package(self, verbose=False):
668
ret = self._run_make(directory=".", target='package',
669
silent=not verbose, ensure_exit_code=False)
670
if ret == 0:
671
self.notify('Packaging complete')
672
return ret
673
674
675
def _get_android_install_parser():
676
parser = argparse.ArgumentParser()
677
parser.add_argument('--app', default='org.mozilla.geckoview_example',
678
help='Android package to install '
679
'(default: org.mozilla.geckoview_example)')
680
parser.add_argument('--verbose', '-v', action='store_true',
681
help='Print verbose output when installing.')
682
return parser
683
684
685
def setup_install_parser():
686
build = MozbuildObject.from_environment(cwd=here)
687
if conditions.is_android(build):
688
return _get_android_install_parser()
689
return argparse.ArgumentParser()
690
691
692
@CommandProvider
693
class Install(MachCommandBase):
694
"""Install a package."""
695
696
@Command('install', category='post-build',
697
conditions=[conditions.has_build],
698
parser=setup_install_parser,
699
description='Install the package on the machine (or device in the case of Android).')
700
def install(self, **kwargs):
701
if conditions.is_android(self):
702
from mozrunner.devices.android_device import verify_android_device
703
ret = verify_android_device(self, install=True, **kwargs) == 0
704
else:
705
ret = self._run_make(directory=".", target='install', ensure_exit_code=False)
706
707
if ret == 0:
708
self.notify('Install complete')
709
return ret
710
711
712
@SettingsProvider
713
class RunSettings():
714
config_settings = [
715
('runprefs.*', 'string', """
716
Pass a pref into Firefox when using `mach run`, of the form `foo.bar=value`.
717
Prefs will automatically be cast into the appropriate type. Integers can be
718
single quoted to force them to be strings.
719
""".strip()),
720
]
721
722
723
def _get_android_run_parser():
724
parser = argparse.ArgumentParser()
725
parser.add_argument('--app', default='org.mozilla.geckoview_example',
726
help='Android package to run '
727
'(default: org.mozilla.geckoview_example)')
728
parser.add_argument('--intent', default='android.intent.action.VIEW',
729
help='Android intent action to launch with '
730
'(default: android.intent.action.VIEW)')
731
parser.add_argument('--setenv', dest='env', action='append', default=[],
732
help='Set target environment variable, like FOO=BAR')
733
parser.add_argument('--profile', '-P', default=None,
734
help='Path to Gecko profile, like /path/to/host/profile '
735
'or /path/to/target/profile')
736
parser.add_argument('--url', default=None,
737
help='URL to open')
738
parser.add_argument('--no-install', action='store_true', default=False,
739
help='Do not try to install application on device before running '
740
'(default: False)')
741
parser.add_argument('--no-wait', action='store_true', default=False,
742
help='Do not wait for application to start before returning '
743
'(default: False)')
744
parser.add_argument('--fail-if-running', action='store_true', default=False,
745
help='Fail if application is already running (default: False)')
746
parser.add_argument('--restart', action='store_true', default=False,
747
help='Stop the application if it is already running (default: False)')
748
return parser
749
750
751
def _get_desktop_run_parser():
752
parser = argparse.ArgumentParser()
753
group = parser.add_argument_group('the compiled program')
754
group.add_argument('params', nargs='...', default=[],
755
help='Command-line arguments to be passed through to the program. Not '
756
'specifying a --profile or -P option will result in a temporary profile '
757
'being used.')
758
group.add_argument('--remote', '-r', action='store_true',
759
help='Do not pass the --no-remote argument by default.')
760
group.add_argument('--background', '-b', action='store_true',
761
help='Do not pass the --foreground argument by default on Mac.')
762
group.add_argument('--noprofile', '-n', action='store_true',
763
help='Do not pass the --profile argument by default.')
764
group.add_argument('--disable-e10s', action='store_true',
765
help='Run the program with electrolysis disabled.')
766
group.add_argument('--enable-crash-reporter', action='store_true',
767
help='Run the program with the crash reporter enabled.')
768
group.add_argument('--setpref', action='append', default=[],
769
help='Set the specified pref before starting the program. Can be set '
770
'multiple times. Prefs can also be set in ~/.mozbuild/machrc in the '
771
'[runprefs] section - see `./mach settings` for more information.')
772
group.add_argument('--temp-profile', action='store_true',
773
help='Run the program using a new temporary profile created inside '
774
'the objdir.')
775
group.add_argument('--macos-open', action='store_true',
776
help="On macOS, run the program using the open(1) command. Per open(1), "
777
"the browser is launched \"just as if you had double-clicked the file's "
778
"icon\". The browser can not be launched under a debugger with this "
779
"option.")
780
781
group = parser.add_argument_group('debugging')
782
group.add_argument('--debug', action='store_true',
783
help='Enable the debugger. Not specifying a --debugger option will result '
784
'in the default debugger being used.')
785
group.add_argument('--debugger', default=None, type=str,
786
help='Name of debugger to use.')
787
group.add_argument('--debugger-args', default=None, metavar='params', type=str,
788
help='Command-line arguments to pass to the debugger itself; '
789
'split as the Bourne shell would.')
790
group.add_argument('--debugparams', action=StoreDebugParamsAndWarnAction,
791
default=None, type=str, dest='debugger_args',
792
help=argparse.SUPPRESS)
793
794
group = parser.add_argument_group('DMD')
795
group.add_argument('--dmd', action='store_true',
796
help='Enable DMD. The following arguments have no effect without this.')
797
group.add_argument('--mode', choices=['live', 'dark-matter', 'cumulative', 'scan'],
798
help='Profiling mode. The default is \'dark-matter\'.')
799
group.add_argument('--stacks', choices=['partial', 'full'],
800
help='Allocation stack trace coverage. The default is \'partial\'.')
801
group.add_argument('--show-dump-stats', action='store_true',
802
help='Show stats when doing dumps.')
803
804
return parser
805
806
807
def setup_run_parser():
808
build = MozbuildObject.from_environment(cwd=here)
809
if conditions.is_android(build):
810
return _get_android_run_parser()
811
return _get_desktop_run_parser()
812
813
814
@CommandProvider
815
class RunProgram(MachCommandBase):
816
"""Run the compiled program."""
817
818
@Command('run', category='post-build',
819
conditions=[conditions.has_build],
820
parser=setup_run_parser,
821
description='Run the compiled program, possibly under a debugger or DMD.')
822
def run(self, **kwargs):
823
if conditions.is_android(self):
824
return self._run_android(**kwargs)
825
return self._run_desktop(**kwargs)
826
827
def _run_android(self, app='org.mozilla.geckoview_example', intent=None, env=[], profile=None,
828
url=None, no_install=None, no_wait=None, fail_if_running=None, restart=None):
829
from mozrunner.devices.android_device import verify_android_device, _get_device
830
from six.moves import shlex_quote
831
832
if app == 'org.mozilla.geckoview_example':
833
activity_name = 'org.mozilla.geckoview_example.GeckoViewActivity'
834
elif app == 'org.mozilla.geckoview.test':
835
activity_name = 'org.mozilla.geckoview.test.TestRunnerActivity'
836
elif 'fennec' in app or 'firefox' in app:
837
activity_name = 'org.mozilla.gecko.BrowserApp'
838
else:
839
raise RuntimeError('Application not recognized: {}'.format(app))
840
841
# `verify_android_device` respects `DEVICE_SERIAL` if it is set and sets it otherwise.
842
verify_android_device(self, app=app, install=not no_install)
843
device_serial = os.environ.get('DEVICE_SERIAL')
844
if not device_serial:
845
print('No ADB devices connected.')
846
return 1
847
848
device = _get_device(self.substs, device_serial=device_serial)
849
850
args = []
851
if profile:
852
if os.path.isdir(profile):
853
host_profile = profile
854
# Always /data/local/tmp, rather than `device.test_root`, because GeckoView only
855
# takes its configuration file from /data/local/tmp, and we want to follow suit.
856
target_profile = '/data/local/tmp/{}-profile'.format(app)
857
device.rm(target_profile, recursive=True, force=True)
858
device.push(host_profile, target_profile)
859
self.log(logging.INFO, "run",
860
{'host_profile': host_profile, 'target_profile': target_profile},
861
'Pushed profile from host "{host_profile}" to target "{target_profile}"')
862
else:
863
target_profile = profile
864
self.log(logging.INFO, "run",
865
{'target_profile': target_profile},
866
'Using profile from target "{target_profile}"')
867
868
args = ['--profile', shlex_quote(target_profile)]
869
870
extras = {}
871
for i, e in enumerate(env):
872
extras['env{}'.format(i)] = e
873
if args:
874
extras['args'] = " ".join(args)
875
extras['use_multiprocess'] = True # Only GVE and TRA process this extra.
876
877
if env or args:
878
restart = True
879
880
if restart:
881
fail_if_running = False
882
self.log(logging.INFO, "run",
883
{'app': app},
884
'Stopping {app} to ensure clean restart.')
885
device.stop_application(app)
886
887
# We'd prefer to log the actual `am start ...` command, but it's not trivial to wire the
888
# device's logger to mach's logger.
889
self.log(logging.INFO, "run",
890
{'app': app, 'activity_name': activity_name},
891
'Starting {app}/{activity_name}.')
892
893
device.launch_application(
894
app_name=app,
895
activity_name=activity_name,
896
intent=intent,
897
extras=extras,
898
url=url,
899
wait=not no_wait,
900
fail_if_running=fail_if_running)
901
902
return 0
903
904
def _run_desktop(self, params, remote, background, noprofile, disable_e10s,
905
enable_crash_reporter, setpref, temp_profile, macos_open, debug, debugger,
906
debugger_args, dmd, mode, stacks, show_dump_stats):
907
from mozprofile import Profile, Preferences
908
909
try:
910
binpath = self.get_binary_path('app')
911
except Exception as e:
912
print("It looks like your program isn't built.",
913
"You can run |mach build| to build it.")
914
print(e)
915
return 1
916
917
args = []
918
if macos_open:
919
if debug:
920
print("The browser can not be launched in the debugger "
921
"when using the macOS open command.")
922
return 1
923
try:
924
m = re.search(r'^.+\.app', binpath)
925
apppath = m.group(0)
926
args = ['open', apppath, '--args']
927
except Exception as e:
928
print("Couldn't get the .app path from the binary path. "
929
"The macOS open option can only be used on macOS")
930
print(e)
931
return 1
932
else:
933
args = [binpath]
934
935
if params:
936
args.extend(params)
937
938
if not remote:
939
args.append('-no-remote')
940
941
if not background and sys.platform == 'darwin':
942
args.append('-foreground')
943
944
if sys.platform.startswith('win') and \
945
'MOZ_LAUNCHER_PROCESS' in self.defines:
946
args.append('-wait-for-browser')
947
948
no_profile_option_given = \
949
all(p not in params for p in ['-profile', '--profile', '-P'])
950
if no_profile_option_given and not noprofile:
951
prefs = {
952
'browser.aboutConfig.showWarning': False,
953
'browser.shell.checkDefaultBrowser': False,
954
'general.warnOnAboutConfig': False,
955
}
956
prefs.update(self._mach_context.settings.runprefs)
957
prefs.update([p.split('=', 1) for p in setpref])
958
for pref in prefs:
959
prefs[pref] = Preferences.cast(prefs[pref])
960
961
tmpdir = os.path.join(self.topobjdir, 'tmp')
962
if not os.path.exists(tmpdir):
963
os.makedirs(tmpdir)
964
965
if (temp_profile):
966
path = tempfile.mkdtemp(dir=tmpdir, prefix='profile-')
967
else:
968
path = os.path.join(tmpdir, 'profile-default')
969
970
profile = Profile(path, preferences=prefs)
971
args.append('-profile')
972
args.append(profile.profile)
973
974
if not no_profile_option_given and setpref:
975
print("setpref is only supported if a profile is not specified")
976
return 1
977
978
if not no_profile_option_given:
979
# The profile name may be non-ascii, but come from the
980
# commandline as str, so convert here with a better guess at
981
# an encoding than the default.
982
encoding = (sys.getfilesystemencoding() or
983
sys.getdefaultencoding())
984
args = [unicode(a, encoding) if not isinstance(a, unicode) else a
985
for a in args]
986
987
extra_env = {
988
'MOZ_DEVELOPER_REPO_DIR': self.topsrcdir,
989
'MOZ_DEVELOPER_OBJ_DIR': self.topobjdir,
990
'RUST_BACKTRACE': 'full',
991
}
992
993
if not enable_crash_reporter:
994
extra_env['MOZ_CRASHREPORTER_DISABLE'] = '1'
995
else:
996
extra_env['MOZ_CRASHREPORTER'] = '1'
997
998
if disable_e10s:
999
extra_env['MOZ_FORCE_DISABLE_E10S'] = '1'
1000
1001
if debug or debugger or debugger_args:
1002
if 'INSIDE_EMACS' in os.environ:
1003
self.log_manager.terminal_handler.setLevel(logging.WARNING)
1004
1005
import mozdebug
1006
if not debugger:
1007
# No debugger name was provided. Look for the default ones on
1008
# current OS.
1009
debugger = mozdebug.get_default_debugger_name(mozdebug.DebuggerSearch.KeepLooking)
1010
1011
if debugger:
1012
self.debuggerInfo = mozdebug.get_debugger_info(debugger, debugger_args)
1013
1014
if not debugger or not self.debuggerInfo:
1015
print("Could not find a suitable debugger in your PATH.")
1016
return 1
1017
1018
# Parameters come from the CLI. We need to convert them before
1019
# their use.
1020
if debugger_args:
1021
from mozbuild import shellutil
1022
try:
1023
debugger_args = shellutil.split(debugger_args)
1024
except shellutil.MetaCharacterException as e:
1025
print("The --debugger-args you passed require a real shell to parse them.")
1026
print("(We can't handle the %r character.)" % e.char)
1027
return 1
1028
1029
# Prepend the debugger args.
1030
args = [self.debuggerInfo.path] + self.debuggerInfo.args + args
1031
1032
if dmd:
1033
dmd_params = []
1034
1035
if mode:
1036
dmd_params.append('--mode=' + mode)
1037
if stacks:
1038
dmd_params.append('--stacks=' + stacks)
1039
if show_dump_stats:
1040
dmd_params.append('--show-dump-stats=yes')
1041
1042
if dmd_params:
1043
extra_env['DMD'] = ' '.join(dmd_params)
1044
else:
1045
extra_env['DMD'] = '1'
1046
1047
return self.run_process(args=args, ensure_exit_code=False,
1048
pass_thru=True, append_env=extra_env)
1049
1050
1051
@CommandProvider
1052
class Buildsymbols(MachCommandBase):
1053
"""Produce a package of debug symbols suitable for use with Breakpad."""
1054
1055
@Command('buildsymbols', category='post-build',
1056
description='Produce a package of Breakpad-format symbols.')
1057
def buildsymbols(self):
1058
return self._run_make(directory=".", target='buildsymbols', ensure_exit_code=False)
1059
1060
1061
@CommandProvider
1062
class Makefiles(MachCommandBase):
1063
@Command('empty-makefiles', category='build-dev',
1064
description='Find empty Makefile.in in the tree.')
1065
def empty(self):
1066
import pymake.parser
1067
import pymake.parserdata
1068
1069
IGNORE_VARIABLES = {
1070
'DEPTH': ('@DEPTH@',),
1071
'topsrcdir': ('@top_srcdir@',),
1072
'srcdir': ('@srcdir@',),
1073
'relativesrcdir': ('@relativesrcdir@',),
1074
'VPATH': ('@srcdir@',),
1075
}
1076
1077
IGNORE_INCLUDES = [
1078
'include $(DEPTH)/config/autoconf.mk',
1079
'include $(topsrcdir)/config/config.mk',
1080
'include $(topsrcdir)/config/rules.mk',
1081
]
1082
1083
def is_statement_relevant(s):
1084
if isinstance(s, pymake.parserdata.SetVariable):
1085
exp = s.vnameexp
1086
if not exp.is_static_string:
1087
return True
1088
1089
if exp.s not in IGNORE_VARIABLES:
1090
return True
1091
1092
return s.value not in IGNORE_VARIABLES[exp.s]
1093
1094
if isinstance(s, pymake.parserdata.Include):
1095
if s.to_source() in IGNORE_INCLUDES:
1096
return False
1097
1098
return True
1099
1100
for path in self._makefile_ins():
1101
relpath = os.path.relpath(path, self.topsrcdir)
1102
try:
1103
statements = [s for s in pymake.parser.parsefile(path)
1104
if is_statement_relevant(s)]
1105
1106
if not statements:
1107
print(relpath)
1108
except pymake.parser.SyntaxError:
1109
print('Warning: Could not parse %s' % relpath, file=sys.stderr)
1110
1111
def _makefile_ins(self):
1112
for root, dirs, files in os.walk(self.topsrcdir):
1113
for f in files:
1114
if f == 'Makefile.in':
1115
yield os.path.join(root, f)
1116
1117
1118
@CommandProvider
1119
class MachDebug(MachCommandBase):
1120
@Command('environment', category='build-dev',
1121
description='Show info about the mach and build environment.')
1122
@CommandArgument('--format', default='pretty',
1123
choices=['pretty', 'configure', 'json'],
1124
help='Print data in the given format.')
1125
@CommandArgument('--output', '-o', type=str,
1126
help='Output to the given file.')
1127
@CommandArgument('--verbose', '-v', action='store_true',
1128
help='Print verbose output.')
1129
def environment(self, format, output=None, verbose=False):
1130
func = getattr(self, '_environment_%s' % format.replace('.', '_'))
1131
1132
if output:
1133
# We want to preserve mtimes if the output file already exists
1134
# and the content hasn't changed.
1135
from mozbuild.util import FileAvoidWrite
1136
with FileAvoidWrite(output) as out:
1137
return func(out, verbose)
1138
return func(sys.stdout, verbose)
1139
1140
def _environment_pretty(self, out, verbose):
1141
state_dir = self._mach_context.state_dir
1142
import platform
1143
print('platform:\n\t%s' % platform.platform(), file=out)
1144
print('python version:\n\t%s' % sys.version, file=out)
1145
print('python prefix:\n\t%s' % sys.prefix, file=out)
1146
print('mach cwd:\n\t%s' % self._mach_context.cwd, file=out)
1147
print('os cwd:\n\t%s' % os.getcwd(), file=out)
1148
print('mach directory:\n\t%s' % self._mach_context.topdir, file=out)
1149
print('state directory:\n\t%s' % state_dir, file=out)
1150
1151
print('object directory:\n\t%s' % self.topobjdir, file=out)
1152
1153
if self.mozconfig['path']:
1154
print('mozconfig path:\n\t%s' % self.mozconfig['path'], file=out)
1155
if self.mozconfig['configure_args']:
1156
print('mozconfig configure args:', file=out)
1157
for arg in self.mozconfig['configure_args']:
1158
print('\t%s' % arg, file=out)
1159
1160
if self.mozconfig['make_extra']:
1161
print('mozconfig extra make args:', file=out)
1162
for arg in self.mozconfig['make_extra']:
1163
print('\t%s' % arg, file=out)
1164
1165
if self.mozconfig['make_flags']:
1166
print('mozconfig make flags:', file=out)
1167
for arg in self.mozconfig['make_flags']:
1168
print('\t%s' % arg, file=out)
1169
1170
config = None
1171
1172
try:
1173
config = self.config_environment
1174
1175
except Exception:
1176
pass
1177
1178
if config:
1179
print('config topsrcdir:\n\t%s' % config.topsrcdir, file=out)
1180
print('config topobjdir:\n\t%s' % config.topobjdir, file=out)
1181
1182
if verbose:
1183
print('config substitutions:', file=out)
1184
for k in sorted(config.substs):
1185
print('\t%s: %s' % (k, config.substs[k]), file=out)
1186
1187
print('config defines:', file=out)
1188
for k in sorted(config.defines):
1189
print('\t%s' % k, file=out)
1190
1191
def _environment_json(self, out, verbose):
1192
import json
1193
1194
class EnvironmentEncoder(json.JSONEncoder):
1195
def default(self, obj):
1196
if isinstance(obj, MozbuildObject):
1197
result = {
1198
'topsrcdir': obj.topsrcdir,
1199
'topobjdir': obj.topobjdir,
1200
'mozconfig': obj.mozconfig,
1201
}
1202
if verbose:
1203
result['substs'] = obj.substs
1204
result['defines'] = obj.defines
1205
return result
1206
elif isinstance(obj, set):
1207
return list(obj)
1208
return json.JSONEncoder.default(self, obj)
1209
json.dump(self, cls=EnvironmentEncoder, sort_keys=True, fp=out)
1210
1211
1212
@CommandProvider
1213
class Vendor(MachCommandBase):
1214
"""Vendor third-party dependencies into the source repository."""
1215
1216
@Command('vendor', category='misc',
1217
description='Vendor third-party dependencies into the source repository.')
1218
def vendor(self):
1219
self.parser.print_usage()
1220
sys.exit(1)
1221
1222
@SubCommand('vendor', 'rust',
1223
description='Vendor rust crates from crates.io into third_party/rust')
1224
@CommandArgument('--ignore-modified', action='store_true',
1225
help='Ignore modified files in current checkout',
1226
default=False)
1227
@CommandArgument('--build-peers-said-large-imports-were-ok', action='store_true',
1228
help='Permit overly-large files to be added to the repository',
1229
default=False)
1230
def vendor_rust(self, **kwargs):
1231
from mozbuild.vendor_rust import VendorRust
1232
vendor_command = self._spawn(VendorRust)
1233
vendor_command.vendor(**kwargs)
1234
1235
@SubCommand('vendor', 'aom',
1236
description='Vendor av1 video codec reference implementation into the '
1237
'source repository.')
1238
@CommandArgument('-r', '--revision',
1239
help='Repository tag or commit to update to.')
1240
@CommandArgument('--repo',
1241
help='Repository url to pull a snapshot from. '
1242
'Supports github and googlesource.')
1243
@CommandArgument('--ignore-modified', action='store_true',
1244
help='Ignore modified files in current checkout',
1245
default=False)
1246
def vendor_aom(self, **kwargs):
1247
from mozbuild.vendor_aom import VendorAOM
1248
vendor_command = self._spawn(VendorAOM)
1249
vendor_command.vendor(**kwargs)
1250
1251
@SubCommand('vendor', 'dav1d',
1252
description='Vendor dav1d implementation of AV1 into the source repository.')
1253
@CommandArgument('-r', '--revision',
1254
help='Repository tag or commit to update to.')
1255
@CommandArgument('--repo',
1256
help='Repository url to pull a snapshot from. Supports gitlab.')
1257
@CommandArgument('--ignore-modified', action='store_true',
1258
help='Ignore modified files in current checkout',
1259
default=False)
1260
def vendor_dav1d(self, **kwargs):
1261
from mozbuild.vendor_dav1d import VendorDav1d
1262
vendor_command = self._spawn(VendorDav1d)
1263
vendor_command.vendor(**kwargs)
1264
1265
@SubCommand('vendor', 'python',
1266
description='Vendor Python packages from pypi.org into third_party/python')
1267
@CommandArgument('--with-windows-wheel', action='store_true',
1268
help='Vendor a wheel for Windows along with the source package',
1269
default=False)
1270
@CommandArgument('packages', default=None, nargs='*',
1271
help='Packages to vendor. If omitted, packages and their dependencies '
1272
'defined in Pipfile.lock will be vendored. If Pipfile has been modified, '
1273
'then Pipfile.lock will be regenerated. Note that transient dependencies '
1274
'may be updated when running this command.')
1275
def vendor_python(self, **kwargs):
1276
from mozbuild.vendor_python import VendorPython
1277
vendor_command = self._spawn(VendorPython)
1278
vendor_command.vendor(**kwargs)
1279
1280
@SubCommand('vendor', 'manifest',
1281
description='Vendor externally hosted repositories into this '
1282
'repository.')
1283
@CommandArgument('files', nargs='+',
1284
help='Manifest files to work on')
1285
@CommandArgumentGroup('verify')
1286
@CommandArgument('--verify', '-v', action='store_true', group='verify',
1287
required=True, help='Verify manifest')
1288
def vendor_manifest(self, files, verify):
1289
from mozbuild.vendor_manifest import verify_manifests
1290
verify_manifests(files)
1291
1292
1293
@CommandProvider
1294
class WebRTCGTestCommands(GTestCommands):
1295
@Command('webrtc-gtest', category='testing',
1296
description='Run WebRTC.org GTest unit tests.')
1297
@CommandArgument('gtest_filter', default=b"*", nargs='?', metavar='gtest_filter',
1298
help="test_filter is a ':'-separated list of wildcard patterns "
1299
"(called the positive patterns), optionally followed by a '-' and "
1300
"another ':'-separated pattern list (called the negative patterns).")
1301
@CommandArgumentGroup('debugging')
1302
@CommandArgument('--debug', action='store_true', group='debugging',
1303
help='Enable the debugger. Not specifying a --debugger option will '
1304
'result in the default debugger being used.')
1305
@CommandArgument('--debugger', default=None, type=str, group='debugging',
1306
help='Name of debugger to use.')
1307
@CommandArgument('--debugger-args', default=None, metavar='params', type=str,
1308
group='debugging',
1309
help='Command-line arguments to pass to the debugger itself; '
1310
'split as the Bourne shell would.')
1311
def gtest(self, gtest_filter, debug, debugger,
1312
debugger_args):
1313
app_path = self.get_binary_path('webrtc-gtest')
1314
args = [app_path]
1315
1316
if debug or debugger or debugger_args:
1317
args = self.prepend_debugger_args(args, debugger, debugger_args)
1318
1319
# Used to locate resources used by tests
1320
cwd = os.path.join(self.topsrcdir, 'media', 'webrtc', 'trunk')
1321
1322
if not os.path.isdir(cwd):
1323
print('Unable to find working directory for tests: %s' % cwd)
1324
return 1
1325
1326
gtest_env = {
1327
# These tests are not run under ASAN upstream, so we need to
1328
# disable some checks.
1329
b'ASAN_OPTIONS': 'alloc_dealloc_mismatch=0',
1330
# Use GTest environment variable to control test execution
1331
# For details see:
1333
b'GTEST_FILTER': gtest_filter
1334
}
1335
1336
return self.run_process(args=args,
1337
append_env=gtest_env,
1338
cwd=cwd,
1339
ensure_exit_code=False,
1340
pass_thru=True)
1341
1342
1343
@CommandProvider
1344
class Repackage(MachCommandBase):
1345
'''Repackages artifacts into different formats.
1346
1347
This is generally used after packages are signed by the signing
1348
scriptworkers in order to bundle things up into shippable formats, such as a
1349
.dmg on OSX or an installer exe on Windows.
1350
'''
1351
@Command('repackage', category='misc',
1352
description='Repackage artifacts into different formats.')
1353
def repackage(self):
1354
print("Usage: ./mach repackage [dmg|installer|mar] [args...]")
1355
1356
@SubCommand('repackage', 'dmg',
1357
description='Repackage a tar file into a .dmg for OSX')
1358
@CommandArgument('--input', '-i', type=str, required=True,
1359
help='Input filename')
1360
@CommandArgument('--output', '-o', type=str, required=True,
1361
help='Output filename')
1362
def repackage_dmg(self, input, output):
1363
if not os.path.exists(input):
1364
print('Input file does not exist: %s' % input)
1365
return 1
1366
1367
if not os.path.exists(os.path.join(self.topobjdir, 'config.status')):
1368
print('config.status not found. Please run |mach configure| '
1369
'prior to |mach repackage|.')
1370
return 1
1371
1372
from mozbuild.repackaging.dmg import repackage_dmg
1373
repackage_dmg(input, output)
1374
1375
@SubCommand('repackage', 'installer',
1376
description='Repackage into a Windows installer exe')
1377
@CommandArgument('--tag', type=str, required=True,
1378
help='The .tag file used to build the installer')
1379
@CommandArgument('--setupexe', type=str, required=True,
1380
help='setup.exe file inside the installer')
1381
@CommandArgument('--package', type=str, required=False,
1382
help='Optional package .zip for building a full installer')
1383
@CommandArgument('--output', '-o', type=str, required=True,
1384
help='Output filename')
1385
@CommandArgument('--package-name', type=str, required=False,
1386
help='Name of the package being rebuilt')
1387
@CommandArgument('--sfx-stub', type=str, required=True,
1388
help='Path to the self-extraction stub.')
1389
@CommandArgument('--use-upx', required=False, action='store_true',
1390
help='Run UPX on the self-extraction stub.')
1391
def repackage_installer(self, tag, setupexe, package, output, package_name, sfx_stub, use_upx):
1392
from mozbuild.repackaging.installer import repackage_installer
1393
repackage_installer(
1394
topsrcdir=self.topsrcdir,
1395
tag=tag,
1396
setupexe=setupexe,
1397
package=package,
1398
output=output,
1399
package_name=package_name,
1400
sfx_stub=sfx_stub,
1401
use_upx=use_upx,
1402
)
1403
1404
@SubCommand('repackage', 'msi',
1405
description='Repackage into a MSI')
1406
@CommandArgument('--wsx', type=str, required=True,
1407
help='The wsx file used to build the installer')
1408
@CommandArgument('--version', type=str, required=True,
1409
help='The Firefox version used to create the installer')
1410
@CommandArgument('--locale', type=str, required=True,
1411
help='The locale of the installer')
1412
@CommandArgument('--arch', type=str, required=True,
1413
help='The archtecture you are building.')
1414
@CommandArgument('--setupexe', type=str, required=True,
1415
help='setup.exe installer')
1416
@CommandArgument('--candle', type=str, required=False,
1417
help='location of candle binary')
1418
@CommandArgument('--light', type=str, required=False,
1419
help='location of light binary')
1420
@CommandArgument('--output', '-o', type=str, required=True,
1421
help='Output filename')
1422
def repackage_msi(self, wsx, version, locale, arch, setupexe, candle, light, output):
1423
from mozbuild.repackaging.msi import repackage_msi
1424
repackage_msi(
1425
topsrcdir=self.topsrcdir,
1426
wsx=wsx,
1427
version=version,
1428
locale=locale,
1429
arch=arch,
1430
setupexe=setupexe,
1431
candle=candle,
1432
light=light,
1433
output=output,
1434
)
1435
1436
@SubCommand('repackage', 'mar',
1437
description='Repackage into complete MAR file')
1438
@CommandArgument('--input', '-i', type=str, required=True,
1439
help='Input filename')
1440
@CommandArgument('--mar', type=str, required=True,
1441
help='Mar binary path')
1442
@CommandArgument('--output', '-o', type=str, required=True,
1443
help='Output filename')
1444
@CommandArgument('--format', type=str, default='lzma',
1445
choices=('lzma', 'bz2'),
1446
help='Mar format')
1447
@CommandArgument('--arch', type=str, required=True,
1448
help='The archtecture you are building.')
1449
@CommandArgument('--mar-channel-id', type=str,
1450
help='Mar channel id')
1451
def repackage_mar(self, input, mar, output, format, arch, mar_channel_id):
1452
from mozbuild.repackaging.mar import repackage_mar
1453
repackage_mar(
1454
self.topsrcdir,
1455
input,
1456
mar,
1457
output,
1458
format,
1459
arch=arch,
1460
mar_channel_id=mar_channel_id,
1461
)
1462
1463
1464
@CommandProvider
1465
class Analyze(MachCommandBase):
1466
""" Get information about a file in the build graph """
1467
@Command('analyze', category='misc',
1468
description='Analyze the build graph.')
1469
def analyze(self):
1470
print("Usage: ./mach analyze [files|report] [args...]")
1471
1472
@SubCommand('analyze', 'files',
1473
description='Get incremental build cost for file(s) from the tup database.')
1474
@CommandArgument('--path', help='Path to tup db',
1475
default=None)
1476
@CommandArgument('files', nargs='*', help='Files to analyze')
1477
def analyze_files(self, path, files):
1478
from mozbuild.analyze.graph import Graph
1479
if path is None:
1480
path = mozpath.join(self.topsrcdir, '.tup', 'db')
1481
if os.path.isfile(path):
1482
g = Graph(path)
1483
g.file_summaries(files)
1484
g.close()
1485
else:
1486
res = 'Please make sure you have a local tup db *or* specify the location with --path.'
1487
print('Could not find a valid tup db in ' + path, res, sep='\n')
1488
return 1
1489
1490
@SubCommand('analyze', 'all',
1491
description='Get a report of files changed within the last n days and '
1492
'their corresponding build cost.')
1493
@CommandArgument('--days', '-d', type=int, default=14,
1494
help='Number of days to include in the report.')
1495
@CommandArgument('--format', default='pretty',
1496
choices=['pretty', 'csv', 'json', 'html'],
1497
help='Print or export data in the given format.')
1498
@CommandArgument('--limit', type=int, default=None,
1499
help='Get the top n most expensive files from the report.')
1500
@CommandArgument('--path', help='Path to cost_dict.gz',
1501
default=None)
1502
def analyze_report(self, days, format, limit, path):
1503
from mozbuild.analyze.hg import Report
1504
self._activate_virtualenv()
1505
try:
1506
self.virtualenv_manager.install_pip_package('tablib==0.12.1')
1507
except Exception:
1508
print('Could not install tablib via pip.')
1509
return 1
1510
if path is None:
1511
# go find tup db and make a cost_dict
1512
from mozbuild.analyze.graph import Graph
1513
db_path = mozpath.join(self.topsrcdir, '.tup', 'db')
1514
if os.path.isfile(db_path):
1515
g = Graph(db_path)
1516
r = Report(days, cost_dict=g.get_cost_dict())
1517
g.close()
1518
r.generate_output(format, limit, self.topobjdir)
1519
else:
1520
res = 'Please specify the location of cost_dict.gz with --path.'
1521
print('Could not find %s to make a cost dictionary.' % db_path, res, sep='\n')
1522
return 1
1523
else:
1524
# path to cost_dict.gz was specified
1525
if os.path.isfile(path):
1526
r = Report(days, path)
1527
r.generate_output(format, limit, self.topobjdir)
1528
else:
1529
res = 'Please specify the location of cost_dict.gz with --path.'
1530
print('Could not find cost_dict.gz at %s' % path, res, sep='\n')
1531
return 1
1532
1533
1534
@SettingsProvider
1535
class TelemetrySettings():
1536
config_settings = [
1537
('build.telemetry', 'boolean', """
1538
Enable submission of build system telemetry.
1539
""".strip(), False),
1540
]
1541
1542
1543
@CommandProvider
1544
class L10NCommands(MachCommandBase):
1545
@Command('package-multi-locale', category='post-build',
1546
description='Package a multi-locale version of the built product '
1547
'for distribution as an APK, DMG, etc.')
1548
@CommandArgument('--locales', metavar='LOCALES', nargs='+',
1549
required=True,
1550
help='List of locales to package, including "en-US"')
1551
@CommandArgument('--verbose', action='store_true',
1552
help='Log informative status messages.')
1553
def package_l10n(self, verbose=False, locales=[]):
1554
backends = self.substs['BUILD_BACKENDS']
1555
if 'RecursiveMake' not in backends:
1556
self.log(logging.ERROR, 'package-multi-locale', {'backends': backends},
1557
"Multi-locale packaging requires the full (non-artifact) "
1558
"'RecursiveMake' build backend; got {backends}.")
1559
return 1
1560
1561
if 'en-US' not in locales:
1562
self.log(logging.WARN, 'package-multi-locale', {'locales': locales},
1563
'List of locales does not include default locale "en-US": '
1564
'{locales}; adding "en-US"')
1565
locales.append('en-US')
1566
locales = list(sorted(locales))
1567
1568
append_env = {
1569
'MOZ_CHROME_MULTILOCALE': ' '.join(locales),
1570
}
1571
1572
for locale in locales:
1573
if locale == 'en-US':
1574
self.log(logging.INFO, 'package-multi-locale', {'locale': locale},
1575
'Skipping default locale {locale}')
1576
continue
1577
1578
self.log(logging.INFO, 'package-multi-locale', {'locale': locale},
1579
'Processing chrome Gecko resources for locale {locale}')
1580
self.run_process(
1581
[mozpath.join(self.topsrcdir, 'mach'), 'build', 'chrome-{}'.format(locale)],
1582
append_env=append_env,
1583
pass_thru=True,
1584
ensure_exit_code=True,
1585
cwd=mozpath.join(self.topsrcdir))
1586
1587
if self.substs['MOZ_BUILD_APP'] == 'mobile/android':
1588
self.log(logging.INFO, 'package-multi-locale', {},
1589
'Invoking `mach android assemble-app`')
1590
self.run_process(
1591
[mozpath.join(self.topsrcdir, 'mach'), 'android', 'assemble-app'],
1592
append_env=append_env,
1593
pass_thru=True,
1594
ensure_exit_code=True,
1595
cwd=mozpath.join(self.topsrcdir))
1596
1597
self.log(logging.INFO, 'package-multi-locale', {},
1598
'Invoking multi-locale `mach package`')
1599
self._run_make(
1600
directory=self.topobjdir,
1601
target=['package', 'AB_CD=multi'],
1602
append_env=append_env,
1603
pass_thru=True,
1604
ensure_exit_code=True)
1605
1606
if self.substs['MOZ_BUILD_APP'] == 'mobile/android':
1607
self.log(logging.INFO, 'package-multi-locale', {},
1608
'Invoking `mach android archive-geckoview`')
1609
self.run_process(
1610
[mozpath.join(self.topsrcdir, 'mach'), 'android',
1611
'archive-geckoview'.format(locale)],
1612
append_env=append_env,
1613
pass_thru=True,
1614
ensure_exit_code=True,
1615
cwd=mozpath.join(self.topsrcdir))
1616
1617
return 0