Source code

Revision control

Other Tools

1
# This Source Code Form is subject to the terms of the Mozilla Public
2
# License, v. 2.0. If a copy of the MPL was not distributed with this
3
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5
from __future__ import absolute_import, print_function, unicode_literals
6
7
import argparse
8
import importlib
9
import os
10
import sys
11
12
from mach.decorators import (
13
CommandProvider,
14
Command,
15
SettingsProvider,
16
SubCommand,
17
)
18
from mozboot.util import get_state_dir
19
from mozbuild.base import BuildEnvironmentNotFoundException, MachCommandBase
20
21
CONFIG_ENVIRONMENT_NOT_FOUND = '''
22
No config environment detected. This means we are unable to properly
23
detect test files in the specified paths or tags. Please run:
24
25
$ mach configure
26
27
and try again.
28
'''.lstrip()
29
30
31
class get_parser(object):
32
def __init__(self, selector):
33
self.selector = selector
34
35
def __call__(self):
36
mod = importlib.import_module('tryselect.selectors.{}'.format(self.selector))
37
return getattr(mod, '{}Parser'.format(self.selector.capitalize()))()
38
39
40
def generic_parser():
41
from tryselect.cli import BaseTryParser
42
parser = BaseTryParser()
43
parser.add_argument('argv', nargs=argparse.REMAINDER)
44
return parser
45
46
47
@SettingsProvider
48
class TryConfig(object):
49
50
@classmethod
51
def config_settings(cls):
52
from mach.registrar import Registrar
53
54
desc = "The default selector to use when running `mach try` without a subcommand."
55
choices = Registrar.command_handlers['try'].subcommand_handlers.keys()
56
57
return [
58
('try.default', 'string', desc, 'syntax', {'choices': choices}),
59
('try.maxhistory', 'int', "Maximum number of pushes to save in history.", 10),
60
]
61
62
63
@CommandProvider
64
class TrySelect(MachCommandBase):
65
66
def __init__(self, context):
67
super(TrySelect, self).__init__(context)
68
from tryselect import push
69
push.MAX_HISTORY = self._mach_context.settings['try']['maxhistory']
70
self.subcommand = self._mach_context.handler.subcommand
71
self.parser = self._mach_context.handler.parser
72
self._presets = None
73
74
@property
75
def presets(self):
76
if self._presets:
77
return self._presets
78
79
from tryselect.preset import MergedHandler
80
81
# Create our handler using both local and in-tree presets. The first
82
# path in this list will be treated as the 'user' file for the purposes
83
# of saving and editing. All subsequent paths are 'read-only'. We check
84
# an environment variable first for testing purposes.
85
if os.environ.get('MACH_TRY_PRESET_PATHS'):
86
preset_paths = os.environ['MACH_TRY_PRESET_PATHS'].split(os.pathsep)
87
else:
88
preset_paths = [
89
os.path.join(get_state_dir(), 'try_presets.yml'),
90
os.path.join(self.topsrcdir, 'tools', 'tryselect', 'try_presets.yml'),
91
]
92
93
self._presets = MergedHandler(*preset_paths)
94
return self._presets
95
96
def handle_presets(self, preset_action, save, preset, **kwargs):
97
"""Handle preset related arguments.
98
99
This logic lives here so that the underlying selectors don't need
100
special preset handling. They can all save and load presets the same
101
way.
102
"""
103
from tryselect.preset import migrate_old_presets
104
from tryselect.util.dicttools import merge
105
106
user_presets = self.presets.handlers[0]
107
108
# TODO: Remove after Jan 1, 2020.
109
migrate_old_presets(user_presets)
110
111
if preset_action == 'list':
112
self.presets.list()
113
sys.exit()
114
115
if preset_action == 'edit':
116
user_presets.edit()
117
sys.exit()
118
119
default = self.parser.get_default
120
if save:
121
selector = self.subcommand or self._mach_context.settings['try']['default']
122
123
# Only save non-default values for simplicity.
124
kwargs = {k: v for k, v in kwargs.items() if v != default(k)}
125
user_presets.save(save, selector=selector, **kwargs)
126
print('preset saved, run with: --preset={}'.format(save))
127
sys.exit()
128
129
if preset:
130
if preset not in self.presets:
131
self.parser.error("preset '{}' does not exist".format(preset))
132
133
name = preset
134
preset = self.presets[name]
135
selector = preset.pop('selector')
136
preset.pop('description', None) # description isn't used by any selectors
137
138
if not self.subcommand:
139
self.subcommand = selector
140
elif self.subcommand != selector:
141
print("error: preset '{}' exists for a different selector "
142
"(did you mean to run 'mach try {}' instead?)".format(
143
name, selector))
144
sys.exit(1)
145
146
# Order of precedence is defaults -> presets -> cli. Configuration
147
# from the right overwrites configuration from the left.
148
defaults = {}
149
nondefaults = {}
150
for k, v in kwargs.items():
151
if v == default(k):
152
defaults[k] = v
153
else:
154
nondefaults[k] = v
155
156
kwargs = merge(defaults, preset, nondefaults)
157
158
return kwargs
159
160
def handle_templates(self, **kwargs):
161
kwargs.setdefault('templates', {})
162
for cls in self.parser.templates.itervalues():
163
context = cls.context(**kwargs)
164
if context is not None:
165
kwargs['templates'].update(context)
166
167
for name in cls.dests:
168
del kwargs[name]
169
170
return kwargs
171
172
def run(self, **kwargs):
173
if 'preset' in self.parser.common_groups:
174
kwargs = self.handle_presets(**kwargs)
175
176
if self.parser.templates:
177
kwargs = self.handle_templates(**kwargs)
178
179
mod = importlib.import_module('tryselect.selectors.{}'.format(self.subcommand))
180
return mod.run(**kwargs)
181
182
@Command('try',
183
category='ci',
184
description='Push selected tasks to the try server',
185
parser=generic_parser)
186
def try_default(self, argv=None, **kwargs):
187
"""Push selected tests to the try server.
188
189
The |mach try| command is a frontend for scheduling tasks to
190
run on try server using selectors. A selector is a subcommand
191
that provides its own set of command line arguments and are
192
listed below.
193
194
If no subcommand is specified, the `syntax` selector is run by
195
default. Run |mach try syntax --help| for more information on
196
scheduling with the `syntax` selector.
197
"""
198
# We do special handling of presets here so that `./mach try --preset foo`
199
# works no matter what subcommand 'foo' was saved with.
200
preset = kwargs['preset']
201
if preset:
202
if preset not in self.presets:
203
self.parser.error("preset '{}' does not exist".format(preset))
204
205
self.subcommand = self.presets[preset]['selector']
206
207
sub = self.subcommand or self._mach_context.settings['try']['default']
208
return self._mach_context.commands.dispatch(
209
'try', subcommand=sub, context=self._mach_context, argv=argv, **kwargs)
210
211
@SubCommand('try',
212
'fuzzy',
213
description='Select tasks on try using a fuzzy finder',
214
parser=get_parser('fuzzy'))
215
def try_fuzzy(self, **kwargs):
216
"""Select which tasks to use with fzf.
217
218
This selector runs all task labels through a fuzzy finding interface.
219
All selected task labels and their dependencies will be scheduled on
220
try.
221
222
Keyboard Shortcuts
223
------------------
224
225
When in the fuzzy finder interface, start typing to filter down the
226
task list. Then use the following keyboard shortcuts to select tasks:
227
228
accept: <enter>
229
cancel: <ctrl-c> or <esc>
230
cursor-up: <ctrl-k> or <up>
231
cursor-down: <ctrl-j> or <down>
232
toggle-select-down: <tab>
233
toggle-select-up: <shift-tab>
234
select-all: <ctrl-a>
235
deselect-all: <ctrl-d>
236
toggle-all: <ctrl-t>
237
clear-input: <alt-bspace>
238
239
There are many more shortcuts enabled by default, you can also define
240
your own shortcuts by setting `--bind` in the $FZF_DEFAULT_OPTS
241
environment variable. See `man fzf` for more info.
242
243
Extended Search
244
---------------
245
246
When typing in search terms, the following modifiers can be applied:
247
248
'word: exact match (line must contain the literal string "word")
249
^word: exact prefix match (line must start with literal "word")
250
word$: exact suffix match (line must end with literal "word")
251
!word: exact negation match (line must not contain literal "word")
252
'a | 'b: OR operator (joins two exact match operators together)
253
254
For example:
255
256
^start 'exact | !ignore fuzzy end$
257
"""
258
if kwargs.pop('interactive'):
259
kwargs['query'].append('INTERACTIVE')
260
261
if kwargs.pop('intersection'):
262
kwargs['intersect_query'] = kwargs['query']
263
del kwargs['query']
264
265
if kwargs.get('save') and not kwargs.get('query'):
266
# If saving preset without -q/--query, allow user to use the
267
# interface to build the query.
268
kwargs_copy = kwargs.copy()
269
kwargs_copy['push'] = False
270
kwargs_copy['save'] = None
271
kwargs['query'] = self.run(save_query=True, **kwargs_copy)
272
273
if kwargs.get('paths'):
274
kwargs['test_paths'] = kwargs['paths']
275
276
return self.run(**kwargs)
277
278
@SubCommand('try',
279
'chooser',
280
description='Schedule tasks by selecting them from a web '
281
'interface.',
282
parser=get_parser('chooser'))
283
def try_chooser(self, **kwargs):
284
"""Push tasks selected from a web interface to try.
285
286
This selector will build the taskgraph and spin up a dynamically
287
created 'trychooser-like' web-page on the localhost. After a selection
288
has been made, pressing the 'Push' button will automatically push the
289
selection to try.
290
"""
291
self._activate_virtualenv()
292
path = os.path.join('tools', 'tryselect', 'selectors', 'chooser', 'requirements.txt')
293
self.virtualenv_manager.install_pip_requirements(path, quiet=True)
294
295
return self.run(**kwargs)
296
297
@SubCommand('try',
298
'again',
299
description='Schedule a previously generated (non try syntax) '
300
'push again.',
301
parser=get_parser('again'))
302
def try_again(self, **kwargs):
303
return self.run(**kwargs)
304
305
@SubCommand('try',
306
'empty',
307
description='Push to try without scheduling any tasks.',
308
parser=get_parser('empty'))
309
def try_empty(self, **kwargs):
310
"""Push to try, running no builds or tests
311
312
This selector does not prompt you to run anything, it just pushes
313
your patches to try, running no builds or tests by default. After
314
the push finishes, you can manually add desired jobs to your push
315
via Treeherder's Add New Jobs feature, located in the per-push
316
menu.
317
"""
318
return self.run(**kwargs)
319
320
@SubCommand('try',
321
'syntax',
322
description='Select tasks on try using try syntax',
323
parser=get_parser('syntax'))
324
def try_syntax(self, **kwargs):
325
"""Push the current tree to try, with the specified syntax.
326
327
Build options, platforms and regression tests may be selected
328
using the usual try options (-b, -p and -u respectively). In
329
addition, tests in a given directory may be automatically
330
selected by passing that directory as a positional argument to the
331
command. For example:
332
333
mach try -b d -p linux64 dom testing/web-platform/tests/dom
334
335
would schedule a try run for linux64 debug consisting of all
336
tests under dom/ and testing/web-platform/tests/dom.
337
338
Test selection using positional arguments is available for
339
mochitests, reftests, xpcshell tests and web-platform-tests.
340
341
Tests may be also filtered by passing --tag to the command,
342
which will run only tests marked as having the specified
343
tags e.g.
344
345
mach try -b d -p win64 --tag media
346
347
would run all tests tagged 'media' on Windows 64.
348
349
If both positional arguments or tags and -u are supplied, the
350
suites in -u will be run in full. Where tests are selected by
351
positional argument they will be run in a single chunk.
352
353
If no build option is selected, both debug and opt will be
354
scheduled. If no platform is selected a default is taken from
355
the AUTOTRY_PLATFORM_HINT environment variable, if set.
356
357
The command requires either its own mercurial extension ("push-to-try",
358
installable from mach vcs-setup) or a git repo using git-cinnabar
359
(installable from mach vcs-setup).
360
361
"""
362
try:
363
if self.substs.get("MOZ_ARTIFACT_BUILDS"):
364
kwargs['local_artifact_build'] = True
365
except BuildEnvironmentNotFoundException:
366
# If we don't have a build locally, we can't tell whether
367
# an artifact build is desired, but we still want the
368
# command to succeed, if possible.
369
pass
370
371
config_status = os.path.join(self.topobjdir, 'config.status')
372
if (kwargs['paths'] or kwargs['tags']) and not config_status:
373
print(CONFIG_ENVIRONMENT_NOT_FOUND)
374
sys.exit(1)
375
376
return self.run(**kwargs)
377
378
@SubCommand('try',
379
'coverage',
380
description='Select tasks on try using coverage data',
381
parser=get_parser('coverage'))
382
def try_coverage(self, **kwargs):
383
"""Select which tasks to use using coverage data.
384
"""
385
return self.run(**kwargs)
386
387
@SubCommand('try',
388
'release',
389
description='Push the current tree to try, configured for a staging release.',
390
parser=get_parser('release'))
391
def try_release(self, **kwargs):
392
"""Push the current tree to try, configured for a staging release.
393
"""
394
return self.run(**kwargs)