Source code

Revision control

Other Tools

1
# This Source Code Form is subject to the terms of the Mozilla Public
2
# License, v. 2.0. If a copy of the MPL was not distributed with this
3
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5
from __future__ import absolute_import, print_function, unicode_literals
6
7
import argparse
8
import importlib
9
import os
10
import sys
11
12
from mach.decorators import (
13
CommandProvider,
14
Command,
15
SettingsProvider,
16
SubCommand,
17
)
18
from mozboot.util import get_state_dir
19
from mozbuild.base import BuildEnvironmentNotFoundException, MachCommandBase
20
21
22
CONFIG_ENVIRONMENT_NOT_FOUND = '''
23
No config environment detected. This means we are unable to properly
24
detect test files in the specified paths or tags. Please run:
25
26
$ mach configure
27
28
and try again.
29
'''.lstrip()
30
31
32
class get_parser(object):
33
def __init__(self, selector):
34
self.selector = selector
35
36
def __call__(self):
37
mod = importlib.import_module('tryselect.selectors.{}'.format(self.selector))
38
return getattr(mod, '{}Parser'.format(self.selector.capitalize()))()
39
40
41
def generic_parser():
42
from tryselect.cli import BaseTryParser
43
parser = BaseTryParser()
44
parser.add_argument('argv', nargs=argparse.REMAINDER)
45
return parser
46
47
48
@SettingsProvider
49
class TryConfig(object):
50
51
@classmethod
52
def config_settings(cls):
53
from mach.registrar import Registrar
54
55
desc = "The default selector to use when running `mach try` without a subcommand."
56
choices = Registrar.command_handlers['try'].subcommand_handlers.keys()
57
58
return [
59
('try.default', 'string', desc, 'syntax', {'choices': choices}),
60
('try.maxhistory', 'int', "Maximum number of pushes to save in history.", 10),
61
]
62
63
64
@CommandProvider
65
class TrySelect(MachCommandBase):
66
67
def __init__(self, context):
68
super(TrySelect, self).__init__(context)
69
from tryselect import push
70
push.MAX_HISTORY = self._mach_context.settings['try']['maxhistory']
71
self.subcommand = self._mach_context.handler.subcommand
72
self.parser = self._mach_context.handler.parser
73
self._presets = None
74
75
@property
76
def presets(self):
77
if self._presets:
78
return self._presets
79
80
from tryselect.preset import MergedHandler
81
82
# Create our handler using both local and in-tree presets. The first
83
# path in this list will be treated as the 'user' file for the purposes
84
# of saving and editing. All subsequent paths are 'read-only'. We check
85
# an environment variable first for testing purposes.
86
if os.environ.get('MACH_TRY_PRESET_PATHS'):
87
preset_paths = os.environ['MACH_TRY_PRESET_PATHS'].split(os.pathsep)
88
else:
89
preset_paths = [
90
os.path.join(get_state_dir(), 'try_presets.yml'),
91
os.path.join(self.topsrcdir, 'tools', 'tryselect', 'try_presets.yml'),
92
]
93
94
self._presets = MergedHandler(*preset_paths)
95
return self._presets
96
97
def handle_presets(self, preset_action, save, preset, **kwargs):
98
"""Handle preset related arguments.
99
100
This logic lives here so that the underlying selectors don't need
101
special preset handling. They can all save and load presets the same
102
way.
103
"""
104
from tryselect.preset import migrate_old_presets
105
from tryselect.util.dicttools import merge
106
107
user_presets = self.presets.handlers[0]
108
109
# TODO: Remove after Jan 1, 2020.
110
migrate_old_presets(user_presets)
111
112
if preset_action == 'list':
113
self.presets.list()
114
sys.exit()
115
116
if preset_action == 'edit':
117
user_presets.edit()
118
sys.exit()
119
120
default = self.parser.get_default
121
if save:
122
selector = self.subcommand or self._mach_context.settings['try']['default']
123
124
# Only save non-default values for simplicity.
125
kwargs = {k: v for k, v in kwargs.items() if v != default(k)}
126
user_presets.save(save, selector=selector, **kwargs)
127
print('preset saved, run with: --preset={}'.format(save))
128
sys.exit()
129
130
if preset:
131
if preset not in self.presets:
132
self.parser.error("preset '{}' does not exist".format(preset))
133
134
name = preset
135
preset = self.presets[name]
136
selector = preset.pop('selector')
137
preset.pop('description', None) # description isn't used by any selectors
138
139
if not self.subcommand:
140
self.subcommand = selector
141
elif self.subcommand != selector:
142
print("error: preset '{}' exists for a different selector "
143
"(did you mean to run 'mach try {}' instead?)".format(
144
name, selector))
145
sys.exit(1)
146
147
# Order of precedence is defaults -> presets -> cli. Configuration
148
# from the right overwrites configuration from the left.
149
defaults = {}
150
nondefaults = {}
151
for k, v in kwargs.items():
152
if v == default(k):
153
defaults[k] = v
154
else:
155
nondefaults[k] = v
156
157
kwargs = merge(defaults, preset, nondefaults)
158
159
return kwargs
160
161
def handle_try_config(self, **kwargs):
162
from tryselect.util.dicttools import merge
163
kwargs.setdefault('try_config', {})
164
for cls in self.parser.templates.itervalues():
165
try_config = cls.try_config(**kwargs)
166
if try_config is not None:
167
kwargs['try_config'] = merge(kwargs['try_config'], try_config)
168
169
for name in cls.dests:
170
del kwargs[name]
171
172
return kwargs
173
174
def run(self, **kwargs):
175
if 'preset' in self.parser.common_groups:
176
kwargs = self.handle_presets(**kwargs)
177
178
if self.parser.templates:
179
kwargs = self.handle_try_config(**kwargs)
180
181
mod = importlib.import_module('tryselect.selectors.{}'.format(self.subcommand))
182
return mod.run(**kwargs)
183
184
@Command('try',
185
category='ci',
186
description='Push selected tasks to the try server',
187
parser=generic_parser)
188
def try_default(self, argv=None, **kwargs):
189
"""Push selected tests to the try server.
190
191
The |mach try| command is a frontend for scheduling tasks to
192
run on try server using selectors. A selector is a subcommand
193
that provides its own set of command line arguments and are
194
listed below.
195
196
If no subcommand is specified, the `syntax` selector is run by
197
default. Run |mach try syntax --help| for more information on
198
scheduling with the `syntax` selector.
199
"""
200
# We do special handling of presets here so that `./mach try --preset foo`
201
# works no matter what subcommand 'foo' was saved with.
202
preset = kwargs['preset']
203
if preset:
204
if preset not in self.presets:
205
self.parser.error("preset '{}' does not exist".format(preset))
206
207
self.subcommand = self.presets[preset]['selector']
208
209
sub = self.subcommand or self._mach_context.settings['try']['default']
210
return self._mach_context.commands.dispatch(
211
'try', subcommand=sub, context=self._mach_context, argv=argv, **kwargs)
212
213
@SubCommand('try',
214
'fuzzy',
215
description='Select tasks on try using a fuzzy finder',
216
parser=get_parser('fuzzy'))
217
def try_fuzzy(self, **kwargs):
218
"""Select which tasks to run with a fuzzy finding interface (fzf).
219
220
When entering the fzf interface you'll be confronted by two panes. The
221
one on the left contains every possible task you can schedule, the one
222
on the right contains the list of selected tasks. In other words, the
223
tasks that will be scheduled once you you press <enter>.
224
225
At first fzf will automatically select whichever task is under your
226
cursor, which simplifies the case when you are looking for a single
227
task. But normally you'll want to select many tasks. To accomplish
228
you'll generally start by typing a query in the search bar to filter
229
down the list of tasks (see Extended Search below). Then you'll either:
230
231
A) Move the cursor to each task you want and press <tab> to select it.
232
Notice it now shows up in the pane to the right.
233
234
OR
235
236
B) Press <ctrl-a> to select every task that matches your filter.
237
238
You can delete your query, type a new one and select further tasks as
239
many times as you like. Once you are happy with your selection, press
240
<enter> to push the selected tasks to try.
241
242
All selected task labels and their dependencies will be scheduled. This
243
means you can select a test task and its build will automatically be
244
filled in.
245
246
247
Keyboard Shortcuts
248
------------------
249
250
When in the fuzzy finder interface, start typing to filter down the
251
task list. Then use the following keyboard shortcuts to select tasks:
252
253
Ctrl-K / Up => Move cursor up
254
Ctrl-J / Down => Move cursor down
255
Tab => Select task + move cursor down
256
Shift-Tab => Select task + move cursor up
257
Ctrl-A => Select all currently filtered tasks
258
Ctrl-D => De-select all currently filtered tasks
259
Ctrl-T => Toggle select all currently filtered tasks
260
Alt-Bspace => Clear query from input bar
261
Enter => Accept selection and exit
262
Ctrl-C / Esc => Cancel selection and exit
263
? => Toggle preview pane
264
265
There are many more shortcuts enabled by default, you can also define
266
your own shortcuts by setting `--bind` in the $FZF_DEFAULT_OPTS
267
environment variable. See `man fzf` for more info.
268
269
270
Extended Search
271
---------------
272
273
When typing in search terms, the following modifiers can be applied:
274
275
'word: exact match (line must contain the literal string "word")
276
^word: exact prefix match (line must start with literal "word")
277
word$: exact suffix match (line must end with literal "word")
278
!word: exact negation match (line must not contain literal "word")
279
'a | 'b: OR operator (joins two exact match operators together)
280
281
For example:
282
283
^start 'exact | !ignore fuzzy end$
284
285
286
Documentation
287
-------------
288
289
For more detailed documentation, please see:
291
"""
292
if kwargs.pop('interactive'):
293
kwargs['query'].append('INTERACTIVE')
294
295
if kwargs.pop('intersection'):
296
kwargs['intersect_query'] = kwargs['query']
297
del kwargs['query']
298
299
if kwargs.get('save') and not kwargs.get('query'):
300
# If saving preset without -q/--query, allow user to use the
301
# interface to build the query.
302
kwargs_copy = kwargs.copy()
303
kwargs_copy['push'] = False
304
kwargs_copy['save'] = None
305
kwargs['query'] = self.run(save_query=True, **kwargs_copy)
306
307
if kwargs.get('paths'):
308
kwargs['test_paths'] = kwargs['paths']
309
310
return self.run(**kwargs)
311
312
@SubCommand('try',
313
'chooser',
314
description='Schedule tasks by selecting them from a web '
315
'interface.',
316
parser=get_parser('chooser'))
317
def try_chooser(self, **kwargs):
318
"""Push tasks selected from a web interface to try.
319
320
This selector will build the taskgraph and spin up a dynamically
321
created 'trychooser-like' web-page on the localhost. After a selection
322
has been made, pressing the 'Push' button will automatically push the
323
selection to try.
324
"""
325
self._activate_virtualenv()
326
path = os.path.join('tools', 'tryselect', 'selectors', 'chooser', 'requirements.txt')
327
self.virtualenv_manager.install_pip_requirements(path, quiet=True)
328
329
return self.run(**kwargs)
330
331
@SubCommand('try',
332
'again',
333
description='Schedule a previously generated (non try syntax) '
334
'push again.',
335
parser=get_parser('again'))
336
def try_again(self, **kwargs):
337
return self.run(**kwargs)
338
339
@SubCommand('try',
340
'empty',
341
description='Push to try without scheduling any tasks.',
342
parser=get_parser('empty'))
343
def try_empty(self, **kwargs):
344
"""Push to try, running no builds or tests
345
346
This selector does not prompt you to run anything, it just pushes
347
your patches to try, running no builds or tests by default. After
348
the push finishes, you can manually add desired jobs to your push
349
via Treeherder's Add New Jobs feature, located in the per-push
350
menu.
351
"""
352
return self.run(**kwargs)
353
354
@SubCommand('try',
355
'syntax',
356
description='Select tasks on try using try syntax',
357
parser=get_parser('syntax'))
358
def try_syntax(self, **kwargs):
359
"""Push the current tree to try, with the specified syntax.
360
361
Build options, platforms and regression tests may be selected
362
using the usual try options (-b, -p and -u respectively). In
363
addition, tests in a given directory may be automatically
364
selected by passing that directory as a positional argument to the
365
command. For example:
366
367
mach try -b d -p linux64 dom testing/web-platform/tests/dom
368
369
would schedule a try run for linux64 debug consisting of all
370
tests under dom/ and testing/web-platform/tests/dom.
371
372
Test selection using positional arguments is available for
373
mochitests, reftests, xpcshell tests and web-platform-tests.
374
375
Tests may be also filtered by passing --tag to the command,
376
which will run only tests marked as having the specified
377
tags e.g.
378
379
mach try -b d -p win64 --tag media
380
381
would run all tests tagged 'media' on Windows 64.
382
383
If both positional arguments or tags and -u are supplied, the
384
suites in -u will be run in full. Where tests are selected by
385
positional argument they will be run in a single chunk.
386
387
If no build option is selected, both debug and opt will be
388
scheduled. If no platform is selected a default is taken from
389
the AUTOTRY_PLATFORM_HINT environment variable, if set.
390
391
The command requires either its own mercurial extension ("push-to-try",
392
installable from mach vcs-setup) or a git repo using git-cinnabar
393
(installable from mach vcs-setup).
394
395
"""
396
try:
397
if self.substs.get("MOZ_ARTIFACT_BUILDS"):
398
kwargs['local_artifact_build'] = True
399
except BuildEnvironmentNotFoundException:
400
# If we don't have a build locally, we can't tell whether
401
# an artifact build is desired, but we still want the
402
# command to succeed, if possible.
403
pass
404
405
config_status = os.path.join(self.topobjdir, 'config.status')
406
if (kwargs['paths'] or kwargs['tags']) and not config_status:
407
print(CONFIG_ENVIRONMENT_NOT_FOUND)
408
sys.exit(1)
409
410
return self.run(**kwargs)
411
412
@SubCommand('try',
413
'coverage',
414
description='Select tasks on try using coverage data',
415
parser=get_parser('coverage'))
416
def try_coverage(self, **kwargs):
417
"""Select which tasks to use using coverage data.
418
"""
419
return self.run(**kwargs)
420
421
@SubCommand('try',
422
'release',
423
description='Push the current tree to try, configured for a staging release.',
424
parser=get_parser('release'))
425
def try_release(self, **kwargs):
426
"""Push the current tree to try, configured for a staging release.
427
"""
428
return self.run(**kwargs)