Source code

Revision control

Other Tools

1
# This Source Code Form is subject to the terms of the Mozilla Public
2
# License, v. 2.0. If a copy of the MPL was not distributed with this
3
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5
from __future__ import absolute_import, print_function, unicode_literals
6
7
import hashlib
8
import json
9
import os
10
import re
11
import shutil
12
import sys
13
from collections import defaultdict
14
15
from mozboot.util import get_state_dir
16
from mozbuild.base import MozbuildObject
17
from mozpack.files import FileFinder
18
from moztest.resolve import TestResolver, get_suite_definition
19
20
import taskgraph
21
from taskgraph.generator import TaskGraphGenerator
22
from taskgraph.parameters import (
23
ParameterMismatch,
24
parameters_loader,
25
)
26
from taskgraph.taskgraph import TaskGraph
27
28
here = os.path.abspath(os.path.dirname(__file__))
29
build = MozbuildObject.from_environment(cwd=here)
30
31
32
PARAMETER_MISMATCH = """
33
ERROR - The parameters being used to generate tasks differ from those expected
34
by your working copy:
35
36
{}
37
38
To fix this, either rebase onto the latest mozilla-central or pass in
39
-p/--parameters. For more information on how to define parameters, see:
41
"""
42
43
44
def invalidate(cache, root):
45
if not os.path.isfile(cache):
46
return
47
48
tc_dir = os.path.join(root, 'taskcluster')
49
tmod = max(os.path.getmtime(os.path.join(tc_dir, p)) for p, _ in FileFinder(tc_dir))
50
cmod = os.path.getmtime(cache)
51
52
if tmod > cmod:
53
os.remove(cache)
54
55
56
def generate_tasks(params=None, full=False):
57
# TODO: Remove after January 1st, 2020.
58
# Try to delete the old taskgraph cache directories.
59
root = build.topsrcdir
60
root_hash = hashlib.sha256(os.path.abspath(root)).hexdigest()
61
old_cache_dirs = [
62
os.path.join(get_state_dir(), 'cache', 'taskgraph'),
63
os.path.join(get_state_dir(), 'cache', root_hash, 'taskgraph'),
64
]
65
for cache_dir in old_cache_dirs:
66
if os.path.isdir(cache_dir):
67
shutil.rmtree(cache_dir)
68
69
cache_dir = os.path.join(get_state_dir(srcdir=True), 'cache', 'taskgraph')
70
attr = 'full_task_set' if full else 'target_task_set'
71
cache = os.path.join(cache_dir, attr)
72
73
invalidate(cache, root)
74
if os.path.isfile(cache):
75
with open(cache, 'r') as fh:
76
return TaskGraph.from_json(json.load(fh))[1]
77
78
if not os.path.isdir(cache_dir):
79
os.makedirs(cache_dir)
80
81
print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
82
83
taskgraph.fast = True
84
cwd = os.getcwd()
85
os.chdir(root)
86
87
root = os.path.join(root, 'taskcluster', 'ci')
88
params = parameters_loader(params, strict=False, overrides={'try_mode': 'try_select'})
89
90
# Cache both full_task_set and target_task_set regardless of whether or not
91
# --full was requested. Caching is cheap and can potentially save a lot of
92
# time.
93
generator = TaskGraphGenerator(root_dir=root, parameters=params)
94
95
def generate(attr):
96
try:
97
tg = getattr(generator, attr)
98
except ParameterMismatch as e:
99
print(PARAMETER_MISMATCH.format(e.args[0]))
100
sys.exit(1)
101
102
# write cache
103
with open(os.path.join(cache_dir, attr), 'w') as fh:
104
json.dump(tg.to_json(), fh)
105
return tg
106
107
tg_full = generate('full_task_set')
108
tg_target = generate('target_task_set')
109
# discard results from these, we only need cache.
110
if full:
111
generate('full_task_graph')
112
generate('target_task_graph')
113
114
os.chdir(cwd)
115
if full:
116
return tg_full
117
return tg_target
118
119
120
def filter_tasks_by_paths(tasks, paths):
121
resolver = TestResolver.from_environment(cwd=here)
122
run_suites, run_tests = resolver.resolve_metadata(paths)
123
flavors = set([(t['flavor'], t.get('subsuite')) for t in run_tests])
124
125
task_regexes = set()
126
for flavor, subsuite in flavors:
127
_, suite = get_suite_definition(flavor, subsuite, strict=True)
128
if 'task_regex' not in suite:
129
print("warning: no tasks could be resolved from flavor '{}'{}".format(
130
flavor, " and subsuite '{}'".format(subsuite) if subsuite else ""))
131
continue
132
133
task_regexes.update(suite['task_regex'])
134
135
def match_task(task):
136
return any(re.search(pattern, task) for pattern in task_regexes)
137
138
return filter(match_task, tasks)
139
140
141
def resolve_tests_by_suite(paths):
142
resolver = TestResolver.from_environment(cwd=here)
143
_, run_tests = resolver.resolve_metadata(paths)
144
145
suite_to_tests = defaultdict(list)
146
147
# A dictionary containing all the input paths that we haven't yet
148
# assigned to a specific test flavor.
149
remaining_paths_by_suite = defaultdict(lambda: set(paths))
150
151
for test in run_tests:
152
key, _ = get_suite_definition(test['flavor'], test.get('subsuite'), strict=True)
153
154
test_path = test.get('srcdir_relpath')
155
if test_path is None:
156
continue
157
found_path = None
158
for path in remaining_paths_by_suite[key]:
159
if test_path.startswith(path):
160
found_path = path
161
break
162
if found_path:
163
suite_to_tests[key].append(found_path)
164
remaining_paths_by_suite[key].remove(found_path)
165
166
return suite_to_tests