Source code

Revision control

Other Tools

1
# This Source Code Form is subject to the terms of the Mozilla Public
2
# License, v. 2.0. If a copy of the MPL was not distributed with this
3
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5
from __future__ import absolute_import, print_function, unicode_literals
6
7
import os
8
import re
9
10
from collections import deque
11
import taskgraph
12
from taskgraph.transforms.base import TransformSequence
13
from taskgraph.transforms.task import _run_task_suffix
14
from .. import GECKO
15
from taskgraph.util.docker import (
16
generate_context_hash,
17
)
18
from taskgraph.util.taskcluster import get_root_url
19
from taskgraph.util.schema import (
20
Schema,
21
)
22
from voluptuous import (
23
Optional,
24
Required,
25
)
26
from .task import task_description_schema
27
28
DIGEST_RE = re.compile('^[0-9a-f]{64}$')
29
30
transforms = TransformSequence()
31
32
docker_image_schema = Schema({
33
# Name of the docker image.
34
Required('name'): basestring,
35
36
# Name of the parent docker image.
37
Optional('parent'): basestring,
38
39
# Treeherder symbol.
40
Required('symbol'): basestring,
41
42
# relative path (from config.path) to the file the docker image was defined
43
# in.
44
Optional('job-from'): basestring,
45
46
# Arguments to use for the Dockerfile.
47
Optional('args'): {basestring: basestring},
48
49
# Name of the docker image definition under taskcluster/docker, when
50
# different from the docker image name.
51
Optional('definition'): basestring,
52
53
# List of package tasks this docker image depends on.
54
Optional('packages'): [basestring],
55
56
Optional(
57
"index",
58
description="information for indexing this build so its artifacts can be discovered",
59
): task_description_schema['index'],
60
61
Optional(
62
"cache",
63
description="Whether this image should be cached based on inputs.",
64
): bool,
65
})
66
67
68
transforms.add_validate(docker_image_schema)
69
70
71
def order_image_tasks(config, tasks):
72
"""Iterate image tasks in an order where parent images come first."""
73
pending = deque(tasks)
74
task_names = {task['name'] for task in pending}
75
emitted = set()
76
while True:
77
try:
78
task = pending.popleft()
79
except IndexError:
80
break
81
parent = task.get('parent')
82
if parent and parent not in emitted:
83
if parent not in task_names:
84
raise Exception('Missing parent image for {}-{}: {}'.format(
85
config.kind, task['name'], parent))
86
pending.append(task)
87
continue
88
emitted.add(task['name'])
89
yield task
90
91
92
@transforms.add
93
def fill_template(config, tasks):
94
available_packages = set()
95
for task in config.kind_dependencies_tasks:
96
if task.kind != 'packages':
97
continue
98
name = task.label.replace('packages-', '')
99
available_packages.add(name)
100
101
context_hashes = {}
102
103
for task in order_image_tasks(config, tasks):
104
image_name = task.pop('name')
105
job_symbol = task.pop('symbol')
106
args = task.pop('args', {})
107
definition = task.pop('definition', image_name)
108
packages = task.pop('packages', [])
109
parent = task.pop('parent', None)
110
111
for p in packages:
112
if p not in available_packages:
113
raise Exception('Missing package job for {}-{}: {}'.format(
114
config.kind, image_name, p))
115
116
# Generating the context hash relies on arguments being set, so we
117
# set this now, although it's not the final value (it's a
118
# task-reference value, see further below). We add the package routes
119
# containing a hash to get the overall docker image hash, so changes
120
# to packages will be reflected in the docker image hash.
121
args['DOCKER_IMAGE_PACKAGES'] = ' '.join('<{}>'.format(p)
122
for p in packages)
123
if parent:
124
args['DOCKER_IMAGE_PARENT'] = '{}:{}'.format(parent, context_hashes[parent])
125
126
args['TASKCLUSTER_ROOT_URL'] = get_root_url(False)
127
128
if not taskgraph.fast:
129
context_path = os.path.join('taskcluster', 'docker', definition)
130
context_hash = generate_context_hash(
131
GECKO, context_path, image_name, args)
132
else:
133
context_hash = '0'*40
134
digest_data = [context_hash]
135
context_hashes[image_name] = context_hash
136
137
description = 'Build the docker image {} for use by dependent tasks'.format(
138
image_name)
139
140
# Adjust the zstandard compression level based on the execution level.
141
# We use faster compression for level 1 because we care more about
142
# end-to-end times. We use slower/better compression for other levels
143
# because images are read more often and it is worth the trade-off to
144
# burn more CPU once to reduce image size.
145
zstd_level = '3' if int(config.params['level']) == 1 else '10'
146
147
# include some information that is useful in reconstructing this task
148
# from JSON
149
taskdesc = {
150
'label': 'build-docker-image-' + image_name,
151
'description': description,
152
'attributes': {'image_name': image_name},
153
'expires-after': '28 days' if config.params.is_try() else '1 year',
154
'scopes': [
155
'secrets:get:project/taskcluster/gecko/hgfingerprint',
156
'secrets:get:project/taskcluster/gecko/hgmointernal',
157
],
158
'treeherder': {
159
'symbol': job_symbol,
160
'platform': 'taskcluster-images/opt',
161
'kind': 'other',
162
'tier': 1,
163
},
164
'run-on-projects': [],
165
'worker-type': 'images',
166
'worker': {
167
'implementation': 'docker-worker',
168
'os': 'linux',
169
'artifacts': [{
170
'type': 'file',
171
'path': '/builds/worker/workspace/artifacts/image.tar.zst',
172
'name': 'public/image.tar.zst',
173
}],
174
'env': {
175
'HG_STORE_PATH': '/builds/worker/checkouts/hg-store',
176
'HASH': context_hash,
177
'PROJECT': config.params['project'],
178
'IMAGE_NAME': image_name,
179
'DOCKER_IMAGE_ZSTD_LEVEL': zstd_level,
180
'GECKO_BASE_REPOSITORY': config.params['base_repository'],
181
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
182
'GECKO_HEAD_REV': config.params['head_rev'],
183
},
184
'chain-of-trust': True,
185
'docker-in-docker': True,
186
'taskcluster-proxy': True,
187
'max-run-time': 7200,
188
# Retry on apt-get errors.
189
'retry-exit-status': [100],
190
},
191
}
192
# Retry for 'funsize-update-generator' if exit status code is -1
193
if image_name in ['funsize-update-generator']:
194
taskdesc['worker']['retry-exit-status'] = [-1]
195
196
worker = taskdesc['worker']
197
198
# We use the in-tree image_builder image to build docker images, but
199
# that can't be used to build the image_builder image itself,
200
# obviously. So we fall back to an image on docker hub, identified
201
# by hash. After the image-builder image is updated, it's best to push
202
# and update this hash as well, to keep image-builder builds up to date.
203
if image_name == 'image_builder':
204
hash = 'sha256:c6622fd3e5794842ad83d129850330b26e6ba671e39c58ee288a616a3a1c4c73'
205
worker['docker-image'] = 'taskcluster/image_builder@' + hash
206
# Keep in sync with the Dockerfile used to generate the
207
# docker image whose digest is referenced above.
208
worker['volumes'] = [
209
'/builds/worker/checkouts',
210
'/builds/worker/workspace',
211
]
212
cache_name = 'imagebuilder-v1'
213
else:
214
worker['docker-image'] = {'in-tree': 'image_builder'}
215
cache_name = 'imagebuilder-sparse-{}'.format(_run_task_suffix())
216
# Force images built against the in-tree image builder to
217
# have a different digest by adding a fixed string to the
218
# hashed data.
219
# Append to this data whenever the image builder's output behavior
220
# is changed, in order to force all downstream images to be rebuilt and
221
# cached distinctly.
222
digest_data.append('image_builder')
223
# Updated for squashing images in Bug 1527394
224
digest_data.append('squashing layers')
225
226
worker['caches'] = [{
227
'type': 'persistent',
228
'name': cache_name,
229
'mount-point': '/builds/worker/checkouts',
230
}]
231
232
for k, v in args.items():
233
if k == 'DOCKER_IMAGE_PACKAGES':
234
worker['env'][k] = {'task-reference': v}
235
else:
236
worker['env'][k] = v
237
238
if packages:
239
deps = taskdesc.setdefault('dependencies', {})
240
for p in sorted(packages):
241
deps[p] = 'packages-{}'.format(p)
242
243
if parent:
244
deps = taskdesc.setdefault('dependencies', {})
245
deps[parent] = 'build-docker-image-{}'.format(parent)
246
worker['env']['DOCKER_IMAGE_PARENT_TASK'] = {
247
'task-reference': '<{}>'.format(parent),
248
}
249
if 'index' in task:
250
taskdesc['index'] = task['index']
251
252
if task.get('cache', True) and not taskgraph.fast:
253
taskdesc['cache'] = {
254
'type': 'docker-images.v2',
255
'name': image_name,
256
'digest-data': digest_data,
257
}
258
259
yield taskdesc