Source code

Revision control

Other Tools

1
#!/usr/bin/env python3
2
# This Source Code Form is subject to the terms of the Mozilla Public
3
# License, v. 2.0. If a copy of the MPL was not distributed with this
4
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
5
6
from __future__ import absolute_import, division, print_function
7
8
import asyncio
9
import aiohttp
10
import configparser
11
import argparse
12
import hashlib
13
import json
14
import logging
15
import os
16
import shutil
17
import tempfile
18
import requests
19
from distutils.util import strtobool
20
21
import redo
22
from scriptworker.utils import retry_async
23
from mardor.reader import MarReader
24
from mardor.signing import get_keysize
25
26
27
log = logging.getLogger(__name__)
28
29
30
ROOT_URL = os.environ['TASKCLUSTER_ROOT_URL']
31
QUEUE_PREFIX = ("https://queue.taskcluster.net/"
32
if ROOT_URL == 'https://taskcluster.net'
33
else ROOT_URL + '/api/queue/')
34
ALLOWED_URL_PREFIXES = (
42
QUEUE_PREFIX,
43
)
44
STAGING_URL_PREFIXES = (
47
)
48
49
DEFAULT_FILENAME_TEMPLATE = "{appName}-{branch}-{version}-{platform}-" \
50
"{locale}-{from_buildid}-{to_buildid}.partial.mar"
51
52
BCJ_OPTIONS = {
53
'x86': ['--x86'],
54
'x86_64': ['--x86'],
55
'aarch64': [],
56
}
57
58
59
def verify_signature(mar, certs):
60
log.info("Checking %s signature", mar)
61
with open(mar, 'rb') as mar_fh:
62
m = MarReader(mar_fh)
63
m.verify(verify_key=certs.get(m.signature_type))
64
65
66
def is_lzma_compressed_mar(mar):
67
log.info("Checking %s for lzma compression", mar)
68
result = MarReader(open(mar, 'rb')).compression_type == 'xz'
69
if result:
70
log.info("%s is lzma compressed", mar)
71
else:
72
log.info("%s is not lzma compressed", mar)
73
return result
74
75
76
def validate_mar_channel_id(mar, channel_ids):
77
log.info("Checking %s for MAR_CHANNEL_ID %s", mar, channel_ids)
78
# We may get a string with a list representation, or a single entry string.
79
channel_ids = set(channel_ids.split(','))
80
81
product_info = MarReader(open(mar, 'rb')).productinfo
82
if not isinstance(product_info, tuple):
83
raise ValueError("Malformed product information in mar: {}".format(product_info))
84
85
found_channel_ids = set(product_info[1].split(','))
86
87
if not found_channel_ids.issubset(channel_ids):
88
raise ValueError("MAR_CHANNEL_ID mismatch, {} not in {}".format(
89
product_info[1], channel_ids))
90
91
log.info("%s channel %s in %s", mar, product_info[1], channel_ids)
92
93
94
@redo.retriable()
95
def get_secret(secret_name):
97
log.debug("Fetching {}".format(secret_name))
98
r = requests.get(secrets_url.format(secret_name))
99
# 403: If unauthorized, just give up.
100
if r.status_code == 403:
101
log.info("Unable to get secret key")
102
return {}
103
r.raise_for_status()
104
return r.json().get('secret', {})
105
106
107
async def retry_download(*args, **kwargs): # noqa: E999
108
"""Retry download() calls."""
109
await retry_async(
110
download,
111
retry_exceptions=(
112
aiohttp.ClientError,
113
asyncio.TimeoutError
114
),
115
args=args,
116
kwargs=kwargs
117
)
118
119
120
async def download(url, dest, mode=None): # noqa: E999
121
log.info("Downloading %s to %s", url, dest)
122
chunk_size = 4096
123
bytes_downloaded = 0
124
async with aiohttp.ClientSession(raise_for_status=True) as session:
125
async with session.get(url, timeout=120) as resp:
126
# Additional early logging for download timeouts.
127
log.debug("Fetching from url %s", resp.url)
128
for history in resp.history:
129
log.debug("Redirection history: %s", history.url)
130
if 'Content-Length' in resp.headers:
131
log.debug('Content-Length expected for %s: %s',
132
url, resp.headers['Content-Length'])
133
log_interval = chunk_size * 1024
134
with open(dest, 'wb') as fd:
135
while True:
136
chunk = await resp.content.read(chunk_size)
137
if not chunk:
138
break
139
fd.write(chunk)
140
bytes_downloaded += len(chunk)
141
log_interval -= len(chunk)
142
if log_interval <= 0:
143
log.debug("Bytes downloaded for %s: %d", url, bytes_downloaded)
144
log_interval = chunk_size * 1024
145
146
log.debug('Downloaded %s bytes', bytes_downloaded)
147
if mode:
148
log.debug("chmod %o %s", mode, dest)
149
os.chmod(dest, mode)
150
151
152
async def run_command(cmd, cwd='/', env=None, label=None, silent=False):
153
if not env:
154
env = dict()
155
process = await asyncio.create_subprocess_shell(cmd,
156
stdout=asyncio.subprocess.PIPE,
157
stderr=asyncio.subprocess.PIPE,
158
cwd=cwd, env=env)
159
if label:
160
label = "{}: ".format(label)
161
else:
162
label = ""
163
164
async def read_output(stream, label, printcmd):
165
while True:
166
line = await stream.readline()
167
if line == b'':
168
break
169
printcmd("%s%s", label, line.decode('utf-8').rstrip())
170
171
if silent:
172
await process.wait()
173
else:
174
await asyncio.gather(
175
read_output(process.stdout, label, log.info),
176
read_output(process.stderr, label, log.warn)
177
)
178
await process.wait()
179
180
181
async def unpack(work_env, mar, dest_dir):
182
os.mkdir(dest_dir)
183
log.debug("Unwrapping %s", mar)
184
env = work_env.env
185
if not is_lzma_compressed_mar(mar):
186
env['MAR_OLD_FORMAT'] = '1'
187
elif 'MAR_OLD_FORMAT' in env:
188
del env['MAR_OLD_FORMAT']
189
190
cmd = "{} {}".format(work_env.paths['unwrap_full_update.pl'], mar)
191
await run_command(cmd, cwd=dest_dir, env=env, label=dest_dir)
192
193
194
def find_file(directory, filename):
195
log.debug("Searching for %s in %s", filename, directory)
196
for root, _, files in os.walk(directory):
197
if filename in files:
198
f = os.path.join(root, filename)
199
log.debug("Found %s", f)
200
return f
201
202
203
def get_option(directory, filename, section, option):
204
log.debug("Extracting [%s]: %s from %s/**/%s", section, option, directory,
205
filename)
206
f = find_file(directory, filename)
207
config = configparser.ConfigParser()
208
config.read(f)
209
rv = config.get(section, option)
210
log.debug("Found %s", rv)
211
return rv
212
213
214
async def generate_partial(work_env, from_dir, to_dir, dest_mar, mar_data,
215
use_old_format):
216
log.info("Generating partial %s", dest_mar)
217
env = work_env.env
218
env["MOZ_PRODUCT_VERSION"] = mar_data['version']
219
env["MAR_CHANNEL_ID"] = mar_data["MAR_CHANNEL_ID"]
220
env['BRANCH'] = mar_data['branch']
221
env['PLATFORM'] = mar_data['platform']
222
if use_old_format:
223
env['MAR_OLD_FORMAT'] = '1'
224
elif 'MAR_OLD_FORMAT' in env:
225
del env['MAR_OLD_FORMAT']
226
make_incremental_update = os.path.join(work_env.workdir,
227
"make_incremental_update.sh")
228
cmd = " ".join([make_incremental_update, dest_mar, from_dir, to_dir])
229
230
await run_command(cmd, cwd=work_env.workdir, env=env, label=dest_mar.split('/')[-1])
231
validate_mar_channel_id(dest_mar, mar_data["MAR_CHANNEL_ID"])
232
233
234
def get_hash(path, hash_type="sha512"):
235
h = hashlib.new(hash_type)
236
with open(path, "rb") as f:
237
h.update(f.read())
238
return h.hexdigest()
239
240
241
class WorkEnv(object):
242
def __init__(self, allowed_url_prefixes, mar=None, mbsdiff=None, arch=None):
243
self.paths = dict()
244
self.urls = {
246
'tools/update-packaging/unwrap_full_update.pl',
248
'latest-mozilla-central/mar-tools/linux64/mar',
250
'latest-mozilla-central/mar-tools/linux64/mbsdiff'
251
}
252
self.allowed_url_prefixes = allowed_url_prefixes
253
if mar:
254
self.urls['mar'] = mar
255
if mbsdiff:
256
self.urls['mbsdiff'] = mbsdiff
257
self.arch = arch
258
259
async def setup(self, mar=None, mbsdiff=None):
260
self.workdir = tempfile.mkdtemp()
261
for filename, url in self.urls.items():
262
if filename in self.paths:
263
os.unlink(self.paths[filename])
264
self.paths[filename] = os.path.join(self.workdir, filename)
265
await retry_download(url, dest=self.paths[filename], mode=0o755)
266
267
async def download_buildsystem_bits(self, repo, revision):
268
prefix = "{repo}/raw-file/{revision}/tools/update-packaging"
269
prefix = prefix.format(repo=repo, revision=revision)
270
for f in ('make_incremental_update.sh', 'common.sh'):
271
url = "{prefix}/{f}".format(prefix=prefix, f=f)
272
await retry_download(url, dest=os.path.join(self.workdir, f), mode=0o755)
273
274
def cleanup(self):
275
shutil.rmtree(self.workdir)
276
277
@property
278
def env(self):
279
my_env = os.environ.copy()
280
my_env['LC_ALL'] = 'C'
281
my_env['MAR'] = self.paths['mar']
282
my_env['MBSDIFF'] = self.paths['mbsdiff']
283
if self.arch:
284
my_env['BCJ_OPTIONS'] = ' '.join(BCJ_OPTIONS[self.arch])
285
return my_env
286
287
288
def verify_allowed_url(mar, allowed_url_prefixes):
289
if not any(mar.startswith(prefix) for prefix in allowed_url_prefixes):
290
raise ValueError("{mar} is not in allowed URL prefixes: {p}".format(
291
mar=mar, p=allowed_url_prefixes
292
))
293
294
295
async def manage_partial(partial_def, filename_template, artifacts_dir,
296
allowed_url_prefixes, signing_certs, arch=None):
297
"""Manage the creation of partial mars based on payload."""
298
299
work_env = WorkEnv(
300
allowed_url_prefixes=allowed_url_prefixes,
301
mar=partial_def.get('mar_binary'),
302
mbsdiff=partial_def.get('mbsdiff_binary'),
303
arch=arch,
304
)
305
await work_env.setup()
306
307
for mar in (partial_def["from_mar"], partial_def["to_mar"]):
308
verify_allowed_url(mar, allowed_url_prefixes)
309
310
complete_mars = {}
311
use_old_format = False
312
check_channels_in_files = list()
313
for mar_type, f in (("from", partial_def["from_mar"]), ("to", partial_def["to_mar"])):
314
dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type))
315
unpack_dir = os.path.join(work_env.workdir, mar_type)
316
317
await retry_download(f, dest)
318
319
if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"):
320
verify_signature(dest, signing_certs)
321
322
complete_mars["%s_size" % mar_type] = os.path.getsize(dest)
323
complete_mars["%s_hash" % mar_type] = get_hash(dest)
324
325
await unpack(work_env, dest, unpack_dir)
326
327
if mar_type == 'to':
328
check_channels_in_files.append(dest)
329
330
if mar_type == 'from':
331
version = get_option(unpack_dir, filename="application.ini",
332
section="App", option="Version")
333
major = int(version.split(".")[0])
334
# The updater for versions less than 56.0 requires BZ2
335
# compressed MAR files
336
if major < 56:
337
use_old_format = True
338
log.info("Forcing BZ2 compression for %s", f)
339
340
log.info("Done.")
341
342
to_path = os.path.join(work_env.workdir, "to")
343
from_path = os.path.join(work_env.workdir, "from")
344
345
mar_data = {
346
"MAR_CHANNEL_ID": os.environ["MAR_CHANNEL_ID"],
347
"version": get_option(to_path, filename="application.ini",
348
section="App", option="Version"),
349
"to_buildid": get_option(to_path, filename="application.ini",
350
section="App", option="BuildID"),
351
"from_buildid": get_option(from_path, filename="application.ini",
352
section="App", option="BuildID"),
353
"appName": get_option(from_path, filename="application.ini",
354
section="App", option="Name"),
355
# Use Gecko repo and rev from platform.ini, not application.ini
356
"repo": get_option(to_path, filename="platform.ini", section="Build",
357
option="SourceRepository"),
358
"revision": get_option(to_path, filename="platform.ini",
359
section="Build", option="SourceStamp"),
360
"from_mar": partial_def["from_mar"],
361
"to_mar": partial_def["to_mar"],
362
"platform": partial_def["platform"],
363
"locale": partial_def["locale"],
364
}
365
366
for filename in check_channels_in_files:
367
validate_mar_channel_id(filename, mar_data["MAR_CHANNEL_ID"])
368
369
for field in ("update_number", "previousVersion", "previousBuildNumber",
370
"toVersion", "toBuildNumber"):
371
if field in partial_def:
372
mar_data[field] = partial_def[field]
373
mar_data.update(complete_mars)
374
375
# if branch not set explicitly use repo-name
376
mar_data['branch'] = partial_def.get('branch', mar_data['repo'].rstrip('/').split('/')[-1])
377
378
if 'dest_mar' in partial_def:
379
mar_name = partial_def['dest_mar']
380
else:
381
# default to formatted name if not specified
382
mar_name = filename_template.format(**mar_data)
383
384
mar_data['mar'] = mar_name
385
dest_mar = os.path.join(work_env.workdir, mar_name)
386
387
# TODO: download these once
388
await work_env.download_buildsystem_bits(repo=mar_data["repo"],
389
revision=mar_data["revision"])
390
391
await generate_partial(work_env, from_path, to_path, dest_mar,
392
mar_data, use_old_format)
393
394
mar_data["size"] = os.path.getsize(dest_mar)
395
396
mar_data["hash"] = get_hash(dest_mar)
397
398
shutil.copy(dest_mar, artifacts_dir)
399
work_env.cleanup()
400
401
return mar_data
402
403
404
async def async_main(args, signing_certs):
405
tasks = []
406
407
allowed_url_prefixes = list(ALLOWED_URL_PREFIXES)
408
if args.allow_staging_prefixes:
409
allowed_url_prefixes += STAGING_URL_PREFIXES
410
411
task = json.load(args.task_definition)
412
# TODO: verify task["extra"]["funsize"]["partials"] with jsonschema
413
for definition in task["extra"]["funsize"]["partials"]:
414
tasks.append(asyncio.ensure_future(retry_async(
415
manage_partial,
416
retry_exceptions=(
417
aiohttp.ClientError,
418
asyncio.TimeoutError
419
),
420
kwargs=dict(
421
partial_def=definition,
422
filename_template=args.filename_template,
423
artifacts_dir=args.artifacts_dir,
424
allowed_url_prefixes=allowed_url_prefixes,
425
signing_certs=signing_certs,
426
arch=args.arch
427
))))
428
manifest = await asyncio.gather(*tasks)
429
return manifest
430
431
432
def main():
433
434
parser = argparse.ArgumentParser()
435
parser.add_argument("--artifacts-dir", required=True)
436
parser.add_argument("--sha1-signing-cert", required=True)
437
parser.add_argument("--sha384-signing-cert", required=True)
438
parser.add_argument("--task-definition", required=True,
439
type=argparse.FileType('r'))
440
parser.add_argument("--allow-staging-prefixes",
441
action="store_true",
442
default=strtobool(
443
os.environ.get('FUNSIZE_ALLOW_STAGING_PREFIXES', "false")),
444
help="Allow files from staging buckets.")
445
parser.add_argument("--filename-template",
446
default=DEFAULT_FILENAME_TEMPLATE)
447
parser.add_argument("-q", "--quiet", dest="log_level",
448
action="store_const", const=logging.WARNING,
449
default=logging.DEBUG)
450
parser.add_argument('--arch', type=str, required=True,
451
choices=BCJ_OPTIONS.keys(),
452
help='The archtecture you are building.')
453
args = parser.parse_args()
454
455
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
456
log.setLevel(args.log_level)
457
458
signing_certs = {
459
'sha1': open(args.sha1_signing_cert, 'rb').read(),
460
'sha384': open(args.sha384_signing_cert, 'rb').read(),
461
}
462
463
assert(get_keysize(signing_certs['sha1']) == 2048)
464
assert(get_keysize(signing_certs['sha384']) == 4096)
465
466
loop = asyncio.get_event_loop()
467
manifest = loop.run_until_complete(async_main(args, signing_certs))
468
loop.close()
469
470
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
471
with open(manifest_file, "w") as fp:
472
json.dump(manifest, fp, indent=2, sort_keys=True)
473
474
log.debug("{}".format(json.dumps(manifest, indent=2, sort_keys=True)))
475
476
477
if __name__ == '__main__':
478
main()