Revision control

Copy as Markdown

from copy import deepcopy
import itertools
from taskgraph import MAX_DEPENDENCIES
from taskgraph.transforms.base import TransformSequence
transforms = TransformSequence()
alerts = TransformSequence()
def deps_complete_script(config, tasks):
"""Setup the script"""
for task in tasks:
# Run this task when all dependencies are completed, rather than
# requiring them to be successful
'requires': 'all-resolved',
'worker-type': 'b-linux',
'worker': {
'chain-of-trust': True,
'docker-image': { 'in-tree': 'linux' },
'max-run-time': 1800,
'env': {
'task-reference': '<decision>'
'TASK_ID': {
'task-reference': '<self>'
'run': {
'using': 'run-task',
'command': '/builds/worker/checkouts/vcs/taskcluster/scripts/',
yield task
def convert_dependencies(config, tasks):
Convert dependencies into soft-dependencies
This means that taskcluster won't schedule the dependencies if only this
task depends on them.
for task in tasks:
task.setdefault("soft-dependencies", [])
task["soft-dependencies"] += [
for dep_task in config.kind_dependencies_tasks.values()
yield task
def add_alert_routes(config, tasks):
Add routes to alert channels when this task fails.
for task in tasks:
alerts = task.pop("alerts", {})
if config.params["level"] != "3":
yield task
task.setdefault('routes', [])
for name, value in alerts.items():
if name not in ("slack-channel", "email", "pulse", "matrix-room"):
raise KeyError("Unknown alert type: {}".format(name))
task['routes'].append("notify.{}.{}.on-failed".format(name, value))
yield task
# Transform that adjusts the dependencies to not exceed MAX_DEPENDENCIES
# This transform checks if the dependency count exceeds MAX_DEPENDENCIES. If
# so, it creates a child jobs with exactly MAX_DEPENDENCIES that the main task
# can then depend on.
# This is separated out from the main transform since it depends on
# running first.
# This code is based off the reverse_chunk_deps transform from Gecko
reverse_chunk = TransformSequence()
def adjust_dependencies_child_job(orig_job, deps, count):
job = deepcopy(orig_job)
job["soft-dependencies"] = deps
job["label"] = "{} - {}".format(orig_job["label"], count)
del job["routes"] # don't send alerts for child jobs
return job
def adjust_dependencies(config, jobs):
for job in jobs:
counter = itertools.count(1)
# sort for deterministic chunking
deps = sorted(job["soft-dependencies"])
while len(deps) > MAX_DEPENDENCIES:
# split off the first N deps
chunk, deps = deps[:MAX_DEPENDENCIES], deps[MAX_DEPENDENCIES:]
# create a separate job to handle them
child_job = adjust_dependencies_child_job(job, chunk, next(counter))
# Yield the child job and add a dependency to it
yield child_job
# Yield the parent job with the rest of the deps
job["soft-dependencies"] = deps
yield job