Revision control
Copy as Markdown
Other Tools
# yamllint disable rule:line-length
# This file is rendered via JSON-e by
# - hg-push - https://github.com/mozilla-releng/fxci-config/blob/main/build-decision/src/build_decision/hg_push.py
# {
# tasks_for: 'hg-push',
# push: {owner, comment, pushlog_id, pushdate},
# repository: {url, project, level},
# now,
# as_slugid: // function
# ownTaskId: // taskId of the task that will be created
# }
#
# - cron tasks - https://github.com/mozilla-releng/fxci-config/blob/main/build-decision/src/build_decision/cron/decision.py
# {
# tasks_for: 'cron',
# push: {revision, pushlog_id, pushdate, owner}
# repository: {url, project, level},
# cron: {task_id, job_name, job_symbol, quoted_args},
# now,
# ownTaskId: // taskId of the task that will be created
# }
#
# - action tasks - See:
# * taskcluster/gecko_taskgraph/actions/registry.py,
#
# The registry generates the hookPayload that appears in actions.json, and
# contains data from the decision task as well as JSON-e code to combine that
# with data supplied as part of the action spec. When the hook is fired, the
# hookPayload is rendered with JSON-e to produce a payload for the hook task
# template.
#
# The ci-admin code wraps the content of this file (.taskcluster.yml) with a
# JSON-e $let statement that produces the context described below, and
# installs that as the hook task template.
#
# {
# tasks_for: 'action',
# push: {owner, pushlog_id, revision, base_revision},
# repository: {url, project, level},
# input,
# taskId, // targetted taskId
# taskGroupId, // targetted taskGroupId
# action: {name, title, description, taskGroupId, symbol, repo_scope, cb_name}
# ownTaskId: // taskId of the task that will be created
# clientId: // clientId that triggered this hook
# }
---
version: 1
reporting: checks-v1
policy:
pullRequests: collaborators
tasks:
# NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering
# takes place, and only the first task is included in the `in_tree_action` hook.
- $switch:
# Decision task for events originating from hg.mozilla.org
'tasks_for in ["hg-push", "cron"] || (tasks_for == "action" && parameters["repository_type"] == "hg")':
$let:
# sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
# ensure there's no trailing `/` on the repo URL
repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
# expire try earlier than other branches
expires:
$if: 'repository.project == "try"'
then: {$fromNow: '28 days'}
else: {$fromNow: '1 year'}
trustDomain: gecko
treeherder_link: '[Treeherder job](https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}&selectedTaskRun=${ownTaskId})'
in:
taskId: {$if: 'tasks_for != "action"', then: '${ownTaskId}'}
taskGroupId:
$if: 'tasks_for == "action"'
then:
'${action.taskGroupId}'
else:
'${ownTaskId}' # same as taskId; this is how automation identifies a decision task
schedulerId: '${trustDomain}-level-${repository.level}'
created: {$fromNow: ''}
deadline: {$fromNow: '1 day'}
expires: {$eval: 'expires'}
metadata:
$merge:
- owner: "${ownerEmail}"
source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
- $if: 'tasks_for == "hg-push"'
then:
name: "Gecko Decision Task"
description: 'The task that creates all of the other tasks in the task graph (${treeherder_link})'
else:
$if: 'tasks_for == "action"'
then:
name: "Action: ${action.title}"
description: |
${action.description}
${treeherder_link}
Action triggered by clientID `${clientId}`
else:
name: "Decision Task for cron job ${cron.job_name}"
description: 'Created by a [cron task](https://firefox-ci-tc.services.mozilla.com/tasks/${cron.task_id}) (${treeherder_link})'
provisionerId: "${trustDomain}-${repository.level}"
workerType: "decision"
tags:
$merge:
- project: ${repository.project}
trust-domain: ${trustDomain}
worker-implementation: docker-worker
- $if: 'tasks_for == "hg-push"'
then:
createdForUser: "${ownerEmail}"
kind: decision-task
else:
$if: 'tasks_for == "action"'
then:
createdForUser: '${ownerEmail}'
kind: 'action-callback'
else:
$if: 'tasks_for == "cron"'
then:
kind: cron-task
routes:
$flattenDeep:
- "tc-treeherder.v2.${repository.project}.${push.revision}"
- $if: 'tasks_for == "hg-push"'
then:
- "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision"
- "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"
- "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
- "notify.email.${ownerEmail}.on-failed"
- "notify.email.${ownerEmail}.on-exception"
# Send a notification email if the push comes from try
- $if: 'repository.project == "try"'
then:
- "notify.email.${ownerEmail}.on-completed"
- $if: 'repository.project == "mozilla-central"'
then:
# Notify #thunderbird-ci
- "notify.matrix-room.!TWztIhgqLawNpRBZTC:mozilla.org.on-completed"
else:
$if: 'tasks_for == "action"'
then:
- "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.actions.${ownTaskId}"
- "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.${ownTaskId}"
else: # cron
- "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}"
- "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision-${cron.job_name}"
- "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision-${cron.job_name}"
# list each cron task on this revision, so actions can find them
- 'index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.cron.${ownTaskId}'
- $if: 'repository.project != "try"'
then:
- "notify.email.ciduty+failedcron@mozilla.com.on-failed"
- "notify.email.ciduty+exceptioncron@mozilla.com.on-exception"
- "notify.email.sheriffs+failedcron@mozilla.org.on-failed"
- "notify.email.sheriffs+exceptioncron@mozilla.org.on-exception"
scopes:
$if: 'tasks_for == "hg-push"'
then:
- 'assume:repo:${repoUrl[8:]}:branch:default'
- 'queue:route:notify.email.${ownerEmail}.*'
- 'in-tree:hook-action:project-${trustDomain}/in-tree-action-${repository.level}-*'
- 'index:insert-task:${trustDomain}.v2.${repository.project}.*'
else:
$if: 'tasks_for == "action"'
then:
# when all actions are hooks, we can calculate this directly rather than using a variable
- '${action.repo_scope}'
else:
- 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
dependencies: []
requires: all-completed
priority:
# There are (typically) 3 types of decision tasks that can run:
# cron, action, and on-push. We always treat cron as highest priority
# because it schedules things like nightlies, releases, etc. and are
# fairly few and far between.
#
# After that comes on-push decision tasks. These are in the critical
# path of all tasks the push will run, and are critical to run quickly
# to get quick results.
#
# Finally, action tasks are used to add tasks to a push, rerun tasks,
# start release automation, and other things. They are sometimes
# very numerous, and are treated as lowest priority to avoid interfering
# with fast scheduling of regular push tasks.
#
# SCM levels all use different workerTypes, so there is no need for priority
# between levels; "low" is the highest priority available at all levels, and
# nothing runs at any higher priority on these workerTypes.
$if: "tasks_for == 'cron'"
then: low
else:
$if: "tasks_for == 'hg-push'"
then: very-low
else: lowest # tasks_for == 'action'
retries:
$if: "tasks_for == 'hg-push' && repository.level != '1'"
then: 0
else: 5
payload:
env:
# run-task uses these to check out the source; the inputs
# to `mach taskgraph decision` are all on the command line.
$merge:
GECKO_BASE_REV: '${push.base_revision}'
GECKO_HEAD_REPOSITORY: '${repoUrl}'
GECKO_HEAD_REF: '${push.revision}'
GECKO_HEAD_REV: '${push.revision}'
HG_STORE_PATH: /builds/worker/checkouts/hg-store
TASKCLUSTER_CACHES: /builds/worker/checkouts
TASKCLUSTER_VOLUMES: /builds/worker/artifacts
MOZ_UPLOAD_DIR: /builds/worker/artifacts
MOZ_AUTOMATION: '1'
# mach generates pyc files when reading `mach_commands.py`
# This causes cached_task digest generation to be random for
# some tasks. Disable bytecode generation to work around that.
PYTHONDONTWRITEBYTECODE: '1'
- $if: 'tasks_for == "action"'
then:
ACTION_TASK_GROUP_ID: '${action.taskGroupId}' # taskGroupId of the target task
ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded)
ACTION_INPUT: {$json: {$eval: 'input'}}
ACTION_CALLBACK: '${action.cb_name}'
cache:
"${trustDomain}-level-${repository.level}-checkouts-sparse-v4": /builds/worker/checkouts
features:
taskclusterProxy: true
chainOfTrust: true
# Note: This task is built server side without the context or tooling that
# exist in tree so we must hard code the hash
image: 'mozillareleases/gecko_decision:5.1.0@sha256:8879ecb7f859e1001b0aa67de440a18c3a896446bfd0e2e87688bb367e590667'
maxRunTime: 3600
command:
- /builds/worker/bin/run-task-hg
- '--gecko-checkout=/builds/worker/checkouts/gecko'
- '--gecko-sparse-profile=build/sparse-profiles/taskgraph'
- '--'
- bash
- -cx
- $let:
extraArgs:
$if: 'tasks_for == "cron"'
then: '${cron.quoted_args}'
else:
$if: 'repository.project in ["autoland", "try"]'
then: '--no-verify'
else: ''
in:
$if: 'tasks_for == "action"'
then: >
cd /builds/worker/checkouts/gecko &&
ln -s /builds/worker/artifacts artifacts &&
./mach --log-no-times taskgraph action-callback
else: >
cd /builds/worker/checkouts/gecko &&
ln -s /builds/worker/artifacts artifacts &&
./mach --log-no-times taskgraph decision
--pushlog-id='${push.pushlog_id}'
--pushdate='${push.pushdate}'
--project='${repository.project}'
--owner='${ownerEmail}'
--level='${repository.level}'
--tasks-for='${tasks_for}'
--repository-type=hg
--base-repository="$GECKO_BASE_REPOSITORY"
--base-rev="$GECKO_BASE_REV"
--head-repository="$GECKO_HEAD_REPOSITORY"
--head-ref="$GECKO_HEAD_REF"
--head-rev="$GECKO_HEAD_REV"
${extraArgs}
artifacts:
'public':
type: 'directory'
path: '/builds/worker/artifacts'
expires: {$eval: expires}
'public/docker-contexts':
type: 'directory'
path: '/builds/worker/checkouts/gecko/docker-contexts'
# This needs to be at least the deadline of the
# decision task + the docker-image task deadlines.
# It is set to a week to allow for some time for
# debugging, but they are not useful long-term.
expires: {$fromNow: '7 day'}
extra:
$merge:
- treeherder:
$merge:
- machine:
platform: gecko-decision
- $if: 'tasks_for == "hg-push"'
then:
symbol: D
else:
$if: 'tasks_for == "action"'
then:
groupName: 'action-callback'
groupSymbol: AC
symbol: "${action.symbol}"
else:
groupSymbol: cron
symbol: "${cron.job_symbol}"
- $if: 'tasks_for == "action"'
then:
parent: '${action.taskGroupId}'
action:
name: '${action.name}'
context:
taskGroupId: '${action.taskGroupId}'
taskId: {$eval: 'taskId'}
input: {$eval: 'input'}
clientId: {$eval: 'clientId'}
- $if: 'tasks_for == "cron"'
then:
cron: {$json: {$eval: 'cron'}}
- tasks_for: '${tasks_for}'
# Email for all pushes should link to treeherder
- $if: 'tasks_for == "hg-push"'
then:
notify:
$merge:
- email:
$merge:
- link:
text: "Treeherder Jobs"
- $if: 'repository.project == "try"'
then:
subject: "Thank you for your try submission of ${push.revision}. It's the best!"
content: "Your try push has been submitted. It's the best! Use the link to view the status of your jobs."
- $if: 'repository.project == "mozilla-central"'
then:
matrixBody: "${repository.project} push notification: https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}"
# Decision task for events originating from Github
'tasks_for[:6] == "github" || (tasks_for in ["action", "pr-action"] && parameters["repository_type"] == "git")':
$let:
$merge:
- trustDomain: gecko
isPullRequest: false
eventType: '${tasks_for}'
eventAction: null
- $switch:
'tasks_for[:6] == "github"':
ownTaskId: {$eval: as_slugid("decision_task")}
eventType: '${tasks_for[7:]}' # strip out 'github-'
eventAction: '${event["action"]}' # empty string if 'action' doesn't exist
'tasks_for in ["action", "pr-action"]':
ownTaskId: '${ownTaskId}'
ownerEmail: '${tasks_for}@noreply.mozilla.org'
baseRepoUrl: '${repository.url}'
repoUrl: '${repository.url}'
project: '${repository.project}'
# Prepending 'refs/heads' ensures `run-task` won't attempt to fetch tags.
ref: "refs/heads/${push.branch}"
baseRev: '${push.revision}'
headRev: '${push.revision}'
- $switch:
'tasks_for == "github-push"':
ownerEmail: '${event.pusher.email}'
baseRepoUrl: '${event.repository.html_url}'
repoUrl: '${event.repository.html_url}'
project: '${event.repository.name}'
ref: '${event.ref}'
baseRev: '${event.before}'
headRev: '${event.after}'
'tasks_for[:19] == "github-pull-request"':
ownerEmail: '${event.pull_request.user.login}@users.noreply.github.com'
baseRepoUrl: '${event.pull_request.base.repo.html_url}'
repoUrl: '${event.pull_request.head.repo.html_url}'
project: '${event.pull_request.base.repo.name}'
# Normalize 'ref' to include 'refs/heads' as pull-requests must be associated
# with a head ref. The benefit of doing this is that `run-task` won't attempt
# to fetch all tags from the repo just on the off-chance it is a tag.
ref: 'refs/heads/${event.pull_request.head.ref}'
baseRev: '${event.pull_request.base.sha}'
headRev: '${event.pull_request.head.sha}'
isPullRequest: true
'tasks_for == "pr-action"':
baseRepoUrl: '${repository.base_url}'
eventType: action
in:
$let:
shortRef:
$if: 'ref[:11] == "refs/heads/"'
then: {$eval: 'ref[11:]'}
else: ${ref}
in:
$if: >
eventType == "action"
|| (eventType == "push" && shortRef == "main")
|| (isPullRequest && eventAction in ["opened", "reopened", "synchronize"])
then:
$let:
level:
$if: '(tasks_for == "action" || eventType == "push") && repoUrl == "https://github.com/mozilla-firefox/firefox" && shortRef == "main"'
then: 3
else: 1
in:
taskId: {$if: 'eventType != "action"', then: '${ownTaskId}'}
taskGroupId: {$if: 'eventType == "action"', then: '${action.taskGroupId}', else: '${ownTaskId}'}
schedulerId: '${trustDomain}-level-${level}'
created: {$fromNow: ''}
deadline: {$fromNow: '1 day'}
expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first
metadata:
$merge:
- owner: "${ownerEmail}"
source: "${repoUrl}/raw/${headRev}/.taskcluster.yml"
- $switch:
'tasks_for == "action"':
name: "Action: ${action.title}"
description: |
${action.description}
Action triggered by clientID `${clientId}`
'tasks_for == "pr-action"':
name: "PR action: ${action.title}"
description: |
${action.description}
PR action triggered by clientID `${clientId}`
$default:
name: "Decision Task (${eventType})"
description: 'The task that creates all of the other tasks in the task graph'
provisionerId: "${trustDomain}-${level}"
workerType: "decision"
tags:
$merge:
- createdForUser: "${ownerEmail}"
project: ${project}
trust-domain: ${trustDomain}
worker-implementation: docker-worker
- $switch:
'eventType == "action"':
kind: action-callback
$default:
kind: decision-task
routes:
$flattenDeep:
- checks
- $switch:
'eventType == "push"':
- "tc-treeherder.v2.${project}.${headRev}"
- "index.${trustDomain}.v2.${project}.latest.taskgraph.decision"
- "index.${trustDomain}.v2.${project}.revision.${headRev}.taskgraph.decision"
'tasks_for == "action"':
- "tc-treeherder.v2.${project}.${headRev}"
- "index.${trustDomain}.v2.${project}.revision.${headRev}.taskgraph.actions.${ownTaskId}"
scopes:
$switch:
isPullRequest:
- 'assume:repo:${baseRepoUrl[8:]}:${eventType}'
'eventType == "push"':
- 'assume:repo:${repoUrl[8:]}:branch:${shortRef}'
'eventType == "action"':
- 'assume:repo:${baseRepoUrl[8:]}:${tasks_for}:${action.action_perm}'
dependencies: []
requires: all-completed
priority:
# There are (typically) 3 types of decision tasks that can run:
# cron, action, and on-push. We always treat cron as highest priority
# because it schedules things like nightlies, releases, etc. and are
# fairly few and far between.
#
# After that comes on-push decision tasks. These are in the critical
# path of all tasks the push will run, and are critical to run quickly
# to get quick results.
#
# Finally, action tasks are used to add tasks to a push, rerun tasks,
# start release automation, and other things. They are sometimes
# very numerous, and are treated as lowest priority to avoid interfering
# with fast scheduling of regular push tasks.
#
# SCM levels all use different workerTypes, so there is no need for priority
# between levels; "low" is the highest priority available at all levels, and
# nothing runs at any higher priority on these workerTypes.
$if: "tasks_for == 'cron'"
then: low
else:
$if: "tasks_for == 'github-push'"
then: very-low
else: lowest # tasks_for == 'action'
retries: 5
payload:
env:
$merge:
- GECKO_BASE_REPOSITORY: '${baseRepoUrl}'
GECKO_BASE_REV: '${baseRev}'
GECKO_HEAD_REPOSITORY: '${repoUrl}'
GECKO_HEAD_REF: '${ref}'
GECKO_HEAD_REV: '${headRev}'
GECKO_REPOSITORY_TYPE: git
REPOSITORIES: {$json: {gecko: "Mozilla Firefox"}}
TASKCLUSTER_CACHES: /builds/worker/checkouts
TASKCLUSTER_VOLUMES: /builds/worker/artifacts
MOZ_UPLOAD_DIR: /builds/worker/artifacts
MOZ_AUTOMATION: '1'
# mach generates pyc files when reading `mach_commands.py`
# This causes cached_task digest generation to be random for
# some tasks. Disable bytecode generation to work around that.
PYTHONDONTWRITEBYTECODE: '1'
MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: 'system'
- $if: 'eventType == "action"'
then:
ACTION_TASK_GROUP_ID: '${action.taskGroupId}'
ACTION_TASK_ID: {$json: {$eval: 'taskId'}}
ACTION_INPUT: {$json: {$eval: 'input'}}
ACTION_CALLBACK: '${action.cb_name}'
cache:
"${trustDomain}-level-${level}-${project}-checkouts-git-shallow-v1": /builds/worker/checkouts
features:
taskclusterProxy: true
chainOfTrust: true
image: mozillareleases/taskgraph:run-task-latest
maxRunTime: 1800
command:
- run-task
- '--gecko-checkout=/builds/worker/checkouts/gecko'
- '--gecko-shallow-clone'
- '--'
- bash
- -cx
- $if: 'eventType == "action"'
then: >
cd /builds/worker/checkouts/gecko &&
ln -s /builds/worker/artifacts artifacts &&
./mach --log-no-times taskgraph action-callback
else: >
cd /builds/worker/checkouts/gecko &&
ln -s /builds/worker/artifacts artifacts &&
./mach --log-no-times taskgraph decision \
--pushlog-id='0' \
--pushdate='0' \
--project='${project}' \
--owner='${ownerEmail}' \
--level='${level}' \
--repository-type=git \
--tasks-for='${tasks_for}' \
--base-repository='${baseRepoUrl}' \
--base-rev='${baseRev}' \
--head-repository='${repoUrl}' \
--head-ref='${ref}' \
--head-rev='${headRev}'
artifacts:
'public':
type: 'directory'
path: '/builds/worker/artifacts'
expires: {$fromNow: '1 year'}
'public/docker-contexts':
type: 'directory'
path: '/builds/worker/checkouts/gecko/docker-contexts'
# This needs to be at least the deadline of the
# decision task + the docker-image task deadlines.
# It is set to a week to allow for some time for
# debugging, but they are not useful long-term.
expires: {$fromNow: '7 day'}
extra:
$merge:
- treeherder:
$merge:
- machine:
platform: gecko-decision
- $switch:
'eventType == "action"':
groupName: 'action-callback'
groupSymbol: AC
symbol: '${action.symbol}'
$default:
symbol: D
- $if: 'eventType == "action"'
then:
parent: '${action.taskGroupId}'
action:
name: '${action.name}'
context:
taskGroupId: '${action.taskGroupId}'
taskId: {$eval: 'taskId'}
input: {$eval: 'input'}
clientId: {$eval: 'clientId'}
- tasks_for: '${tasks_for}'