Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
from itertools import islice
from taskgraph import MAX_DEPENDENCIES
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.dependencies import get_primary_dependency
from gecko_taskgraph.util.platforms import architecture
from gecko_taskgraph.util.scriptworker import (
generate_artifact_registry_gcs_sources,
get_beetmover_apt_repo_scope,
get_beetmover_repo_action_scope,
)
transforms = TransformSequence()
@transforms.add
def beetmover_apt(config, tasks):
product = (
"firefox"
if config.params["release_type"] == "nightly"
else config.params["release_product"]
)
filtered_tasks = filter_beetmover_apt_tasks(config, tasks, product)
# There are too many beetmover-repackage dependencies for a single task
# and we hit the taskgraph dependencies limit.
# To work around this limitation, we chunk the would be task
# into tasks dependendent on, at most, half of MAX_DEPENDENCIES.
batches = batched(filtered_tasks, MAX_DEPENDENCIES // 2)
for index, batch in enumerate(batches):
dependencies = {}
gcs_sources = []
for task in batch:
dep = get_primary_dependency(config, task)
assert dep
dependencies[dep.label] = dep.label
gcs_sources.extend(generate_artifact_registry_gcs_sources(dep))
description = f"Batch {index + 1} of beetmover APT submissions for the {config.params['release_type']} .deb packages"
platform = "firefox-release/opt"
treeherder = {
"platform": platform,
"tier": 1,
"kind": "other",
"symbol": f"BM-apt(batch-{index + 1})",
}
apt_repo_scope = get_beetmover_apt_repo_scope(config)
repo_action_scope = get_beetmover_repo_action_scope(config)
attributes = {
"required_signoffs": ["mar-signing"],
"shippable": True,
"shipping_product": product,
}
task = {
"label": f"{config.kind}-{index + 1}-{platform}",
"description": description,
"worker-type": "beetmover",
"treeherder": treeherder,
"scopes": [apt_repo_scope, repo_action_scope],
"attributes": attributes,
"shipping-phase": "ship",
"shipping-product": product,
"dependencies": dependencies,
}
worker = {
"implementation": "beetmover-import-from-gcs-to-artifact-registry",
"product": product,
"gcs-sources": gcs_sources,
}
task["worker"] = worker
yield task
def batched(iterable, n):
"Batch data into tuples of length n. The last batch may be shorter."
# batched('ABCDEFG', 3) --> ABC DEF G
if n < 1:
raise ValueError("n must be at least one")
it = iter(iterable)
batch = tuple(islice(it, n))
while batch:
yield batch
batch = tuple(islice(it, n))
def filter_beetmover_apt_tasks(config, tasks, product):
for task in tasks:
task["primary-dependency"] = get_primary_dependency(config, task)
if filter_beetmover_apt_task(task, product):
yield task
def filter_beetmover_apt_task(task, product):
# We only create beetmover-apt tasks for l10n beetmover-repackage tasks that
# beetmove langpack .deb packages. The langpack .deb packages support all
# architectures, so we generate them only on x86_64 tasks.
return (
is_x86_64_l10n_task(task) or is_not_l10n_task(task)
) and is_task_for_product(task, product)
def is_x86_64_l10n_task(task):
dep = task["primary-dependency"]
locale = dep.attributes.get("locale")
return locale and architecture(dep.attributes["build_platform"]) == "x86_64"
def is_not_l10n_task(task):
dep = task["primary-dependency"]
locale = dep.attributes.get("locale")
return not locale
def is_task_for_product(task, product):
dep = task["primary-dependency"]
return dep.attributes.get("shipping_product") == product