Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
"""
Chunk the partner repack tasks by subpartner and locale
"""
import copy
from mozbuild.chunkify import chunkify
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.dependencies import get_primary_dependency
from gecko_taskgraph.util.partners import (
apply_partner_priority,
get_repack_ids_by_platform,
)
transforms = TransformSequence()
transforms.add(apply_partner_priority)
@transforms.add
def chunk_partners(config, jobs):
for job in jobs:
dep_job = get_primary_dependency(config, job)
assert dep_job
build_platform = dep_job.attributes["build_platform"]
repack_id = dep_job.task.get("extra", {}).get("repack_id")
repack_ids = dep_job.task.get("extra", {}).get("repack_ids")
copy_repack_ids = job.pop("copy-repack-ids", False)
if copy_repack_ids:
assert repack_ids, "dep_job {} doesn't have repack_ids!".format(
dep_job.label
)
job.setdefault("extra", {})["repack_ids"] = repack_ids
yield job
# first downstream of the repack task, no chunking or fanout has been done yet
elif not any([repack_id, repack_ids]):
platform_repack_ids = get_repack_ids_by_platform(config, build_platform)
# we chunk mac signing
if config.kind in (
"release-partner-repack-signing",
"release-eme-free-repack-signing",
"release-eme-free-repack-mac-signing",
"release-partner-repack-mac-signing",
):
repacks_per_chunk = job.get("repacks-per-chunk")
chunks, remainder = divmod(len(platform_repack_ids), repacks_per_chunk)
if remainder:
chunks = int(chunks + 1)
for this_chunk in range(1, chunks + 1):
chunk = chunkify(platform_repack_ids, this_chunk, chunks)
partner_job = copy.deepcopy(job)
partner_job.setdefault("extra", {}).setdefault("repack_ids", chunk)
partner_job["extra"]["repack_suffix"] = str(this_chunk)
yield partner_job
# linux and windows we fan out immediately to one task per partner-sub_partner-locale
else:
for repack_id in platform_repack_ids:
partner_job = copy.deepcopy(job) # don't overwrite dict values here
partner_job.setdefault("extra", {})
partner_job["extra"]["repack_id"] = repack_id
yield partner_job
# fan out chunked mac signing for repackage
elif repack_ids:
for repack_id in repack_ids:
partner_job = copy.deepcopy(job)
partner_job.setdefault("extra", {}).setdefault("repack_id", repack_id)
yield partner_job
# otherwise we've fully fanned out already, continue by passing repack_id on
else:
partner_job = copy.deepcopy(job)
partner_job.setdefault("extra", {}).setdefault("repack_id", repack_id)
yield partner_job