Source code
Revision control
Copy as Markdown
Other Tools
Test Info:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import pytest
from mozunit import main
from taskgraph.task import Task
from taskgraph.util.schema import SchemaValidationError
from gecko_taskgraph.transforms.duplicate import transforms
def make_task(
label,
kind="build-thing",
attributes=None,
dependencies=None,
soft_dependencies=None,
if_dependencies=None,
optimization=None,
):
return Task(
kind=kind,
label=label,
attributes=attributes or {"run_on_projects": ["all"]},
task={
"metadata": {"name": label},
"tags": {"label": label, "kind": kind},
"extra": {"treeherder": {"symbol": "B", "tier": 1}},
"routes": ["index.gecko.v2.trunk.thing", "tc-treeherder.v2.thing"],
},
dependencies=dependencies or {},
soft_dependencies=soft_dependencies or [],
if_dependencies=if_dependencies or [],
optimization=optimization,
)
def run_duplicate(
run_transform,
duplicate,
upstream,
overrides=None,
kind="build-thing-copy",
kind_dependencies=None,
templates=None,
):
if templates is None:
template = {"name": "copy", "duplicate": duplicate}
template.update(overrides or {})
templates = [template]
if kind_dependencies is None:
kind_dependencies = sorted({t.kind for t in upstream})
return list(
run_transform(
transforms,
templates,
kind=kind,
config={"kind-dependencies": kind_dependencies},
kind_dependencies_tasks={t.label: t for t in upstream},
)
)
def test_copies_every_selected_task(run_transform):
upstream = [make_task("build-thing-a"), make_task("build-thing-b")]
tasks = run_duplicate(
run_transform, {"kinds": ["build-thing"], "suffix": "-v"}, upstream
)
assert sorted(t["label"] for t in tasks) == ["build-thing-a-v", "build-thing-b-v"]
def test_ignores_other_kinds(run_transform):
upstream = [make_task("build-thing-a"), make_task("other-b", kind="other")]
tasks = run_duplicate(
run_transform, {"kinds": ["build-thing"], "suffix": "-v"}, upstream
)
assert [t["label"] for t in tasks] == ["build-thing-a-v"]
def test_kinds_defaults_to_kind_dependencies(run_transform):
upstream = [make_task("build-thing-a"), make_task("other-b", kind="other")]
tasks = run_duplicate(run_transform, {"suffix": "-v"}, upstream)
assert sorted(t["label"] for t in tasks) == ["build-thing-a-v", "other-b-v"]
def test_with_attributes_narrows_the_selection(run_transform):
upstream = [
make_task("build-thing-a", attributes={"build-type": "release"}),
make_task("build-thing-b", attributes={"build-type": "debug"}),
]
tasks = run_duplicate(
run_transform,
{
"kinds": ["build-thing"],
"suffix": "-v",
"with-attributes": {"build-type": "release"},
},
upstream,
)
assert [t["label"] for t in tasks] == ["build-thing-a-v"]
def test_with_attributes_accepts_a_list_of_values(run_transform):
upstream = [
make_task("build-thing-a", attributes={"build-type": "release"}),
make_task("build-thing-b", attributes={"build-type": "debug"}),
make_task("build-thing-c", attributes={"build-type": "asan"}),
]
tasks = run_duplicate(
run_transform,
{
"kinds": ["build-thing"],
"suffix": "-v",
"with-attributes": {"build-type": ["release", "asan"]},
},
upstream,
)
assert [t["label"] for t in tasks] == ["build-thing-a-v", "build-thing-c-v"]
def test_with_attributes_skips_a_task_missing_the_attribute(run_transform):
upstream = [
make_task("build-thing-a", attributes={"build-type": "release"}),
make_task("build-thing-b", attributes={}),
]
tasks = run_duplicate(
run_transform,
{
"kinds": ["build-thing"],
"suffix": "-v",
"with-attributes": {"build-type": "release"},
},
upstream,
)
assert [t["label"] for t in tasks] == ["build-thing-a-v"]
def test_include_labels_keeps_only_matches(run_transform):
upstream = [make_task("build-thing-a"), make_task("build-thing-b")]
tasks = run_duplicate(
run_transform,
{
"kinds": ["build-thing"],
"suffix": "-v",
"include-labels": ["-a$"],
},
upstream,
)
assert [t["label"] for t in tasks] == ["build-thing-a-v"]
def test_exclude_labels_wins_over_include_labels(run_transform):
upstream = [make_task("build-thing-a"), make_task("build-thing-b")]
tasks = run_duplicate(
run_transform,
{
"kinds": ["build-thing"],
"suffix": "-v",
"include-labels": ["^build-thing-"],
"exclude-labels": ["-b$"],
},
upstream,
)
assert [t["label"] for t in tasks] == ["build-thing-a-v"]
def test_exclude_labels_wins_over_with_attributes(run_transform):
upstream = [
make_task("build-thing-a", attributes={"build-type": "release"}),
make_task("build-thing-beta", attributes={"build-type": "release"}),
]
tasks = run_duplicate(
run_transform,
{
"kinds": ["build-thing"],
"suffix": "-v",
"with-attributes": {"build-type": "release"},
"exclude-labels": ["-beta$"],
},
upstream,
)
assert [t["label"] for t in tasks] == ["build-thing-a-v"]
def test_excludes_a_channel_no_attribute_records(run_transform):
# The android build kinds mark a release or beta task with `release-type`
# but leave a nightly task with no attributes at all, so the label is the
# only thing every channel has in common.
upstream = [
make_task("build-thing-debug", attributes={}),
make_task("build-thing-release", attributes={"release-type": "release"}),
make_task("build-thing-nightly", attributes={}),
]
tasks = run_duplicate(
run_transform,
{
"kinds": ["build-thing"],
"suffix": "-v",
"exclude-labels": ["-(beta|release|nightly)(-|$)"],
},
upstream,
)
assert [t["label"] for t in tasks] == ["build-thing-debug-v"]
def test_drops_index_routes_and_keeps_the_rest(run_transform):
tasks = run_duplicate(
run_transform,
{"kinds": ["build-thing"], "suffix": "-v"},
[make_task("build-thing-a")],
)
assert tasks[0]["task"]["routes"] == ["tc-treeherder.v2.thing"]
def test_renames_task_identity(run_transform):
tasks = run_duplicate(
run_transform,
{
"kinds": ["build-thing"],
"suffix": "-v",
"treeherder-symbol-suffix": "-x",
},
[make_task("build-thing-a")],
)
inner = tasks[0]["task"]
assert inner["metadata"]["name"] == "build-thing-a-v"
assert inner["tags"]["label"] == "build-thing-a-v"
assert inner["tags"]["kind"] == "build-thing-copy"
assert inner["extra"]["treeherder"]["symbol"] == "B-x"
def test_set_name_labels_the_copy_after_its_own_kind(run_transform):
tasks = run_duplicate(
run_transform,
{"kinds": ["build-thing"], "set-name": "strip-kind"},
[make_task("build-thing-a")],
)
assert tasks[0]["label"] == "build-thing-copy-a"
assert tasks[0]["task"]["metadata"]["name"] == "build-thing-copy-a"
def test_set_name_and_suffix_combine(run_transform):
tasks = run_duplicate(
run_transform,
{"kinds": ["build-thing"], "set-name": "strip-kind", "suffix": "-v"},
[make_task("build-thing-a")],
)
assert tasks[0]["label"] == "build-thing-copy-a-v"
def test_rejects_two_sources_naming_one_copy(run_transform):
upstream = [
make_task("build-thing-a"),
make_task("test-thing-a", kind="test-thing"),
]
with pytest.raises(Exception, match="would name the copies of both"):
run_duplicate(
run_transform,
{"kinds": ["build-thing", "test-thing"], "set-name": "strip-kind"},
upstream,
)
def test_records_where_each_copy_came_from(run_transform):
tasks = run_duplicate(
run_transform,
{"kinds": ["build-thing"], "suffix": "-v"},
[make_task("build-thing-a")],
)
assert tasks[0]["attributes"]["duplicate-of"] == "build-thing-a"
def test_sibling_keys_overlay_the_copy(run_transform):
tasks = run_duplicate(
run_transform,
{"kinds": ["build-thing"], "suffix": "-v"},
[make_task("build-thing-a")],
overrides={
"attributes": {"run_on_projects": []},
"task": {"extra": {"treeherder": {"tier": 3}}},
},
)
# A list is overwritten outright, not appended to.
assert tasks[0]["attributes"]["run_on_projects"] == []
# Nested dicts are descended into, so the sibling key survives.
assert tasks[0]["task"]["extra"]["treeherder"] == {"symbol": "B", "tier": 3}
def test_sibling_keys_reach_the_resolved_payload(run_transform):
tasks = run_duplicate(
run_transform,
{"kinds": ["build-thing"], "suffix": "-v"},
[make_task("build-thing-a")],
overrides={"task": {"payload": {"env": {"MOZ_ENABLE_PROFILING": "1"}}}},
)
assert tasks[0]["task"]["payload"]["env"] == {"MOZ_ENABLE_PROFILING": "1"}
def test_overrides_leave_the_source_task_alone(run_transform):
source = make_task("build-thing-a")
run_duplicate(
run_transform,
{"kinds": ["build-thing"], "suffix": "-v"},
[source],
overrides={"attributes": {"run_on_projects": []}},
)
assert source.attributes["run_on_projects"] == ["all"]
assert source.task["metadata"]["name"] == "build-thing-a"
def test_chain_from_redirects_dependencies_to_their_copies(run_transform):
upstream = [
make_task("dep-a-p", kind="dep-copy", attributes={"duplicate-of": "dep-a"}),
make_task("dep-b-p", kind="dep-copy", attributes={"duplicate-of": "dep-b"}),
make_task("dep-c-p", kind="dep-copy", attributes={"duplicate-of": "dep-c"}),
make_task(
"build-thing-a",
dependencies={"dep": "dep-a", "toolchain": "toolchain-z"},
soft_dependencies=["dep-b", "other-z"],
if_dependencies=["dep-c"],
),
]
tasks = run_duplicate(
run_transform,
{
"kinds": ["build-thing"],
"suffix": "-p",
"chain-from": ["dep-copy"],
},
upstream,
)
assert tasks[0]["dependencies"] == {"dep": "dep-a-p", "toolchain": "toolchain-z"}
assert tasks[0]["soft-dependencies"] == ["dep-b-p", "other-z"]
assert tasks[0]["if-dependencies"] == ["dep-c-p"]
def test_chain_from_ignores_kinds_it_does_not_name(run_transform):
upstream = [
make_task("dep-a-p", kind="dep-copy", attributes={"duplicate-of": "dep-a"}),
make_task("dep-a-s", kind="dep-asan", attributes={"duplicate-of": "dep-a"}),
make_task("build-thing-a", dependencies={"dep": "dep-a"}),
]
tasks = run_duplicate(
run_transform,
{"kinds": ["build-thing"], "suffix": "-v", "chain-from": ["dep-asan"]},
upstream,
)
assert tasks[0]["dependencies"] == {"dep": "dep-a-s"}
def test_rejects_two_chained_kinds_duplicating_one_task(run_transform):
upstream = [
make_task("dep-a-p", kind="dep-copy", attributes={"duplicate-of": "dep-a"}),
make_task("dep-a-s", kind="dep-asan", attributes={"duplicate-of": "dep-a"}),
make_task("build-thing-a", dependencies={"dep": "dep-a"}),
]
with pytest.raises(Exception, match="dep-a is duplicated by both"):
run_duplicate(
run_transform,
{
"kinds": ["build-thing"],
"suffix": "-v",
"chain-from": ["dep-copy", "dep-asan"],
},
upstream,
)
def test_rejects_a_chain_from_kind_missing_from_kind_dependencies(run_transform):
with pytest.raises(Exception, match="`duplicate.chain-from`.*absent"):
run_duplicate(
run_transform,
{"kinds": ["build-thing"], "suffix": "-v", "chain-from": ["absent"]},
[make_task("build-thing-a")],
kind_dependencies=["build-thing"],
)
def test_one_kind_can_hold_several_templates(run_transform):
upstream = [make_task("build-thing-a")]
tasks = run_duplicate(
run_transform,
None,
upstream,
kind_dependencies=["build-thing"],
templates=[
{
"name": "profiling",
"duplicate": {"kinds": ["build-thing"], "suffix": "-p"},
"task": {"extra": {"treeherder": {"tier": 3}}},
},
{
"name": "asan",
"duplicate": {"kinds": ["build-thing"], "suffix": "-s"},
},
],
)
assert [t["label"] for t in tasks] == ["build-thing-a-p", "build-thing-a-s"]
assert tasks[0]["task"]["extra"]["treeherder"]["tier"] == 3
assert tasks[1]["task"]["extra"]["treeherder"]["tier"] == 1
def test_keeps_a_removal_optimization(run_transform):
tasks = run_duplicate(
run_transform,
{"kinds": ["build-thing"], "suffix": "-v"},
[make_task("build-thing-a", optimization={"skip-unless-backstop": None})],
)
assert tasks[0]["optimization"] == {"skip-unless-backstop": None}
def test_rejects_a_replacement_optimization(run_transform):
with pytest.raises(Exception, match="index-search"):
run_duplicate(
run_transform,
{"kinds": ["build-thing"], "suffix": "-v"},
[make_task("build-thing-a", optimization={"index-search": ["some.index"]})],
)
def test_passes_through_a_template_without_a_duplicate_block(run_transform):
tasks = run_duplicate(
run_transform,
None,
[make_task("build-thing-a")],
kind_dependencies=["build-thing"],
templates=[{"name": "mine"}],
)
assert tasks == [{"name": "mine"}]
def test_rejects_an_unknown_key_inside_duplicate(run_transform):
with pytest.raises(SchemaValidationError):
run_duplicate(
run_transform,
{"kinds": ["build-thing"], "suffix": "-v", "typo": True},
[make_task("build-thing-a")],
)
def test_rejects_a_kind_missing_from_kind_dependencies(run_transform):
with pytest.raises(Exception, match="missing from `kind-dependencies`"):
run_duplicate(
run_transform,
{"kinds": ["build-thing", "absent"], "suffix": "-v"},
[make_task("build-thing-a")],
kind_dependencies=["build-thing"],
)
def test_requires_a_suffix_or_a_set_name(run_transform):
with pytest.raises(Exception, match="tell a copy apart from its source"):
run_duplicate(
run_transform,
{"kinds": ["build-thing"]},
[make_task("build-thing-a")],
)
def test_rejects_an_unknown_set_name(run_transform):
with pytest.raises(Exception, match="unknown"):
run_duplicate(
run_transform,
{"kinds": ["build-thing"], "set-name": "not-a-function"},
[make_task("build-thing-a")],
)
if __name__ == "__main__":
main()