Source code

Revision control

Copy as Markdown

Other Tools

# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
# vim: set filetype=python:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
option(
env="MOZ_FETCHES_DIR",
nargs=1,
when="MOZ_AUTOMATION",
help="Directory containing fetched artifacts",
)
@depends("MOZ_FETCHES_DIR", when="MOZ_AUTOMATION")
def moz_fetches_dir(value):
if value:
return value[0]
@depends(vcs_checkout_type, milestone.is_nightly, "MOZ_AUTOMATION")
def bootstrap_default(vcs_checkout_type, is_nightly, automation):
if automation:
return False
# We only enable if building off a VCS checkout of central.
if is_nightly and vcs_checkout_type:
return True
option(
"--enable-bootstrap",
default=bootstrap_default,
help="{Automatically bootstrap or update some toolchains|Disable bootstrap or update of toolchains}",
)
@depends(developer_options, "--enable-bootstrap", moz_fetches_dir)
def bootstrap_search_path_order(developer_options, bootstrap, moz_fetches_dir):
if moz_fetches_dir:
log.debug("Prioritizing MOZ_FETCHES_DIR in toolchain path.")
return "prepend"
if bootstrap:
log.debug(
"Prioritizing mozbuild state dir in toolchain paths because "
"bootstrap mode is enabled."
)
return "prepend"
if developer_options:
log.debug(
"Prioritizing mozbuild state dir in toolchain paths because "
"you are not building in release mode."
)
return "prepend"
log.debug(
"Prioritizing system over mozbuild state dir in "
"toolchain paths because you are building in "
"release mode."
)
return "append"
toolchains_base_dir = moz_fetches_dir | mozbuild_state_path
@dependable
@imports("os")
@imports(_from="os", _import="environ")
def original_path():
return environ["PATH"].split(os.pathsep)
@depends(host, when="--enable-bootstrap")
@imports("os")
@imports("traceback")
@imports(_from="mozbuild.toolchains", _import="toolchain_task_definitions")
@imports(_from="__builtin__", _import="Exception")
def bootstrap_toolchain_tasks(host):
prefix = {
("x86_64", "GNU", "Linux"): "linux64",
("x86_64", "OSX", "Darwin"): "macosx64",
("aarch64", "OSX", "Darwin"): "macosx64-aarch64",
("x86_64", "WINNT", "WINNT"): "win64",
("aarch64", "WINNT", "WINNT"): "win64-aarch64",
}.get((host.cpu, host.os, host.kernel))
try:
tasks = toolchain_task_definitions()
except Exception as e:
message = traceback.format_exc()
log.warning(str(e))
log.debug(message)
return None
# We only want to use toolchains annotated with "local-toolchain". We also limit the
# amount of data to what we use, so that trace logs can be more useful.
tasks = {
k: {
"index": t.optimization["index-search"],
"artifact": t.attributes["toolchain-artifact"],
}
for k, t in tasks.items()
if t.attributes.get("local-toolchain") and "index-search" in t.optimization
}
return namespace(prefix=prefix, tasks=tasks)
@template
def bootstrap_path(path, **kwargs):
when = kwargs.pop("when", None)
if kwargs:
configure_error("bootstrap_path only takes `when` as a keyword argument")
@depends(
"--enable-bootstrap",
toolchains_base_dir,
moz_fetches_dir,
bootstrap_toolchain_tasks,
build_environment,
dependable(path),
when=when,
)
@imports("os")
@imports("subprocess")
@imports("sys")
@imports(_from="mozbuild.util", _import="ensureParentDir")
@imports(_from="importlib", _import="import_module")
@imports(_from="__builtin__", _import="open")
@imports(_from="__builtin__", _import="Exception")
def bootstrap_path(
bootstrap, toolchains_base_dir, moz_fetches_dir, tasks, build_env, path
):
if not path:
return
path_parts = path.split("/")
def try_bootstrap(exists):
if not tasks:
return False
prefixes = [""]
if tasks.prefix:
prefixes.insert(0, "{}-".format(tasks.prefix))
for prefix in prefixes:
label = "toolchain-{}{}".format(prefix, path_parts[0])
task = tasks.tasks.get(label)
if task:
break
log.debug("Trying to bootstrap %s", label)
if not task:
return False
task_index = task["index"]
log.debug("Resolved %s to %s", label, task_index[0])
task_index = task_index[0].split(".")[-1]
artifact = task["artifact"]
# `mach artifact toolchain` doesn't support authentication for
# private artifacts.
if not artifact.startswith("public/"):
log.debug("Cannot bootstrap %s: not a public artifact", label)
return False
index_file = os.path.join(toolchains_base_dir, "indices", path_parts[0])
try:
with open(index_file) as fh:
index = fh.read().strip()
except Exception:
# On automation, if there's an artifact in MOZ_FETCHES_DIR, we assume it's
# up-to-date.
index = task_index if moz_fetches_dir else None
if index == task_index and exists:
log.debug("%s is up-to-date", label)
return True
# Manually import with import_module so that we can gracefully disable bootstrap
# when e.g. building from a js standalone tarball, that doesn't contain the
# taskgraph code. In those cases, `mach artifact toolchain --from-build` would
# also fail.
try:
IndexSearch = import_module(
"gecko_taskgraph.optimize.strategies"
).IndexSearch
except Exception:
log.debug("Cannot bootstrap %s: missing taskgraph module", label)
return False
task_id = IndexSearch().should_replace_task(task, {}, None, task["index"])
if task_id:
# If we found the task in the index, use the `mach artifact toolchain`
# fast path.
flags = ["--from-task", f"{task_id}:{artifact}"]
else:
# Otherwise, use the slower path, which will print a better error than
# we would be able to.
flags = ["--from-build", label]
log.info(
"%s bootstrapped toolchain in %s",
"Updating" if exists else "Installing",
os.path.join(toolchains_base_dir, path_parts[0]),
)
os.makedirs(toolchains_base_dir, exist_ok=True)
subprocess.run(
[
sys.executable,
os.path.join(build_env.topsrcdir, "mach"),
"--log-no-times",
"artifact",
"toolchain",
]
+ flags,
cwd=toolchains_base_dir,
check=True,
)
ensureParentDir(index_file)
with open(index_file, "w") as fh:
fh.write(task_index)
return True
path = os.path.join(toolchains_base_dir, *path_parts)
if bootstrap:
try:
if not try_bootstrap(os.path.exists(path)):
# If there aren't toolchain artifacts to use for this build,
# don't return a path.
return None
except Exception as e:
log.error("%s", e)
die("If you can't fix the above, retry with --disable-bootstrap.")
# We re-test whether the path exists because it may have been created by
# try_bootstrap. Automation will not have gone through the bootstrap
# process, but we want to return the path if it exists.
if os.path.exists(path):
return path
return bootstrap_path
@template
def bootstrap_search_path(path, paths=original_path, **kwargs):
@depends(
bootstrap_path(path, **kwargs),
bootstrap_search_path_order,
paths,
original_path,
)
def bootstrap_search_path(path, order, paths, original_path):
if paths is None:
paths = original_path
if not path:
return paths
if order == "prepend":
return [path] + paths
return paths + [path]
return bootstrap_search_path