Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import argparse
import ast
import difflib
import errno
import shlex
import sys
import types
import uuid
from collections.abc import Iterable
from pathlib import Path
from typing import Dict, Optional, Union
from mozfile import load_source
from .base import MissingFileError, UnknownCommandError
INVALID_ENTRY_POINT = r"""
Entry points should return a list of command providers or directories
containing command providers. The following entry point is invalid:
%s
You are seeing this because there is an error in an external module attempting
to implement a mach command. Please fix the error, or uninstall the module from
your system.
""".lstrip()
class MachCommandReference:
"""A reference to a mach command.
Holds the metadata for a mach command.
"""
module: Path
def __init__(
self,
module: Union[str, Path],
command_dependencies: Optional[list] = None,
):
self.module = Path(module)
self.command_dependencies = command_dependencies or []
MACH_COMMANDS = {
"addstory": MachCommandReference("toolkit/content/widgets/mach_commands.py"),
"addtest": MachCommandReference("testing/mach_commands.py"),
"addwidget": MachCommandReference("toolkit/content/widgets/mach_commands.py"),
"android": MachCommandReference("mobile/android/mach_commands.py"),
"android-emulator": MachCommandReference("mobile/android/mach_commands.py"),
"artifact": MachCommandReference(
"python/mozbuild/mozbuild/artifact_commands.py",
),
"awsy-test": MachCommandReference("testing/awsy/mach_commands.py"),
"bootstrap": MachCommandReference(
"python/mozboot/mozboot/mach_commands.py",
),
"browsertime": MachCommandReference("tools/browsertime/mach_commands.py"),
"build": MachCommandReference(
"python/mozbuild/mozbuild/build_commands.py",
),
"build-backend": MachCommandReference(
"python/mozbuild/mozbuild/build_commands.py",
),
"buildsymbols": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
"buildtokens": MachCommandReference("toolkit/content/widgets/mach_commands.py"),
"busted": MachCommandReference("tools/mach_commands.py"),
"cargo": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
"clang-format": MachCommandReference(
"python/mozbuild/mozbuild/code_analysis/mach_commands.py"
),
"clang-tidy": MachCommandReference(
"python/mozbuild/mozbuild/code_analysis/mach_commands.py"
),
"clobber": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
"compare-locales": MachCommandReference("tools/compare-locales/mach_commands.py"),
"compileflags": MachCommandReference(
"python/mozbuild/mozbuild/compilation/codecomplete.py"
),
"configure": MachCommandReference("python/mozbuild/mozbuild/build_commands.py"),
"cppunittest": MachCommandReference("testing/mach_commands.py"),
"cramtest": MachCommandReference("testing/mach_commands.py"),
"crashtest": MachCommandReference("layout/tools/reftest/mach_commands.py"),
"data-review": MachCommandReference(
"toolkit/components/glean/build_scripts/mach_commands.py"
),
"doc": MachCommandReference("tools/moztreedocs/mach_commands.py"),
"doctor": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
"environment": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
"eslint": MachCommandReference("tools/lint/mach_commands.py"),
"event-into-legacy": MachCommandReference(
"toolkit/components/glean/build_scripts/mach_commands.py"
),
"fetch-condprofile": MachCommandReference("testing/condprofile/mach_commands.py"),
"file-info": MachCommandReference(
"python/mozbuild/mozbuild/frontend/mach_commands.py"
),
"firefox-ui-functional": MachCommandReference(
"testing/firefox-ui/mach_commands.py"
),
"fluent-migration-test": MachCommandReference("testing/mach_commands.py"),
"format": MachCommandReference("tools/lint/mach_commands.py"),
"geckodriver": MachCommandReference("testing/geckodriver/mach_commands.py"),
"geckoview-junit": MachCommandReference(
"testing/mochitest/mach_commands.py", ["test"]
),
"gen-use-counter-metrics": MachCommandReference("dom/base/mach_commands.py"),
"generate-test-certs": MachCommandReference(
"security/manager/tools/mach_commands.py"
),
"gradle": MachCommandReference("mobile/android/mach_commands.py"),
"gradle-install": MachCommandReference("mobile/android/mach_commands.py"),
"gtest": MachCommandReference(
"python/mozbuild/mozbuild/mach_commands.py", ["test"]
),
"hazards": MachCommandReference("js/src/devtools/rootAnalysis/mach_commands.py"),
"ide": MachCommandReference("python/mozbuild/mozbuild/backend/mach_commands.py"),
"import-pr": MachCommandReference("tools/vcs/mach_commands.py"),
"install": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
"install-moz-phab": MachCommandReference("tools/phabricator/mach_commands.py"),
"jit-test": MachCommandReference("testing/mach_commands.py"),
"jsapi-tests": MachCommandReference("testing/mach_commands.py"),
"jsshell-bench": MachCommandReference("testing/mach_commands.py"),
"jstestbrowser": MachCommandReference("layout/tools/reftest/mach_commands.py"),
"jstests": MachCommandReference("testing/mach_commands.py"),
"l10n-cross-channel": MachCommandReference(
"tools/compare-locales/mach_commands.py"
),
"lint": MachCommandReference("tools/lint/mach_commands.py"),
"logspam": MachCommandReference("tools/mach_commands.py"),
"mach-commands": MachCommandReference("python/mach/mach/commands/commandinfo.py"),
"mach-completion": MachCommandReference("python/mach/mach/commands/commandinfo.py"),
"mach-debug-commands": MachCommandReference(
"python/mach/mach/commands/commandinfo.py"
),
"macos-sign": MachCommandReference("tools/signing/macos/mach_commands.py"),
"manifest": MachCommandReference("testing/mach_commands.py"),
"marionette-test": MachCommandReference("testing/marionette/mach_commands.py"),
"mochitest": MachCommandReference("testing/mochitest/mach_commands.py", ["test"]),
"mots": MachCommandReference("tools/mach_commands.py"),
"mozbuild-reference": MachCommandReference(
"python/mozbuild/mozbuild/frontend/mach_commands.py",
),
"mozharness": MachCommandReference("testing/mozharness/mach_commands.py"),
"mozregression": MachCommandReference("tools/mach_commands.py"),
"node": MachCommandReference("tools/mach_commands.py"),
"npm": MachCommandReference("tools/mach_commands.py"),
"package": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
"package-multi-locale": MachCommandReference(
"python/mozbuild/mozbuild/mach_commands.py"
),
"pastebin": MachCommandReference("tools/mach_commands.py"),
"perf-data-review": MachCommandReference(
"toolkit/components/glean/build_scripts/mach_commands.py"
),
"perftest": MachCommandReference("python/mozperftest/mozperftest/mach_commands.py"),
"perftest-test": MachCommandReference(
"python/mozperftest/mozperftest/mach_commands.py",
),
"perftest-tools": MachCommandReference(
"python/mozperftest/mozperftest/mach_commands.py"
),
"power": MachCommandReference("tools/power/mach_commands.py"),
"prettier-format": MachCommandReference(
"python/mozbuild/mozbuild/code_analysis/mach_commands.py"
),
"puppeteer-test": MachCommandReference("remote/mach_commands.py"),
"python": MachCommandReference("python/mach_commands.py"),
"python-test": MachCommandReference("python/mach_commands.py"),
"raptor": MachCommandReference("testing/raptor/mach_commands.py"),
"raptor-test": MachCommandReference("testing/raptor/mach_commands.py"),
"reftest": MachCommandReference("layout/tools/reftest/mach_commands.py"),
"release": MachCommandReference("python/mozrelease/mozrelease/mach_commands.py"),
"release-history": MachCommandReference("taskcluster/mach_commands.py"),
"remote": MachCommandReference("remote/mach_commands.py"),
"repackage": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
"resource-usage": MachCommandReference(
"python/mozbuild/mozbuild/build_commands.py",
),
"run": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
"run-condprofile": MachCommandReference("testing/condprofile/mach_commands.py"),
"rusttests": MachCommandReference("testing/mach_commands.py"),
"settings": MachCommandReference("python/mach/mach/commands/settings.py"),
"show-log": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
"static-analysis": MachCommandReference(
"python/mozbuild/mozbuild/code_analysis/mach_commands.py"
),
"storybook": MachCommandReference(
"browser/components/storybook/mach_commands.py", ["run"]
),
"talos-test": MachCommandReference("testing/talos/mach_commands.py"),
"taskcluster-build-image": MachCommandReference("taskcluster/mach_commands.py"),
"taskcluster-image-digest": MachCommandReference("taskcluster/mach_commands.py"),
"taskcluster-load-image": MachCommandReference("taskcluster/mach_commands.py"),
"taskgraph": MachCommandReference("taskcluster/mach_commands.py"),
"telemetry-tests-client": MachCommandReference(
"toolkit/components/telemetry/tests/marionette/mach_commands.py"
),
"test": MachCommandReference("testing/mach_commands.py"),
"test-info": MachCommandReference("testing/mach_commands.py"),
"test-interventions": MachCommandReference(
"testing/webcompat/mach_commands.py",
),
"tps-build": MachCommandReference("testing/tps/mach_commands.py"),
"try": MachCommandReference("tools/tryselect/mach_commands.py"),
"ts": MachCommandReference("tools/ts/mach_commands.py"),
"uniffi": MachCommandReference(
"toolkit/components/uniffi-bindgen-gecko-js/mach_commands.py"
),
"update-glean": MachCommandReference(
"toolkit/components/glean/build_scripts/mach_commands.py"
),
"update-glean-tags": MachCommandReference(
"toolkit/components/glean/build_scripts/mach_commands.py"
),
"valgrind-test": MachCommandReference("build/valgrind/mach_commands.py"),
"vcs-setup": MachCommandReference(
"python/mozboot/mozboot/mach_commands.py",
),
"vendor": MachCommandReference(
"python/mozbuild/mozbuild/vendor/mach_commands.py",
),
"warnings-list": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
"warnings-summary": MachCommandReference(
"python/mozbuild/mozbuild/mach_commands.py"
),
"watch": MachCommandReference(
"python/mozbuild/mozbuild/mach_commands.py",
),
"web-platform-tests": MachCommandReference(
"testing/web-platform/mach_commands.py",
),
"web-platform-tests-update": MachCommandReference(
"testing/web-platform/mach_commands.py",
),
"webidl-example": MachCommandReference("dom/bindings/mach_commands.py"),
"webidl-parser-test": MachCommandReference("dom/bindings/mach_commands.py"),
"wpt": MachCommandReference("testing/web-platform/mach_commands.py"),
"wpt-fetch-logs": MachCommandReference("testing/web-platform/mach_commands.py"),
"wpt-fission-regressions": MachCommandReference(
"testing/web-platform/mach_commands.py"
),
"wpt-interop-score": MachCommandReference("testing/web-platform/mach_commands.py"),
"wpt-manifest-update": MachCommandReference(
"testing/web-platform/mach_commands.py"
),
"wpt-metadata-merge": MachCommandReference("testing/web-platform/mach_commands.py"),
"wpt-metadata-summary": MachCommandReference(
"testing/web-platform/mach_commands.py"
),
"wpt-serve": MachCommandReference("testing/web-platform/mach_commands.py"),
"wpt-test-paths": MachCommandReference("testing/web-platform/mach_commands.py"),
"wpt-unittest": MachCommandReference("testing/web-platform/mach_commands.py"),
"wpt-update": MachCommandReference("testing/web-platform/mach_commands.py"),
"xpcshell": MachCommandReference("js/xpconnect/mach_commands.py"),
"xpcshell-test": MachCommandReference(
"testing/xpcshell/mach_commands.py", ["test"]
),
}
class DecoratorVisitor(ast.NodeVisitor):
def __init__(self):
self.results = {}
def visit_FunctionDef(self, node):
# We only care about `Command` and `SubCommand` decorators, since
# they are the only ones that can specify virtualenv_name
decorators = [
decorator
for decorator in node.decorator_list
if isinstance(decorator, ast.Call)
and isinstance(decorator.func, ast.Name)
and decorator.func.id in ["SubCommand", "Command"]
]
relevant_kwargs = ["command", "subcommand", "virtualenv_name"]
for decorator in decorators:
kwarg_dict = {}
for name, arg in zip(["command", "subcommand"], decorator.args):
kwarg_dict[name] = arg.s
for keyword in decorator.keywords:
if keyword.arg not in relevant_kwargs:
# We only care about these 3 kwargs, so we can safely skip the rest
continue
kwarg_dict[keyword.arg] = getattr(keyword.value, "s", "")
command = kwarg_dict.pop("command")
self.results.setdefault(command, {})
sub_command = kwarg_dict.pop("subcommand", None)
virtualenv_name = kwarg_dict.pop("virtualenv_name", None)
if sub_command:
self.results[command].setdefault("subcommands", {})
sub_command_dict = self.results[command]["subcommands"].setdefault(
sub_command, {}
)
if virtualenv_name:
sub_command_dict["virtualenv_name"] = virtualenv_name
elif virtualenv_name:
# If there is no `subcommand` we are in the `@Command`
# decorator, and need to store the virtualenv_name for
# the 'command'.
self.results[command]["virtualenv_name"] = virtualenv_name
self.generic_visit(node)
def command_virtualenv_info_for_module(module_path):
with module_path.open("r") as file:
content = file.read()
tree = ast.parse(content)
visitor = DecoratorVisitor()
visitor.visit(tree)
return visitor.results
class DetermineCommandVenvAction(argparse.Action):
def __init__(
self,
option_strings,
dest,
topsrcdir,
required=True,
):
self.topsrcdir = topsrcdir
argparse.Action.__init__(
self,
option_strings,
dest,
required=required,
help=argparse.SUPPRESS,
nargs=argparse.REMAINDER,
)
def __call__(self, parser, namespace, values, option_string=None):
if len(values) == 0:
return
command = values[0]
aliases = namespace.mach_command_aliases
if command in aliases:
alias = aliases[command]
arg_string = shlex.split(alias)
command = arg_string.pop(0)
# the "help" command does not have a module file, it's handled
# a bit later and should be skipped here.
if command == "help":
return
command_reference = MACH_COMMANDS.get(command)
if not command_reference:
# Try to find similarly named commands, may raise UnknownCommandError.
suggested_command = suggest_command(command)
sys.stderr.write(
f"We're assuming the '{command}' command is '{suggested_command}' and we're executing it for you.\n\n"
)
command = suggested_command
command_reference = MACH_COMMANDS.get(command)
setattr(namespace, "command_name", command)
if len(values) > 1:
potential_sub_command_name = values[1]
else:
potential_sub_command_name = None
module_path = Path(self.topsrcdir) / command_reference.module
module_dict = command_virtualenv_info_for_module(module_path)
command_dict = module_dict.get(command, {})
if not command_dict:
return
site = command_dict.get("virtualenv_name", "common")
if potential_sub_command_name and not potential_sub_command_name.startswith(
"-"
):
all_sub_commands_dict = command_dict.get("subcommands", {})
if all_sub_commands_dict:
sub_command_dict = all_sub_commands_dict.get(
potential_sub_command_name, {}
)
if sub_command_dict:
site = sub_command_dict.get("virtualenv_name", "common")
setattr(namespace, "site_name", site)
def suggest_command(command):
names = MACH_COMMANDS.keys()
# We first try to look for a valid command that is very similar to the given command.
suggested_commands = difflib.get_close_matches(command, names, cutoff=0.8)
# If we find more than one matching command, or no command at all,
# we give command suggestions instead (with a lower matching threshold).
# All commands that start with the given command (for instance:
# 'mochitest-plain', 'mochitest-chrome', etc. for 'mochitest-')
# are also included.
if len(suggested_commands) != 1:
suggested_commands = set(difflib.get_close_matches(command, names, cutoff=0.5))
suggested_commands |= {cmd for cmd in names if cmd.startswith(command)}
raise UnknownCommandError(command, "run", suggested_commands)
return suggested_commands[0]
def load_commands_from_directory(path: Path):
"""Scan for mach commands from modules in a directory.
This takes a path to a directory, loads the .py files in it, and
registers and found mach command providers with this mach instance.
"""
for f in sorted(path.iterdir()):
if not f.suffix == ".py" or f.name == "__init__.py":
continue
full_path = path / f
module_name = f"mach.commands.{str(f)[0:-3]}"
load_commands_from_file(full_path, module_name=module_name)
def load_commands_from_file(path: Union[str, Path], module_name=None):
"""Scan for mach commands from a file.
This takes a path to a file and loads it as a Python module under the
module name specified. If no name is specified, a random one will be
chosen.
"""
if module_name is None:
# Ensure parent module is present otherwise we'll (likely) get
# an error due to unknown parent.
if "mach.commands" not in sys.modules:
mod = types.ModuleType("mach.commands")
sys.modules["mach.commands"] = mod
module_name = f"mach.commands.{uuid.uuid4().hex}"
try:
load_source(module_name, str(path))
except IOError as e:
if e.errno != errno.ENOENT:
raise
raise MissingFileError(f"{path} does not exist")
def load_commands_from_spec(
spec: Dict[str, MachCommandReference], topsrcdir: str, missing_ok=False
):
"""Load mach commands based on the given spec.
Takes a dictionary mapping command names to their metadata.
"""
modules = set(spec[command].module for command in spec)
for path in modules:
try:
load_commands_from_file(Path(topsrcdir) / path)
except MissingFileError:
if not missing_ok:
raise
def load_commands_from_entry_point(group="mach.providers"):
"""Scan installed packages for mach command provider entry points. An
entry point is a function that returns a list of paths to files or
directories containing command providers.
This takes an optional group argument which specifies the entry point
group to use. If not specified, it defaults to 'mach.providers'.
"""
try:
import pkg_resources
except ImportError:
print(
"Could not find setuptools, ignoring command entry points",
file=sys.stderr,
)
return
for entry in pkg_resources.iter_entry_points(group=group, name=None):
paths = [Path(path) for path in entry.load()()]
if not isinstance(paths, Iterable):
print(INVALID_ENTRY_POINT % entry)
sys.exit(1)
for path in paths:
if path.is_file():
load_commands_from_file(path)
elif path.is_dir():
load_commands_from_directory(path)
else:
print(f"command provider '{path}' does not exist")
def load_command_module_from_command_name(command_name: str, topsrcdir: str):
command_reference = MACH_COMMANDS.get(command_name)
load_commands_from_spec(
{command_name: command_reference}, topsrcdir, missing_ok=False
)