Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import argparse
import logging
import subprocess
import sys
from datetime import datetime, timedelta
from operator import itemgetter
from mach.decorators import Command, CommandArgument, SubCommand
from mozbuild.base import MozbuildObject
def _get_busted_bugs(payload):
import requests
payload = dict(payload)
payload["include_fields"] = "id,summary,last_change_time,resolution"
payload["blocks"] = 1543241
response.raise_for_status()
return response.json().get("bugs", [])
@Command(
"busted",
category="misc",
description="Query known bugs in our tooling, and file new ones.",
)
def busted_default(command_context):
unresolved = _get_busted_bugs({"resolution": "---"})
creation_time = datetime.now() - timedelta(days=15)
creation_time = creation_time.strftime("%Y-%m-%dT%H-%M-%SZ")
resolved = _get_busted_bugs({"creation_time": creation_time})
resolved = [bug for bug in resolved if bug["resolution"]]
all_bugs = sorted(
unresolved + resolved, key=itemgetter("last_change_time"), reverse=True
)
if all_bugs:
for bug in all_bugs:
print(
"[%s] Bug %s - %s"
% (
(
"UNRESOLVED"
if not bug["resolution"]
else "RESOLVED - %s" % bug["resolution"]
),
bug["id"],
bug["summary"],
)
)
else:
print("No known tooling issues found.")
@SubCommand("busted", "file", description="File a bug for busted tooling.")
@CommandArgument(
"against",
help=(
"The specific mach command that is busted (i.e. if you encountered "
"an error with `mach build`, run `mach busted file build`). If "
"the issue is not connected to any particular mach command, you "
"can also run `mach busted file general`."
),
)
def busted_file(command_context, against):
import webbrowser
if (
against != "general"
and against not in command_context._mach_context.commands.command_handlers
):
print(
"%s is not a valid value for `against`. `against` must be "
"the name of a `mach` command, or else the string "
'"general".' % against
)
return 1
if against == "general":
product = "Firefox Build System"
component = "General"
else:
import inspect
import mozpack.path as mozpath
# Look up the file implementing that command, then cross-refernce
# moz.build files to get the product/component.
handler = command_context._mach_context.commands.command_handlers[against]
sourcefile = mozpath.relpath(
inspect.getsourcefile(handler.func), command_context.topsrcdir
)
reader = command_context.mozbuild_reader(config_mode="empty")
try:
res = reader.files_info([sourcefile])[sourcefile]["BUG_COMPONENT"]
product, component = res.product, res.component
except TypeError:
# The file might not have a bug set.
product = "Firefox Build System"
component = "General"
uri = (
"product=%s&component=%s&blocked=1543241" % (product, component)
)
webbrowser.open_new_tab(uri)
def pastebin_create_parser():
# Accept all args so they can be promptly ignored.
parser = argparse.ArgumentParser()
parser.add_argument("argv", nargs=argparse.REMAINDER, help=argparse.SUPPRESS)
return parser
@Command(
"pastebin",
category="misc",
hidden=True,
parser=pastebin_create_parser,
)
def pastebin(command_context, argv):
"""Obsolete command line interface to `paste.mozilla.org`."""
print(
"pastebin.mozilla.org has been decommissioned.\n"
"Please use your favorite search engine to find alternatives."
)
return 1
class PypiBasedTool:
"""
Helper for loading a tool that is hosted on pypi. The package is expected
to expose a `mach_interface` module which has `new_release_on_pypi`,
`parser`, and `run` functions.
"""
def __init__(self, module_name, pypi_name=None):
self.name = module_name
self.pypi_name = pypi_name or module_name
def _import(self):
# Lazy loading of the tools mach interface.
# Note that only the mach_interface module should be used from this file.
import importlib
try:
return importlib.import_module("%s.mach_interface" % self.name)
except ImportError:
return None
def create_parser(self, subcommand=None):
# Create the command line parser.
# If the tool is not installed, or not up to date, it will
# first be installed.
cmd = MozbuildObject.from_environment()
cmd.activate_virtualenv()
tool = self._import()
if not tool:
# The tool is not here at all, install it
cmd.virtualenv_manager.install_pip_package(self.pypi_name)
print(
"%s was installed. please re-run your"
" command. If you keep getting this message please "
" manually run: 'pip install -U %s'." % (self.pypi_name, self.pypi_name)
)
else:
# Check if there is a new release available
release = tool.new_release_on_pypi()
if release:
print(release)
# there is one, so install it. Note that install_pip_package
# does not work here, so just run pip directly.
subprocess.check_call(
[
cmd.virtualenv_manager.python_path,
"-m",
"pip",
"install",
f"{self.pypi_name}=={release}",
]
)
print(
"%s was updated to version %s. please"
" re-run your command." % (self.pypi_name, release)
)
else:
# Tool is up to date, return the parser.
if subcommand:
return tool.parser(subcommand)
else:
return tool.parser()
# exit if we updated or installed mozregression because
# we may have already imported mozregression and running it
# as this may cause issues.
sys.exit(0)
def run(self, **options):
tool = self._import()
tool.run(options)
def mozregression_create_parser():
# Create the mozregression command line parser.
# if mozregression is not installed, or not up to date, it will
# first be installed.
loader = PypiBasedTool("mozregression")
return loader.create_parser()
@Command(
"mozregression",
category="misc",
description="Regression range finder for nightly and inbound builds.",
parser=mozregression_create_parser,
)
def run(command_context, **options):
command_context.activate_virtualenv()
mozregression = PypiBasedTool("mozregression")
mozregression.run(**options)
@Command(
"node",
category="devenv",
description="Run the NodeJS interpreter used for building.",
)
@CommandArgument("args", nargs=argparse.REMAINDER)
def node(command_context, args):
from mozbuild.nodeutil import find_node_executable
# Avoid logging the command
command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
node_path, _ = find_node_executable()
return command_context.run_process(
[node_path] + args,
pass_thru=True, # Allow user to run Node interactively.
ensure_exit_code=False, # Don't throw on non-zero exit code.
)
@Command(
"npm",
category="devenv",
description="Run the npm executable from the NodeJS used for building.",
)
@CommandArgument("args", nargs=argparse.REMAINDER)
def npm(command_context, args):
from mozbuild.nodeutil import find_npm_executable
# Avoid logging the command
command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
import os
# Add node and npm from mozbuild to front of system path
#
# This isn't pretty, but npm currently executes itself with
# `#!/usr/bin/env node`, which means it just uses the node in the
# current PATH. As a result, stuff gets built wrong and installed
# in the wrong places and probably other badness too without this:
npm_path, _ = find_npm_executable()
if not npm_path:
exit(-1, "could not find npm executable")
path = os.path.abspath(os.path.dirname(npm_path))
os.environ["PATH"] = "{}{}{}".format(path, os.pathsep, os.environ["PATH"])
# karma-firefox-launcher needs the path to firefox binary.
firefox_bin = command_context.get_binary_path(validate_exists=False)
if os.path.exists(firefox_bin):
os.environ["FIREFOX_BIN"] = firefox_bin
return command_context.run_process(
[npm_path, "--scripts-prepend-node-path=auto"] + args,
pass_thru=True, # Avoid eating npm output/error messages
ensure_exit_code=False, # Don't throw on non-zero exit code.
)
def logspam_create_parser(subcommand):
# Create the logspam command line parser.
# if logspam is not installed, or not up to date, it will
# first be installed.
loader = PypiBasedTool("logspam", "mozilla-log-spam")
return loader.create_parser(subcommand)
from functools import partial
@Command(
"logspam",
category="misc",
description="Warning categorizer for treeherder test runs.",
)
def logspam(command_context):
pass
@SubCommand("logspam", "report", parser=partial(logspam_create_parser, "report"))
def report(command_context, **options):
command_context.activate_virtualenv()
logspam = PypiBasedTool("logspam")
logspam.run(command="report", **options)
@SubCommand("logspam", "bisect", parser=partial(logspam_create_parser, "bisect"))
def bisect(command_context, **options):
command_context.activate_virtualenv()
logspam = PypiBasedTool("logspam")
logspam.run(command="bisect", **options)
@SubCommand("logspam", "file", parser=partial(logspam_create_parser, "file"))
def create(command_context, **options):
command_context.activate_virtualenv()
logspam = PypiBasedTool("logspam")
logspam.run(command="file", **options)
# mots_loader will be used when running commands and subcommands, as well as
# when creating the parsers.
mots_loader = PypiBasedTool("mots")
def mots_create_parser(subcommand=None):
return mots_loader.create_parser(subcommand)
def mots_run_subcommand(command, command_context, **options):
command_context.activate_virtualenv()
mots_loader.run(command=command, **options)
class motsSubCommand(SubCommand):
"""A helper subclass that reduces repitition when defining subcommands."""
def __init__(self, subcommand):
super().__init__(
"mots",
subcommand,
parser=partial(mots_create_parser, subcommand),
)
@Command(
"mots",
category="misc",
description="Manage module information in-tree using the mots CLI.",
parser=mots_create_parser,
)
def mots(command_context, **options):
"""The main mots command call."""
command_context.activate_virtualenv()
mots_loader.run(**options)
# Define subcommands that will be proxied through mach.
for sc in (
"clean",
"check-hashes",
"export",
"export-and-clean",
"module",
"query",
"settings",
"user",
"validate",
):
# Pass through args and kwargs, but add the subcommand string as the first argument.
motsSubCommand(sc)(lambda *a, **kw: mots_run_subcommand(sc, *a, **kw))