Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import copy
import os
from argparse import ArgumentParser, Namespace
import mozlog
here = os.path.abspath(os.path.dirname(__file__))
try:
from mozbuild.base import MachCommandConditions as conditions
from mozbuild.base import MozbuildObject
build_obj = MozbuildObject.from_environment(cwd=here)
except Exception:
build_obj = None
conditions = None
from mozperftest.metrics import get_layers as metrics_layers # noqa
from mozperftest.system import get_layers as system_layers # noqa
from mozperftest.test import get_layers as test_layers # noqa
from mozperftest.utils import convert_day # noqa
FLAVORS = (
"desktop-browser",
"mobile-browser",
"doc",
"xpcshell",
"webpagetest",
"mochitest",
)
class Options:
general_args = {
"--flavor": {
"choices": FLAVORS,
"metavar": "{{{}}}".format(", ".join(FLAVORS)),
"default": "desktop-browser",
"help": "Only run tests of this flavor.",
},
"tests": {
"nargs": "*",
"metavar": "TEST",
"default": [],
"help": "Test to run. Can be a single test file or URL or a directory"
" of tests (to run recursively). If omitted, the entire suite is run.",
},
"--test-iterations": {
"type": int,
"default": 1,
"help": "Number of times the whole test is executed",
},
"--output": {
"type": str,
"default": "artifacts",
"help": "Path to where data will be stored, defaults to a top-level "
"`artifacts` folder.",
},
"--hooks": {
"type": str,
"default": None,
"help": "Script containing hooks. Can be a path or a URL.",
},
"--verbose": {"action": "store_true", "default": False, "help": "Verbose mode"},
"--push-to-try": {
"action": "store_true",
"default": False,
"help": "Pushin the test to try",
},
"--try-platform": {
"nargs": "*",
"type": str,
"default": "linux",
"help": "Platform to use on try",
"choices": ["linux", "mac", "win"],
},
"--on-try": {
"action": "store_true",
"default": False,
"help": "Running the test on try",
},
"--test-date": {
"type": convert_day,
"default": "today",
"help": "Used in multi-commit testing, it specifies the day to get test builds from. "
"Must follow the format `YYYY.MM.DD` or be `today` or `yesterday`.",
},
}
args = copy.deepcopy(general_args)
for layer in system_layers() + test_layers() + metrics_layers():
if layer.activated:
# add an option to deactivate it
option_name = "--no-%s" % layer.name
option_help = "Deactivates the %s layer" % layer.name
else:
option_name = "--%s" % layer.name
option_help = "Activates the %s layer" % layer.name
Options.args[option_name] = {
"action": "store_true",
"default": False,
"help": option_help,
}
for option, value in layer.arguments.items():
parsed_option = "--%s-%s" % (layer.name, option.replace("_", "-"))
if parsed_option in Options.args:
raise KeyError("%s option already defined!" % parsed_option)
Options.args[parsed_option] = value
class PerftestArgumentParser(ArgumentParser):
"""%(prog)s [options] [test paths]"""
def __init__(self, app=None, **kwargs):
ArgumentParser.__init__(
self, usage=self.__doc__, conflict_handler="resolve", **kwargs
)
# XXX see if this list will vary depending on the flavor & app
self.oldcwd = os.getcwd()
self.app = app
if not self.app and build_obj:
if conditions.is_android(build_obj):
self.app = "android"
if not self.app:
self.app = "generic"
for name, options in Options.args.items():
self.add_argument(name, **options)
mozlog.commandline.add_logging_group(self)
self.set_by_user = []
def parse_helper(self, args):
for arg in args:
arg_part = arg.partition("--")[-1].partition("-")
layer_name = f"--{arg_part[0]}"
layer_exists = arg_part[1] and layer_name in Options.args
if layer_exists:
args.append(layer_name)
def get_user_args(self, args):
# suppress args that were not provided by the user.
res = {}
for key, value in args.items():
if key not in self.set_by_user:
continue
res[key] = value
return res
def _parse_known_args(self, arg_strings, namespace):
# at this point, the namespace is filled with default values
# defined in the args
# let's parse what the user really gave us in the CLI
# in a new namespace
user_namespace, extras = super()._parse_known_args(arg_strings, Namespace())
self.set_by_user = list([name for name, value in user_namespace._get_kwargs()])
# we can now merge both
for key, value in user_namespace._get_kwargs():
setattr(namespace, key, value)
return namespace, extras
def parse_args(self, args=None, namespace=None):
self.parse_helper(args)
return super().parse_args(args, namespace)
def parse_known_args(self, args=None, namespace=None):
self.parse_helper(args)
return super().parse_known_args(args, namespace)
class SideBySideOptions:
args = [
[
["-t", "--test-name"],
{
"type": str,
"required": True,
"dest": "test_name",
"help": "The name of the test task to get videos from.",
},
],
[
["--new-test-name"],
{
"type": str,
"default": None,
"help": "The name of the test task to get videos from in the new revision.",
},
],
[
["--base-revision"],
{
"type": str,
"required": True,
"help": "The base revision to compare a new revision to.",
},
],
[
["--new-revision"],
{
"type": str,
"required": True,
"help": "The base revision to compare a new revision to.",
},
],
[
["--base-branch"],
{
"type": str,
"default": "autoland",
"help": "Branch to search for the base revision.",
},
],
[
["--new-branch"],
{
"type": str,
"default": "autoland",
"help": "Branch to search for the new revision.",
},
],
[
["--base-platform"],
{
"type": str,
"required": True,
"dest": "platform",
"help": "Platform to return results for.",
},
],
[
["--new-platform"],
{
"type": str,
"default": None,
"help": "Platform to return results for in the new revision.",
},
],
[
["-o", "--overwrite"],
{
"action": "store_true",
"default": False,
"help": "If set, the downloaded task group data will be deleted before "
+ "it gets re-downloaded.",
},
],
[
["--cold"],
{
"action": "store_true",
"default": False,
"help": "If set, we'll only look at cold pageload tests.",
},
],
[
["--warm"],
{
"action": "store_true",
"default": False,
"help": "If set, we'll only look at warm pageload tests.",
},
],
[
["--most-similar"],
{
"action": "store_true",
"default": False,
"help": "If set, we'll search for a video pairing that is the most similar.",
},
],
[
["--search-crons"],
{
"action": "store_true",
"default": False,
"help": "If set, we will search for the tasks within the cron jobs as well. ",
},
],
[
["--skip-download"],
{
"action": "store_true",
"default": False,
"help": "If set, we won't try to download artifacts again and we'll "
+ "try using what already exists in the output folder.",
},
],
[
["--output"],
{
"type": str,
"default": None,
"help": "This is where the data will be saved. Defaults to CWD. "
+ "You can include a name for the file here, otherwise it will "
+ "default to side-by-side.mp4.",
},
],
[
["--metric"],
{
"type": str,
"default": "speedindex",
"help": "Metric to use for side-by-side comparison.",
},
],
[
["--vismetPath"],
{
"type": str,
"default": False,
"help": "Paths to visualmetrics.py for step chart generation.",
},
],
[
["--original"],
{
"action": "store_true",
"default": False,
"help": "If set, use the original videos in the side-by-side instead "
+ "of the postprocessed videos.",
},
],
[
["--skip-slow-gif"],
{
"action": "store_true",
"default": False,
"help": "If set, the slow-motion GIFs won't be produced.",
},
],
]
class ChangeDetectorOptions:
args = [
# TODO: Move the common tool arguments to a common
# argument class.
[
["--task-name"],
{
"type": str,
"nargs": "*",
"default": [],
"dest": "task_names",
"help": "The full name of the test task to get data from e.g. "
"test-android-hw-a51-11-0-aarch64-shippable-qr/opt-"
"browsertime-tp6m-geckoview-sina-nofis.",
},
],
[
["-t", "--test-name"],
{
"type": str,
"default": None,
"dest": "test_name",
"help": "The name of the test task to get data from e.g. "
"browsertime-tp6m-geckoview-sina-nofis.",
},
],
[
["--platform"],
{
"type": str,
"default": None,
"help": "Platform to analyze e.g. "
"test-android-hw-a51-11-0-aarch64-shippable-qr/opt.",
},
],
[
["--new-test-name"],
{
"type": str,
"help": "The name of the test task to get data from in the "
"base revision e.g. browsertime-tp6m-geckoview-sina-nofis.",
},
],
[
["--new-platform"],
{
"type": str,
"help": "Platform to analyze in base revision e.g. "
"test-android-hw-a51-11-0-aarch64-shippable-qr/opt.",
},
],
[
["--depth"],
{
"type": int,
"default": None,
"help": "This sets how the change detector should run. "
"Default is None, which is a direct comparison between the "
"revisions. -1 will autocompute the number of revisions to "
"look at between the base, and new. Any other positive integer "
"acts as a maximum number to look at.",
},
],
[
["--base-revision"],
{
"type": str,
"required": True,
"help": "The base revision to compare a new revision to.",
},
],
[
["--new-revision"],
{
"type": str,
"required": True,
"help": "The new revision to compare a base revision to.",
},
],
[
["--base-branch"],
{
"type": str,
"default": "try",
"help": "Branch to search for the base revision.",
},
],
[
["--new-branch"],
{
"type": str,
"default": "try",
"help": "Branch to search for the new revision.",
},
],
[
["--skip-download"],
{
"action": "store_true",
"default": False,
"help": "If set, we won't try to download artifacts again and we'll "
+ "try using what already exists in the output folder.",
},
],
[
["-o", "--overwrite"],
{
"action": "store_true",
"default": False,
"help": "If set, the downloaded task group data will be deleted before "
+ "it gets re-downloaded.",
},
],
]
class ToolingOptions:
args = {
"side-by-side": SideBySideOptions.args,
"change-detector": ChangeDetectorOptions.args,
}
class PerftestToolsArgumentParser(ArgumentParser):
"""%(prog)s [options] [test paths]"""
tool = None
def __init__(self, *args, **kwargs):
ArgumentParser.__init__(
self, usage=self.__doc__, conflict_handler="resolve", **kwargs
)
if PerftestToolsArgumentParser.tool is None:
raise SystemExit("No tool specified, cannot continue parsing")
else:
for name, options in ToolingOptions.args[PerftestToolsArgumentParser.tool]:
self.add_argument(*name, **options)