Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import hashlib
import json
import os
import pathlib
import statistics
import sys
import time
import jsonschema
from mozperftest.layers import Layer
from mozperftest.metrics.common import (
COMMON_ARGS,
filtered_metrics,
get_available_metrics,
)
from mozperftest.metrics.exceptions import PerfherderValidDataError
from mozperftest.metrics.notebook.constant import Constant
from mozperftest.metrics.notebook.transformer import get_transformer
from mozperftest.metrics.utils import has_callable_method, is_number, write_json
from mozperftest.utils import strtobool
PERFHERDER_SCHEMA = pathlib.Path(
"testing", "performance", "common", "performance-artifact-schema.json"
)
class Perfherder(Layer):
"""Output data in the perfherder format."""
name = "perfherder"
activated = False
arguments = COMMON_ARGS
arguments.update({
"stats": {
"action": "store_true",
"default": False,
"help": "If set, browsertime statistics will be reported.",
},
"timestamp": {
"type": float,
"default": None,
"help": (
"Timestamp to use for the perfherder data. Can be the "
"current date or a past date if needed."
),
},
})
def run(self, metadata):
"""Processes the given results into a perfherder-formatted data blob.
If the `--perfherder` flag isn't provided, then the
results won't be processed into a perfherder-data blob. If the
flavor is unknown to us, then we assume that it comes from
browsertime.
XXX If needed, make a way to do flavor-specific processing
:param results list/dict/str: Results to process.
:param perfherder bool: True if results should be processed
into a perfherder-data blob.
:param flavor str: The flavor that is being processed.
"""
prefix = self.get_arg("prefix")
output = self.get_arg("output")
# XXX Make an arugment for exclusions from metrics
# (or go directly to regex's for metrics)
exclusions = None
if not self.get_arg("stats"):
exclusions = ["statistics."]
# Get filtered metrics
transformer = self.get_arg("transformer")
metrics = self.get_arg("metrics")
if not metrics:
metrics = [
{"name": name}
for name in get_available_metrics(
metadata, output, prefix, transformer=transformer
)
]
results, fullsettings = filtered_metrics(
metadata,
output,
prefix,
metrics=metrics,
transformer=transformer,
settings=True,
exclude=exclusions,
split_by=self.get_arg("split-by"),
simplify_names=self.get_arg("simplify-names"),
simplify_exclude=self.get_arg("simplify-exclude"),
)
if not any([results[name] for name in results]):
self.warning("No results left after filtering")
return metadata
app_info = {"name": self.get_arg("app", default="firefox")}
if metadata.binary_version:
app_info["version"] = metadata.binary_version
# converting the metrics list into a mapping where
# keys are the metrics nane
if metrics is not None:
metrics = dict([(m["name"], m) for m in metrics])
else:
metrics = {}
all_perfherder_data = None
for name, res in results.items():
settings = dict(fullsettings[name])
# updating the settings with values provided in metrics, if any
if name in metrics:
settings.update(metrics[name])
# XXX Instead of just passing replicates here, we should build
# up a partial perfherder data blob (with options) and subtest
# overall values.
subtests = []
for r in res:
subtest = {}
vals = [v["value"] for v in r["data"] if is_number(v["value"])]
if vals:
subtest["name"] = r["subtest"]
subtest["replicates"] = vals
subtest["result"] = r
subtests.append(subtest)
perfherder_data = self._build_blob(
subtests,
name=name,
extra_options=settings.get("extraOptions", [])
+ metadata.get_extra_options(),
should_alert=strtobool(settings.get("shouldAlert", False)),
application=app_info,
alert_threshold=float(settings.get("alertThreshold", 2.0)),
alert_severity=settings.get("alertSeverity"),
lower_is_better=strtobool(settings.get("lowerIsBetter", True)),
unit=settings.get("unit", "ms"),
summary=settings.get("value"),
framework=settings.get("framework"),
metrics_info=metrics,
transformer=res[0].get("transformer", None),
)
if all_perfherder_data is None:
all_perfherder_data = perfherder_data
else:
all_perfherder_data["suites"].extend(perfherder_data["suites"])
if prefix:
# If a prefix was given, store it in the perfherder data as well
all_perfherder_data["prefix"] = prefix
timestamp = self.get_arg("timestamp")
if timestamp is not None:
all_perfherder_data["pushTimestamp"] = timestamp
# Validate the final perfherder data blob
with pathlib.Path(metadata._mach_cmd.topsrcdir, PERFHERDER_SCHEMA).open() as f:
schema = json.load(f)
jsonschema.validate(all_perfherder_data, schema)
sequence = int(time.monotonic() * 1000)
payload = json.dumps(all_perfherder_data, sort_keys=True).encode("utf-8")
digest = hashlib.sha1(payload).hexdigest()[:8]
perfherder_file = f"perfherder-data-{sequence}-{digest}.json"
if prefix:
perfherder_file = f"{prefix}-{perfherder_file}"
self.info(
f"Writing perfherder results to {os.path.join(output, perfherder_file)}"
)
# XXX "suites" key error occurs when using self.info so a print
# is being done for now.
# print() will produce a BlockingIOError on large outputs, so we use
# sys.stdout
sys.stdout.write("PERFHERDER_DATA: ")
json.dump(all_perfherder_data, sys.stdout)
sys.stdout.write("\n")
sys.stdout.flush()
metadata.set_output(write_json(all_perfherder_data, output, perfherder_file))
return metadata
def _build_blob(
self,
subtests,
name="browsertime",
test_type="pageload",
extra_options=None,
should_alert=False,
subtest_should_alert=None,
suiteshould_alert=False,
framework=None,
application=None,
alert_threshold=2.0,
alert_severity=None,
lower_is_better=True,
unit="ms",
summary=None,
metrics_info=None,
transformer=None,
):
"""Build a PerfHerder data blob from the given subtests.
NOTE: This is a WIP, see the many TODOs across this file.
Given a dictionary of subtests, and the values. Build up a
perfherder data blob. Note that the naming convention for
these arguments is different then the rest of the scripts
to make it easier to see where they are going to in the perfherder
data.
For the `should_alert` field, if should_alert is True but `subtest_should_alert`
is empty, then all subtests along with the suite will generate alerts.
Otherwise, if the subtest_should_alert contains subtests to alert on, then
only those will alert and nothing else (including the suite). If the
suite value should alert, then set `suiteshould_alert` to True.
:param subtests dict: A dictionary of subtests and the values.
XXX TODO items for subtests:
(1) Allow it to contain replicates and individual settings
for each of the subtests.
(2) The geomean of the replicates will be taken for now,
but it should be made more flexible in some way.
(3) We need some way to handle making multiple suites.
:param name str: Name to give to the suite.
:param test_type str: The type of test that was run.
:param extra_options list: A list of extra options to store.
:param should_alert bool: Whether all values in the suite should
generate alerts or not.
:param subtest_should_alert list: A list of subtests to alert on. If this
is not empty, then it will disable the suite-level alerts.
:param suiteshould_alert bool: Used if `subtest_should_alert` is not
empty, and if True, then the suite-level value will generate
alerts.
:param framework dict: Information about the framework that
is being tested.
:param application dict: Information about the application that
is being tested. Must include name, and optionally a version.
:param alert_threshold float: The change in percentage this
metric must undergo to to generate an alert.
:param alert_severity str: The severity of the alerts produced by this
suite (`critical`, `subcritical`, or `normal`). Left out of the blob
when None, which makes Perfherder fall back to `normal`.
:param lower_is_better bool: If True, then lower values are better
than higher ones.
:param unit str: The unit of the data.
:param summary float: The summary value to use in the perfherder
data blob. By default, the mean of all the subtests will be
used.
:param metrics_info dict: Contains a mapping of metric names to the
options that are used on the metric.
:param transformer str: The name of a predefined tranformer, a module
path to a transform, or a path to the file containing the transformer.
:return dict: The PerfHerder data blob.
"""
if extra_options is None:
extra_options = []
if subtest_should_alert is None:
subtest_should_alert = []
if framework is None:
framework = {"name": "mozperftest"}
if application is None:
application = {"name": "firefox", "version": "9000"}
if metrics_info is None:
metrics_info = {}
# Use the transform to produce a suite value
const = Constant()
tfm_cls = None
transformer_obj = None
if transformer and transformer in const.predefined_transformers:
# A pre-built transformer name was given
tfm_cls = const.predefined_transformers[transformer]
transformer_obj = tfm_cls()
elif transformer is not None:
tfm_cls = get_transformer(transformer)
transformer_obj = tfm_cls()
else:
self.warning(
"No transformer found for this suite. Cannot produce a summary value."
)
perf_subtests = []
suite = {
"name": name,
"type": test_type,
"unit": unit,
"extraOptions": extra_options,
"lowerIsBetter": lower_is_better,
"alertThreshold": alert_threshold,
"shouldAlert": (should_alert and not subtest_should_alert)
or suiteshould_alert,
"subtests": perf_subtests,
}
# Only set when given, otherwise Perfherder defaults it to `normal`
if alert_severity is not None:
suite["alertSeverity"] = alert_severity
perfherder = {
"suites": [suite],
"framework": framework,
"application": application,
}
allvals = []
for subtest_res in subtests:
measurement = subtest_res["name"]
reps = subtest_res["replicates"]
extra_info = subtest_res["result"]
allvals.extend(reps)
if len(reps) == 0:
self.warning(f"No replicates found for {measurement}, skipping")
continue
# Gather extra settings specified from within a metric specification
subtest_alert_threshold = alert_threshold
subtest_alert_severity = alert_severity
subtest_lower_is_better = lower_is_better
subtest_unit = unit
for met in metrics_info:
if met not in measurement:
continue
extra_options.extend(metrics_info[met].get("extraOptions", []))
subtest_alert_threshold = metrics_info[met].get(
"alertThreshold", alert_threshold
)
subtest_unit = metrics_info[met].get("unit", unit)
subtest_lower_is_better = metrics_info[met].get(
"lowerIsBetter", lower_is_better
)
subtest_alert_severity = metrics_info[met].get(
"alertSeverity", alert_severity
)
if metrics_info[met].get("shouldAlert", should_alert):
subtest_should_alert.append(measurement)
break
# The settings specified in the result (from `extra_info`) override
# the settings from the metric info which comes from the `--perfherder-metrics`
# argument. The `or` check ensures that if the extra_info setting is defined,
# but is `None`, then we use the option defined above from the CLI args.
subtest = {
"name": measurement,
"replicates": reps,
"value": extra_info.get("value"),
"unit": extra_info.get("unit") or subtest_unit,
"alertThreshold": extra_info.get("alert_threshold")
or subtest_alert_threshold,
}
# These two need to be done with if statements since they are boolean
# and the `or` check won't work with them (e.g. when a result specified setting
# is False, and the CLI arg is True).
if extra_info.get("shouldAlert") is not None:
subtest["shouldAlert"] = extra_info["shouldAlert"]
else:
subtest["shouldAlert"] = (
should_alert or measurement in subtest_should_alert
)
if extra_info.get("lowerIsBetter") is not None:
subtest["lowerIsBetter"] = extra_info["lowerIsBetter"]
else:
subtest["lowerIsBetter"] = subtest_lower_is_better
if extra_info.get("alertSeverity") is not None:
subtest["alertSeverity"] = extra_info["alertSeverity"]
elif subtest_alert_severity is not None:
subtest["alertSeverity"] = subtest_alert_severity
if has_callable_method(transformer_obj, "subtest_summary"):
subtest["value"] = transformer_obj.subtest_summary(subtest)
if subtest["value"] is None:
subtest["value"] = statistics.mean(reps)
perf_subtests.append(subtest)
if len(allvals) == 0:
raise PerfherderValidDataError(
"Could not build perfherder data blob because no valid data was provided, "
+ "only int/float data is accepted."
)
suite["alertThreshold"] = alert_threshold
suite["extraOptions"] = list(set(suite["extraOptions"]))
if has_callable_method(transformer_obj, "summary"):
val = transformer_obj.summary(suite)
if val is not None:
suite["value"] = val
elif summary:
suite["value"] = summary
return perfherder