Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import errno
import json
import os
import sys
from pathlib import Path
from mozfile import which
from mozperftest.layers import Layer
from mozperftest.utils import run_script, silence
METRICS_FIELDS = (
"SpeedIndex",
"FirstVisualChange",
"LastVisualChange",
"VisualProgress",
"videoRecordingStart",
)
class VisualData:
def open_data(self, data):
res = {
"name": "visualmetrics",
"subtest": data["name"],
"data": [
{"file": "visualmetrics", "value": value, "xaxis": xaxis}
for xaxis, value in enumerate(data["values"])
],
}
return res
def transform(self, data):
return data
def merge(self, data):
return data
class VisualMetrics(Layer):
"""Wrapper around Browsertime's visualmetrics.py script"""
name = "visualmetrics"
activated = False
arguments = {}
def setup(self):
self.metrics = {}
self.metrics_fields = []
# making sure we have ffmpeg and imagemagick available
for tool in ("ffmpeg", "convert"):
if sys.platform in ("win32", "msys"):
tool += ".exe"
path = which(tool)
if not path:
raise OSError(errno.ENOENT, f"Could not find {tool}")
def run(self, metadata):
if "VISUALMETRICS_PY" not in os.environ:
raise OSError(
"The VISUALMETRICS_PY environment variable is not set."
"Make sure you run the browsertime layer"
)
path = Path(os.environ["VISUALMETRICS_PY"])
if not path.exists():
raise FileNotFoundError(str(path))
self.visualmetrics = path
treated = 0
for result in metadata.get_results():
result_dir = result.get("results")
if result_dir is None:
continue
result_dir = Path(result_dir)
if not result_dir.is_dir():
continue
browsertime_json = Path(result_dir, "browsertime.json")
if not browsertime_json.exists():
continue
treated += self.run_visual_metrics(browsertime_json)
self.info(f"Treated {treated} videos.")
if len(self.metrics) > 0:
metadata.add_result(
{
"name": metadata.script["name"] + "-vm",
"framework": {"name": "mozperftest"},
"transformer": "mozperftest.metrics.visualmetrics:VisualData",
"results": list(self.metrics.values()),
}
)
# we also extend --perfherder-metrics and --console-metrics if they
# are activated
def add_to_option(name):
existing = self.get_arg(name, [])
for field in self.metrics_fields:
existing.append({"name": field, "unit": "ms"})
self.env.set_arg(name, existing)
if self.get_arg("perfherder"):
add_to_option("perfherder-metrics")
if self.get_arg("console"):
add_to_option("console-metrics")
else:
self.warning("No video was treated.")
return metadata
def run_visual_metrics(self, browsertime_json):
verbose = self.get_arg("verbose")
self.info(f"Looking at {browsertime_json}")
venv = self.mach_cmd.virtualenv_manager
class _display:
def __enter__(self, *args, **kw):
return self
__exit__ = __enter__
may_silence = not verbose and silence or _display
with browsertime_json.open() as f:
browsertime_json_data = json.loads(f.read())
videos = 0
global_options = [
str(self.visualmetrics),
"--orange",
"--perceptual",
"--contentful",
"--force",
"--renderignore",
"5",
"--viewport",
]
if verbose:
global_options += ["-vvv"]
for site in browsertime_json_data:
# collecting metrics from browserScripts
# because it can be used in splitting
for index, bs in enumerate(site["browserScripts"]):
for name, val in bs.items():
if not isinstance(val, (str, int)):
continue
self.append_metrics(index, name, val)
extra = {"lowerIsBetter": True, "unit": "ms"}
for index, video in enumerate(site["files"]["video"]):
videos += 1
video_path = browsertime_json.parent / video
output = "[]"
with may_silence():
res, output = run_script(
venv.python_path,
global_options + ["--video", str(video_path), "--json"],
verbose=verbose,
label="visual metrics",
display=False,
)
if not res:
self.error(f"Failed {res}")
continue
output = output.strip()
if verbose:
self.info(str(output))
try:
output = json.loads(output)
except json.JSONDecodeError:
self.error("Could not read the json output from visualmetrics.py")
continue
for name, value in output.items():
if name.endswith(
"Progress",
):
self._expand_visual_progress(index, name, value, **extra)
else:
self.append_metrics(index, name, value, **extra)
return videos
def _expand_visual_progress(self, index, name, value, **fields):
def _split_percent(val):
# value is of the form "567=94%"
val = val.split("=")
value, percent = val[0].strip(), val[1].strip()
if percent.endswith("%"):
percent = percent[:-1]
return int(percent), int(value)
percents = [_split_percent(elmt) for elmt in value.split(",")]
# we want to keep the first added value for each percent
# so the trick here is to create a dict() with the reversed list
percents = dict(reversed(percents))
# we are keeping the last 5 percents
percents = list(percents.items())
percents.sort()
for percent, value in percents[:5]:
self.append_metrics(index, f"{name}{percent}", value, **fields)
def append_metrics(self, index, name, value, **fields):
if name not in self.metrics_fields:
self.metrics_fields.append(name)
if name not in self.metrics:
self.metrics[name] = {"name": name, "values": []}
self.metrics[name]["values"].append(value)
self.metrics[name].update(**fields)