Source code
Revision control
Copy as Markdown
Other Tools
"""Utility functions for Raptor"""
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import importlib
import inspect
import os
import pathlib
from collections.abc import Iterable
from distutils.util import strtobool
import yaml
from base_python_support import BasePythonSupport
from logger.logger import RaptorLogger
from mozgeckoprofiler import view_gecko_profile
LOG = RaptorLogger(component="raptor-utils")
here = os.path.dirname(os.path.realpath(__file__))
external_tools_path = os.environ.get("EXTERNALTOOLSPATH", None)
if external_tools_path is not None:
# running in production via mozharness
TOOLTOOL_PATH = os.path.join(external_tools_path, "tooltool.py")
def flatten(data, parent_dir, sep="/"):
"""
Converts a dictionary with nested entries like this
{
"dict1": {
"dict2": {
"key1": value1,
"key2": value2,
...
},
...
},
...
"dict3": {
"key3": value3,
"key4": value4,
...
}
...
}
to a "flattened" dictionary like this that has no nested entries:
{
"dict1-dict2-key1": value1,
"dict1-dict2-key2": value2,
...
"dict3-key3": value3,
"dict3-key4": value4,
...
}
:param Iterable data : json data.
:param tuple parent_dir: json fields.
:return dict: {subtest: value}
"""
result = {}
if not data:
return result
if isinstance(data, list):
for item in data:
for k, v in flatten(item, parent_dir, sep=sep).items():
result.setdefault(k, []).extend(v)
if isinstance(data, dict):
for k, v in data.items():
current_dir = parent_dir + (k,)
subtest = sep.join(current_dir)
if isinstance(v, Iterable) and not isinstance(v, str):
for x, y in flatten(v, current_dir, sep=sep).items():
result.setdefault(x, []).extend(y)
elif v or v == 0:
result.setdefault(subtest, []).append(v)
return result
def transform_platform(str_to_transform, config_platform, config_processor=None):
"""Transform platform name i.e. 'mitmproxy-rel-bin-{platform}.manifest'
transforms to 'mitmproxy-rel-bin-osx.manifest'.
Also transform '{x64}' if needed for 64 bit / win 10"""
if "{platform}" not in str_to_transform and "{x64}" not in str_to_transform:
return str_to_transform
if "win" in config_platform:
platform_id = "win"
elif config_platform == "mac":
platform_id = "osx"
else:
platform_id = "linux64"
if "{platform}" in str_to_transform:
str_to_transform = str_to_transform.replace("{platform}", platform_id)
if "{x64}" in str_to_transform and config_processor is not None:
if "x86_64" in config_processor:
str_to_transform = str_to_transform.replace("{x64}", "_x64")
else:
str_to_transform = str_to_transform.replace("{x64}", "")
return str_to_transform
def transform_subtest(str_to_transform, subtest_name):
"""Transform subtest name i.e. 'mitm5-linux-firefox-{subtest}.manifest'
transforms to 'mitm5-linux-firefox-amazon.manifest'."""
if "{subtest}" not in str_to_transform:
return str_to_transform
return str_to_transform.replace("{subtest}", subtest_name)
def view_gecko_profile_from_raptor():
# automatically load the latest raptor gecko-profile archive in profiler.firefox.com
LOG_GECKO = RaptorLogger(component="raptor-view-gecko-profile")
profile_zip_path = os.environ.get("RAPTOR_LATEST_GECKO_PROFILE_ARCHIVE", None)
if profile_zip_path is None or not os.path.exists(profile_zip_path):
LOG_GECKO.info(
"No local raptor gecko profiles were found so not "
"launching profiler.firefox.com"
)
return
LOG_GECKO.info("Profile saved locally to: %s" % profile_zip_path)
view_gecko_profile(profile_zip_path)
def write_yml_file(yml_file, yml_data):
# write provided data to specified local yaml file
LOG.info("writing %s to %s" % (yml_data, yml_file))
try:
with open(yml_file, "w") as outfile:
yaml.dump(yml_data, outfile, default_flow_style=False)
except Exception as e:
LOG.critical("failed to write yaml file, exeption: %s" % e)
def bool_from_str(boolean_string):
return bool(strtobool(boolean_string))
def import_support_class(path):
"""This function returns a Transformer class with the given path.
:param str path: The path points to the custom transformer.
:param bool ret_members: If true then return inspect.getmembers().
:return Transformer if not ret_members else inspect.getmembers().
"""
file = pathlib.Path(path)
if not file.exists():
raise Exception(f"The support_class path {path} does not exist.")
# Importing a source file directly
spec = importlib.util.spec_from_file_location(name=file.name, location=path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
# TODO: Add checks for methods that can be used for results/output parsing
members = inspect.getmembers(
module,
lambda c: inspect.isclass(c)
and c != BasePythonSupport
and issubclass(c, BasePythonSupport),
)
if not members:
raise Exception(
f"The path {path} was found but it was not a valid support_class."
)
return members[0][-1]