Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
""" Creates or updates profiles.
The profile creation works as following:
For each scenario:
- The latest indexed profile is picked on TC, if none we create a fresh profile
- The scenario is done against it
- The profile is uploaded on TC, replacing the previous one as the freshest
For each platform we keep a changelog file that keep track of each update
with the Task ID. That offers us the ability to get a profile from a specific
date in the past.
Artifacts are staying in TaskCluster for 3 months, and then they are removed,
so the oldest profile we can get is 3 months old. Profiles are being updated
continuously, so even after 3 months they are still getting "older".
When Firefox changes its version, profiles from the previous version
should work as expected. Each profile tarball comes with a metadata file
that keep track of the Firefox version that was used and the profile age.
"""
import os
import tempfile
import shutil
from arsenic import get_session
from arsenic.browsers import Firefox
from condprof.util import fresh_profile, logger, obfuscate_file, obfuscate, get_version
from condprof.helpers import close_extra_windows
from condprof.scenarii import scenarii
from condprof.client import get_profile, ProfileNotFoundError
from condprof.archiver import Archiver
from condprof.customization import get_customization
from condprof.metadata import Metadata
START, INIT_GECKODRIVER, START_SESSION, START_SCENARIO = range(4)
class ProfileCreator:
def __init__(
self,
scenario,
customization,
archive,
changelog,
force_new,
env,
skip_logs=False,
remote_test_root="/sdcard/test_root/",
):
self.env = env
self.scenario = scenario
self.customization = customization
self.archive = archive
self.changelog = changelog
self.force_new = force_new
self.skip_logs = skip_logs
self.remote_test_root = remote_test_root
self.customization_data = get_customization(customization)
self.tmp_dir = None
# Make a temporary directory for the logs if an
# archive dir is not provided
if not self.archive:
self.tmp_dir = tempfile.mkdtemp()
def _log_filename(self, name):
filename = "%s-%s-%s.log" % (
name,
self.scenario,
self.customization_data["name"],
)
return os.path.join(self.archive or self.tmp_dir, filename)
async def run(self, headless=True):
logger.info(
"Building %s x %s" % (self.scenario, self.customization_data["name"])
)
if self.scenario in self.customization_data.get("ignore_scenario", []):
logger.info("Skipping (ignored scenario in that customization)")
return
filter_by_platform = self.customization_data.get("platforms")
if filter_by_platform and self.env.target_platform not in filter_by_platform:
logger.info("Skipping (ignored platform in that customization)")
return
with self.env.get_device(
2828, verbose=True, remote_test_root=self.remote_test_root
) as device:
try:
with self.env.get_browser():
metadata = await self.build_profile(device, headless)
except Exception:
raise
finally:
if not self.skip_logs:
self.env.dump_logs()
if not self.archive:
return
logger.info("Creating generic archive")
names = ["profile-%(platform)s-%(name)s-%(customization)s.tgz"]
if metadata["name"] == "full" and metadata["customization"] == "default":
names = [
"profile-%(platform)s-%(name)s-%(customization)s.tgz",
"profile-v%(version)s-%(platform)s-%(name)s-%(customization)s.tgz",
]
for name in names:
# remove `cache` from profile
shutil.rmtree(os.path.join(self.env.profile, "cache"), ignore_errors=True)
shutil.rmtree(os.path.join(self.env.profile, "cache2"), ignore_errors=True)
archiver = Archiver(self.scenario, self.env.profile, self.archive)
# the archive name is of the form
# profile[-vXYZ.x]-<platform>-<scenario>-<customization>.tgz
name = name % metadata
archive_name = os.path.join(self.archive, name)
dir = os.path.dirname(archive_name)
if not os.path.exists(dir):
os.makedirs(dir)
archiver.create_archive(archive_name)
logger.info("Archive created at %s" % archive_name)
statinfo = os.stat(archive_name)
logger.info("Current size is %d" % statinfo.st_size)
logger.info("Extracting logs")
if "logs" in metadata:
logs = metadata.pop("logs")
for prefix, prefixed_logs in logs.items():
for log in prefixed_logs:
content = obfuscate(log["content"])[1]
with open(os.path.join(dir, prefix + "-" + log["name"]), "wb") as f:
f.write(content.encode("utf-8"))
if metadata.get("result", 0) != 0:
logger.info("The scenario returned a bad exit code")
raise Exception(metadata.get("result_message", "scenario error"))
self.changelog.append("update", **metadata)
async def build_profile(self, device, headless):
scenario = self.scenario
profile = self.env.profile
customization_data = self.customization_data
scenario_func = scenarii[scenario]
if scenario in customization_data.get("scenario", {}):
options = customization_data["scenario"][scenario]
logger.info("Loaded options for that scenario %s" % str(options))
else:
options = {}
# Adding general options
options["platform"] = self.env.target_platform
if not self.force_new:
try:
custom_name = customization_data["name"]
get_profile(profile, self.env.target_platform, scenario, custom_name)
except ProfileNotFoundError:
# XXX we'll use a fresh profile for now
fresh_profile(profile, customization_data)
else:
fresh_profile(profile, customization_data)
logger.info("Updating profile located at %r" % profile)
metadata = Metadata(profile)
logger.info("Starting the Gecko app...")
adb_logs = self._log_filename("adb")
self.env.prepare(logfile=adb_logs)
geckodriver_logs = self._log_filename("geckodriver")
logger.info("Writing geckodriver logs in %s" % geckodriver_logs)
step = START
try:
firefox_instance = Firefox(**self.env.get_browser_args(headless))
step = INIT_GECKODRIVER
with open(geckodriver_logs, "w") as glog:
geckodriver = self.env.get_geckodriver(log_file=glog)
step = START_SESSION
async with get_session(geckodriver, firefox_instance) as session:
step = START_SCENARIO
self.env.check_session(session)
logger.info("Running the %s scenario" % scenario)
metadata.update(await scenario_func(session, options))
logger.info("%s scenario done." % scenario)
await close_extra_windows(session)
except Exception:
logger.error("%s scenario broke!" % scenario)
if step == START:
logger.info("Could not initialize the browser")
elif step == INIT_GECKODRIVER:
logger.info("Could not initialize Geckodriver")
elif step == START_SESSION:
logger.info(
"Could not start the session, check %s first" % geckodriver_logs
)
else:
logger.info("Could not run the scenario, probably a faulty scenario")
raise
finally:
self.env.stop_browser()
for logfile in (adb_logs, geckodriver_logs):
if os.path.exists(logfile):
obfuscate_file(logfile)
self.env.collect_profile()
# writing metadata
metadata.write(
name=self.scenario,
customization=self.customization_data["name"],
version=self.env.get_browser_version(),
platform=self.env.target_platform,
)
logger.info("Profile at %s.\nDone." % profile)
return metadata