Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import os
import tarfile
from datetime import datetime, timedelta
import mozversioncontrol
import requests
from mozbuild.artifact_builds import GECKO_ESR_TREES, GECKO_NIGHTLY_TREES
from mozbuild.util import get_root_url
try:
from cStringIO import StringIO as BytesIO
except ImportError:
from io import BytesIO
HEADERS = {"User-Agent": "wpt manifest download"}
ARTIFACT_PATH = "/artifacts/public/manifests.tar.gz"
REPOS = {tree: tree.split("/")[-1] for tree in GECKO_NIGHTLY_TREES + GECKO_ESR_TREES}
def index_url(name):
return f"{get_root_url()}/api/index/v1/task/gecko.v2.{name}.source.manifest-upload"
def default_url():
return index_url("mozilla-central.latest") + ARTIFACT_PATH
def get(logger, url, **kwargs):
logger.debug(url)
if "headers" not in kwargs:
kwargs["headers"] = HEADERS
kwargs.setdefault("timeout", 30)
return requests.get(url, **kwargs)
def abs_path(path):
return os.path.abspath(os.path.expanduser(path))
def manifest_url(logger, repo_root):
try:
repo = mozversioncontrol.get_repository_object(repo_root)
except mozversioncontrol.InvalidRepoPath:
logger.warning("No VCS found for path %s" % repo_root)
return latest_url(logger)
if repo.name in ("git", "jj"):
return git_taskcluster_url(logger, repo.get_ancestors(limit=20))
return taskcluster_url(
logger,
repo.get_commits(
limit=50,
follow=[
"testing/web-platform/tests",
"testing/web-platform/mozilla/tests",
],
),
)
def should_download(logger, manifest_paths, rebuild_time=timedelta(days=5)):
# TODO: Improve logic for when to download. Maybe if x revisions behind?
for manifest_path in manifest_paths:
if not os.path.exists(manifest_path):
return True
mtime = datetime.fromtimestamp(os.path.getmtime(manifest_path))
if mtime < datetime.now() - rebuild_time:
return True
if os.path.getsize(manifest_path) == 0:
return True
logger.info("Skipping manifest download because existing file is recent")
return False
def git_taskcluster_url(logger, commits):
for revision in commits:
for index_name in REPOS.values():
tc_index_url = index_url(f"{index_name}.revision.{revision}")
try:
req = get(logger, tc_index_url)
except requests.exceptions.RequestException:
return default_url()
if req.status_code == 200:
return tc_index_url + ARTIFACT_PATH
if req.status_code != 404:
return default_url()
return latest_url(logger)
def taskcluster_url(logger, commits):
cset_url = (
"changeset={changeset}&version=2&tipsonly=1"
)
for revision in commits:
req = None
if revision == 40 * "0":
continue
for repo_path, index_name in REPOS.items():
try:
req_headers = HEADERS.copy()
req_headers.update({"Accept": "application/json"})
req = get(
logger,
cset_url.format(changeset=revision, repo=repo_path),
headers=req_headers,
)
req.raise_for_status()
except requests.exceptions.RequestException:
if req is not None and req.status_code == 404:
# The API returns a 404 if it can't find a changeset for the revision.
logger.debug("%s not found in %s" % (revision, repo_path))
continue
else:
return default_url()
result = req.json()
pushes = result["pushes"]
if not pushes:
logger.debug("Error reading response; 'pushes' key not found")
continue
[cset] = next(iter(pushes.values()))["changesets"]
tc_index_url = index_url(f"{index_name}.revision.{cset}")
try:
req = get(logger, tc_index_url)
except requests.exceptions.RequestException:
return default_url()
if req.status_code == 200:
return tc_index_url + ARTIFACT_PATH
return latest_url(logger)
def latest_url(logger):
logger.info(
"Can't find a commit-specific manifest so just using the most recent one"
)
return default_url()
def download_manifest(logger, test_paths, url_func, force=False):
manifest_paths = [
(item["manifest_path"] if isinstance(item, dict) else item.manifest_path)
for item in test_paths.values()
]
if not force and not should_download(logger, manifest_paths):
return True
url = url_func()
if not url:
logger.warning("No generated manifest found")
return False
logger.info("Downloading manifest from %s" % url)
try:
req = get(logger, url)
except Exception:
logger.warning("Downloading pregenerated manifest failed")
return False
if req.status_code != 200:
logger.warning(
"Downloading pregenerated manifest failed; got "
"HTTP status %d" % req.status_code
)
return False
tar = tarfile.open(mode="r:gz", fileobj=BytesIO(req.content))
for paths in test_paths.values():
manifest_rel_path = (
paths["manifest_rel_path"]
if isinstance(paths, dict)
else paths.manifest_rel_path
)
manifest_path = (
paths["manifest_path"] if isinstance(paths, dict) else paths.manifest_path
)
try:
member = tar.getmember(manifest_rel_path.replace(os.path.sep, "/"))
except KeyError:
logger.warning("Failed to find downloaded manifest %s" % manifest_rel_path)
else:
try:
logger.debug("Unpacking %s to %s" % (member.name, manifest_path))
src = tar.extractfile(member)
with open(manifest_path, "wb") as dest:
dest.write(src.read())
src.close()
except OSError:
import traceback
logger.warning(
"Failed to decompress %s:\n%s"
% (manifest_rel_path, traceback.format_exc())
)
return False
os.utime(manifest_path, None)
return True
def download_from_taskcluster(logger, repo_root, test_paths, force=False):
return download_manifest(
logger,
test_paths,
lambda: manifest_url(logger, repo_root),
force,
)