Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
from __future__ import absolute_import, print_function, unicode_literals
from collections import defaultdict
import json
import os
import sys
from mach.decorators import CommandArgument, Command, SubCommand
import mozpack.path as mozpath
TOPSRCDIR = os.path.abspath(os.path.join(__file__, "../../../../../"))
class InvalidPathException(Exception):
"""Represents an error due to an invalid path."""
@Command(
"mozbuild-reference",
category="build-dev",
description="View reference documentation on mozbuild files.",
virtualenv_name="docs",
)
@CommandArgument(
"symbol",
default=None,
nargs="*",
help="Symbol to view help on. If not specified, all will be shown.",
)
@CommandArgument(
"--name-only",
"-n",
default=False,
action="store_true",
help="Print symbol names only.",
)
def reference(command_context, symbol, name_only=False):
from mozbuild.sphinx import (
format_module,
function_reference,
special_reference,
variable_reference,
)
import mozbuild.frontend.context as m
if name_only:
for s in sorted(m.VARIABLES.keys()):
print(s)
for s in sorted(m.FUNCTIONS.keys()):
print(s)
for s in sorted(m.SPECIAL_VARIABLES.keys()):
print(s)
return 0
if len(symbol):
for s in symbol:
if s in m.VARIABLES:
for line in variable_reference(s, *m.VARIABLES[s]):
print(line)
continue
elif s in m.FUNCTIONS:
for line in function_reference(s, *m.FUNCTIONS[s]):
print(line)
continue
elif s in m.SPECIAL_VARIABLES:
for line in special_reference(s, *m.SPECIAL_VARIABLES[s]):
print(line)
continue
print("Could not find symbol: %s" % s)
return 1
return 0
for line in format_module(m):
print(line)
return 0
@Command(
"file-info", category="build-dev", description="Query for metadata about files."
)
def file_info(command_context):
"""Show files metadata derived from moz.build files.
moz.build files contain "Files" sub-contexts for declaring metadata
against file patterns. This command suite is used to query that data.
"""
@SubCommand(
"file-info",
"bugzilla-component",
"Show Bugzilla component info for files listed.",
)
@CommandArgument("-r", "--rev", help="Version control revision to look up info from")
@CommandArgument(
"--format",
choices={"json", "plain"},
default="plain",
help="Output format",
dest="fmt",
)
@CommandArgument("paths", nargs="+", help="Paths whose data to query")
def file_info_bugzilla(command_context, paths, rev=None, fmt=None):
"""Show Bugzilla component for a set of files.
Given a requested set of files (which can be specified using
wildcards), print the Bugzilla component for each file.
"""
components = defaultdict(set)
try:
for p, m in _get_files_info(command_context, paths, rev=rev).items():
components[m.get("BUG_COMPONENT")].add(p)
except InvalidPathException as e:
print(e)
return 1
if fmt == "json":
data = {}
for component, files in components.items():
if not component:
continue
for f in files:
data[f] = [component.product, component.component]
json.dump(data, sys.stdout, sort_keys=True, indent=2)
return
elif fmt == "plain":
comp_to_file = sorted(
(
"UNKNOWN"
if component is None
else "%s :: %s" % (component.product, component.component),
sorted(files),
)
for component, files in components.items()
)
for component, files in comp_to_file:
print(component)
for f in files:
print(" %s" % f)
else:
print("unhandled output format: %s" % fmt)
return 1
@SubCommand(
"file-info", "missing-bugzilla", "Show files missing Bugzilla component info"
)
@CommandArgument("-r", "--rev", help="Version control revision to look up info from")
@CommandArgument(
"--format",
choices={"json", "plain"},
dest="fmt",
default="plain",
help="Output format",
)
@CommandArgument("paths", nargs="+", help="Paths whose data to query")
def file_info_missing_bugzilla(command_context, paths, rev=None, fmt=None):
missing = set()
try:
for p, m in _get_files_info(command_context, paths, rev=rev).items():
if "BUG_COMPONENT" not in m:
missing.add(p)
except InvalidPathException as e:
print(e)
return 1
if fmt == "json":
json.dump({"missing": sorted(missing)}, sys.stdout, indent=2)
return
elif fmt == "plain":
for f in sorted(missing):
print(f)
else:
print("unhandled output format: %s" % fmt)
return 1
@SubCommand(
"file-info",
"bugzilla-automation",
"Perform Bugzilla metadata analysis as required for automation",
)
@CommandArgument("out_dir", help="Where to write files")
def bugzilla_automation(command_context, out_dir):
"""Analyze and validate Bugzilla metadata as required by automation.
This will write out JSON and gzipped JSON files for Bugzilla metadata.
The exit code will be non-0 if Bugzilla metadata fails validation.
"""
import gzip
missing_component = set()
seen_components = set()
component_by_path = {}
# TODO operate in VCS space. This requires teaching the VCS reader
# to understand wildcards and/or for the relative path issue in the
# VCS finder to be worked out.
for p, m in sorted(_get_files_info(command_context, ["**"]).items()):
if "BUG_COMPONENT" not in m:
missing_component.add(p)
print(
"FileToBugzillaMappingError: Missing Bugzilla component: "
"%s - Set the BUG_COMPONENT in the moz.build file to fix "
"the issue." % p
)
continue
c = m["BUG_COMPONENT"]
seen_components.add(c)
component_by_path[p] = [c.product, c.component]
print("Examined %d files" % len(component_by_path))
# We also have a normalized versions of the file to components mapping
# that requires far less storage space by eliminating redundant strings.
indexed_components = {
i: [c.product, c.component] for i, c in enumerate(sorted(seen_components))
}
components_index = {tuple(v): k for k, v in indexed_components.items()}
normalized_component = {"components": indexed_components, "paths": {}}
for p, c in component_by_path.items():
d = normalized_component["paths"]
while "/" in p:
base, p = p.split("/", 1)
d = d.setdefault(base, {})
d[p] = components_index[tuple(c)]
if not os.path.exists(out_dir):
os.makedirs(out_dir)
components_json = os.path.join(out_dir, "components.json")
print("Writing %s" % components_json)
with open(components_json, "w") as fh:
json.dump(component_by_path, fh, sort_keys=True, indent=2)
missing_json = os.path.join(out_dir, "missing.json")
print("Writing %s" % missing_json)
with open(missing_json, "w") as fh:
json.dump({"missing": sorted(missing_component)}, fh, indent=2)
indexed_components_json = os.path.join(out_dir, "components-normalized.json")
print("Writing %s" % indexed_components_json)
with open(indexed_components_json, "w") as fh:
# Don't indent so file is as small as possible.
json.dump(normalized_component, fh, sort_keys=True)
# Write compressed versions of JSON files.
for p in (components_json, indexed_components_json, missing_json):
gzip_path = "%s.gz" % p
print("Writing %s" % gzip_path)
with open(p, "rb") as ifh, gzip.open(gzip_path, "wb") as ofh:
while True:
data = ifh.read(32768)
if not data:
break
ofh.write(data)
# Causes CI task to fail if files are missing Bugzilla annotation.
if missing_component:
return 1
def _get_files_info(command_context, paths, rev=None):
reader = command_context.mozbuild_reader(config_mode="empty", vcs_revision=rev)
# Normalize to relative from topsrcdir.
relpaths = []
for p in paths:
a = mozpath.abspath(p)
if not mozpath.basedir(a, [command_context.topsrcdir]):
raise InvalidPathException("path is outside topsrcdir: %s" % p)
relpaths.append(mozpath.relpath(a, command_context.topsrcdir))
# Expand wildcards.
# One variable is for ordering. The other for membership tests.
# (Membership testing on a list can be slow.)
allpaths = []
all_paths_set = set()
for p in relpaths:
if "*" not in p:
if p not in all_paths_set:
if not os.path.exists(mozpath.join(command_context.topsrcdir, p)):
print("(%s does not exist; ignoring)" % p, file=sys.stderr)
continue
all_paths_set.add(p)
allpaths.append(p)
continue
if rev:
raise InvalidPathException("cannot use wildcard in version control mode")
# finder is rooted at / for now.
search = mozpath.join(command_context.topsrcdir, p)[1:]
for path, f in reader.finder.find(search):
path = path[len(command_context.topsrcdir) :]
if path not in all_paths_set:
all_paths_set.add(path)
allpaths.append(path)
return reader.files_info(allpaths)
@SubCommand(
"file-info", "schedules", "Show the combined SCHEDULES for the files listed."
)
@CommandArgument("paths", nargs="+", help="Paths whose data to query")
def file_info_schedules(command_context, paths):
"""Show what is scheduled by the given files.
Given a requested set of files (which can be specified using
wildcards), print the total set of scheduled components.
"""
from mozbuild.frontend.reader import EmptyConfig, BuildReader
config = EmptyConfig(TOPSRCDIR)
reader = BuildReader(config)
schedules = set()
for p, m in reader.files_info(paths).items():
schedules |= set(m["SCHEDULES"].components)
print(", ".join(schedules))