Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# This modules provides functionality for dealing with compiler warnings.
import errno
import io
import json
import os
import re
import mozpack.path as mozpath
import six
from mozbuild.util import hash_file
# Regular expression to strip ANSI color sequences from a string. This is
# needed to properly analyze Clang compiler output, which may be colorized.
# It assumes ANSI escape sequences.
RE_STRIP_COLORS = re.compile(r"\x1b\[[\d;]+m")
# This captures Clang diagnostics with the standard formatting.
RE_CLANG_WARNING_AND_ERROR = re.compile(
r"""
(?P<file>[^:]+)
:
(?P<line>\d+)
:
(?P<column>\d+)
:
\s(?P<type>warning|error):\s
(?P<message>.+)
\[(?P<flag>[^\]]+)
""",
re.X,
)
# This captures Clang-cl warning format.
RE_CLANG_CL_WARNING_AND_ERROR = re.compile(
r"""
(?P<file>.*)
\((?P<line>\d+),(?P<column>\d+)\)
\s?:\s+(?P<type>warning|error):\s
(?P<message>.*)
\[(?P<flag>[^\]]+)
""",
re.X,
)
IN_FILE_INCLUDED_FROM = "In file included from "
class CompilerWarning(dict):
"""Represents an individual compiler warning."""
def __init__(self):
dict.__init__(self)
self["filename"] = None
self["line"] = None
self["column"] = None
self["message"] = None
self["flag"] = None
def copy(self):
"""Returns a copy of this compiler warning."""
w = CompilerWarning()
w.update(self)
return w
# Since we inherit from dict, functools.total_ordering gets confused.
# Thus, we define a key function, a generic comparison, and then
# implement all the rich operators with those; approach is from:
def _cmpkey(self):
return (self["filename"], self["line"], self["column"])
def _compare(self, other, func):
if not isinstance(other, CompilerWarning):
return NotImplemented
return func(self._cmpkey(), other._cmpkey())
def __eq__(self, other):
return self._compare(other, lambda s, o: s == o)
def __neq__(self, other):
return self._compare(other, lambda s, o: s != o)
def __lt__(self, other):
return self._compare(other, lambda s, o: s < o)
def __le__(self, other):
return self._compare(other, lambda s, o: s <= o)
def __gt__(self, other):
return self._compare(other, lambda s, o: s > o)
def __ge__(self, other):
return self._compare(other, lambda s, o: s >= o)
def __hash__(self):
"""Define so this can exist inside a set, etc."""
return hash(tuple(sorted(self.items())))
class WarningsDatabase(object):
"""Holds a collection of warnings.
The warnings database is a semi-intelligent container that holds warnings
encountered during builds.
The warnings database is backed by a JSON file. But, that is transparent
to consumers.
Under most circumstances, the warnings database is insert only. When a
warning is encountered, the caller simply blindly inserts it into the
database. The database figures out whether it is a dupe, etc.
During the course of development, it is common for warnings to change
slightly as source code changes. For example, line numbers will disagree.
The WarningsDatabase handles this by storing the hash of a file a warning
occurred in. At warning insert time, if the hash of the file does not match
what is stored in the database, the existing warnings for that file are
purged from the database.
Callers should periodically prune old, invalid warnings from the database
by calling prune(). A good time to do this is at the end of a build.
"""
def __init__(self):
"""Create an empty database."""
self._files = {}
def __len__(self):
i = 0
for value in self._files.values():
i += len(value["warnings"])
return i
def __iter__(self):
for value in self._files.values():
for warning in value["warnings"]:
yield warning
def __contains__(self, item):
for value in self._files.values():
for warning in value["warnings"]:
if warning == item:
return True
return False
@property
def warnings(self):
"""All the CompilerWarning instances in this database."""
for value in self._files.values():
for w in value["warnings"]:
yield w
def type_counts(self, dirpath=None):
"""Returns a mapping of warning types to their counts."""
types = {}
for value in self._files.values():
for warning in value["warnings"]:
if dirpath and not mozpath.normsep(warning["filename"]).startswith(
dirpath
):
continue
flag = warning["flag"]
count = types.get(flag, 0)
count += 1
types[flag] = count
return types
def has_file(self, filename):
"""Whether we have any warnings for the specified file."""
return filename in self._files
def warnings_for_file(self, filename):
"""Obtain the warnings for the specified file."""
f = self._files.get(filename, {"warnings": []})
for warning in f["warnings"]:
yield warning
def insert(self, warning, compute_hash=True):
assert isinstance(warning, CompilerWarning)
filename = warning["filename"]
new_hash = None
if compute_hash:
new_hash = hash_file(filename)
if filename in self._files:
if new_hash != self._files[filename]["hash"]:
del self._files[filename]
value = self._files.get(
filename,
{
"hash": new_hash,
"warnings": set(),
},
)
value["warnings"].add(warning)
self._files[filename] = value
def prune(self):
"""Prune the contents of the database.
This removes warnings that are no longer valid. A warning is no longer
valid if the file it was in no longer exists or if the content has
changed.
The check for changed content catches the case where a file previously
contained warnings but no longer does.
"""
# Need to calculate up front since we are mutating original object.
filenames = list(six.iterkeys(self._files))
for filename in filenames:
if not os.path.exists(filename):
del self._files[filename]
continue
if self._files[filename]["hash"] is None:
continue
current_hash = hash_file(filename)
if current_hash != self._files[filename]["hash"]:
del self._files[filename]
continue
def serialize(self, fh):
"""Serialize the database to an open file handle."""
obj = {"files": {}}
# All this hackery because JSON can't handle sets.
for k, v in six.iteritems(self._files):
obj["files"][k] = {}
for k2, v2 in six.iteritems(v):
normalized = v2
if isinstance(v2, set):
normalized = list(v2)
obj["files"][k][k2] = normalized
to_write = six.ensure_text(json.dumps(obj, indent=2))
fh.write(to_write)
def deserialize(self, fh):
"""Load serialized content from a handle into the current instance."""
obj = json.load(fh)
self._files = obj["files"]
# Normalize data types.
for filename, value in six.iteritems(self._files):
if "warnings" in value:
normalized = set()
for d in value["warnings"]:
w = CompilerWarning()
w.update(d)
normalized.add(w)
self._files[filename]["warnings"] = normalized
def load_from_file(self, filename):
"""Load the database from a file."""
with io.open(filename, "r", encoding="utf-8") as fh:
self.deserialize(fh)
def save_to_file(self, filename):
"""Save the database to a file."""
try:
# Ensure the directory exists
os.makedirs(os.path.dirname(filename))
except OSError as e:
if e.errno != errno.EEXIST:
raise
with io.open(filename, "w", encoding="utf-8", newline="\n") as fh:
self.serialize(fh)
class WarningsCollector(object):
"""Collects warnings from text data.
Instances of this class receive data (usually the output of compiler
invocations) and parse it into warnings.
The collector works by incrementally receiving data, usually line-by-line
output from the compiler. Therefore, it can maintain state to parse
multi-line warning messages.
"""
def __init__(self, cb, objdir=None):
"""Initialize a new collector.
``cb`` is a callable that is called with a ``CompilerWarning``
instance whenever a new warning is parsed.
``objdir`` is the object directory. Used for normalizing paths.
"""
self.cb = cb
self.objdir = objdir
self.included_from = []
def process_line(self, line):
"""Take a line of text and process it for a warning."""
filtered = RE_STRIP_COLORS.sub("", line)
# Clang warnings in files included from the one(s) being compiled will
# start with "In file included from /path/to/file:line:". Here, we
# record those.
if filtered.startswith(IN_FILE_INCLUDED_FROM):
included_from = filtered[len(IN_FILE_INCLUDED_FROM) :]
parts = included_from.split(":")
self.included_from.append(parts[0])
return
warning = CompilerWarning()
filename = None
# TODO make more efficient so we run minimal regexp matches.
match_clang = RE_CLANG_WARNING_AND_ERROR.match(filtered)
match_clang_cl = RE_CLANG_CL_WARNING_AND_ERROR.match(filtered)
if match_clang:
d = match_clang.groupdict()
filename = d["file"]
warning["type"] = d["type"]
warning["line"] = int(d["line"])
warning["column"] = int(d["column"])
warning["flag"] = d["flag"]
warning["message"] = d["message"].rstrip()
elif match_clang_cl:
d = match_clang_cl.groupdict()
filename = d["file"]
warning["type"] = d["type"]
warning["line"] = int(d["line"])
warning["column"] = int(d["column"])
warning["flag"] = d["flag"]
warning["message"] = d["message"].rstrip()
else:
self.included_from = []
return None
filename = os.path.normpath(filename)
# Sometimes we get relative includes. These typically point to files in
# the object directory. We try to resolve the relative path.
if not os.path.isabs(filename):
filename = self._normalize_relative_path(filename)
warning["filename"] = filename
self.cb(warning)
return warning
def _normalize_relative_path(self, filename):
# Special case files in dist/include.
idx = filename.find("/dist/include")
if idx != -1:
return self.objdir + filename[idx:]
for included_from in self.included_from:
source_dir = os.path.dirname(included_from)
candidate = os.path.normpath(os.path.join(source_dir, filename))
if os.path.exists(candidate):
return candidate
return filename