Source code

Revision control

Copy as Markdown

Other Tools

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at
import os
from mozpack import path as mozpath
from mozpack.files import FileFinder
class FilterPath(object):
"""Helper class to make comparing and matching file paths easier."""
def __init__(self, path):
self.path = os.path.normpath(path)
self._finder = None
def finder(self):
if self._finder:
return self._finder
self._finder = FileFinder(mozpath.normsep(self.path))
return self._finder
def ext(self):
return os.path.splitext(self.path)[1].strip(".")
def exists(self):
return os.path.exists(self.path)
def isfile(self):
return os.path.isfile(self.path)
def isdir(self):
return os.path.isdir(self.path)
def join(self, *args):
return FilterPath(os.path.join(self.path, *args))
def match(self, patterns):
a = mozpath.normsep(self.path)
for p in patterns:
if isinstance(p, FilterPath):
p = p.path
p = mozpath.normsep(p)
if mozpath.match(a, p):
return True
return False
def contains(self, other):
"""Return True if other is a subdirectory of self or equals self."""
if isinstance(other, FilterPath):
other = other.path
a = os.path.abspath(self.path)
b = os.path.normpath(os.path.abspath(other))
parts_a = a.split(os.sep)
parts_b = b.split(os.sep)
if len(parts_a) > len(parts_b):
return False
for i, part in enumerate(parts_a):
if part != parts_b[i]:
return False
return True
def __repr__(self):
return repr(self.path)
def collapse(paths, base=None, dotfiles=False):
"""Given an iterable of paths, collapse them into the smallest possible set
of paths that contain the original set (without containing any extra paths).
For example, if directory 'a' contains two files b.txt and c.txt, calling:
collapse(['a/b.txt', 'a/c.txt'])
returns ['a']. But if a third file d.txt also exists, then it will return
['a/b.txt', 'a/c.txt'] since ['a'] would also include that extra file.
:param paths: An iterable of paths (files and directories) to collapse.
:returns: The smallest set of paths (files and directories) that contain
the original set of paths and only the original set.
if not paths:
if not base:
return []
# Need to test whether directory chain is empty. If it is then bubble
# the base back up so that it counts as 'covered'.
for _, _, names in os.walk(base):
if names:
return []
return [base]
if not base:
paths = list(map(mozpath.abspath, paths))
base = mozpath.commonprefix(paths).rstrip("/")
# Make sure `commonprefix` factors in sibling directories that have the
# same prefix in their basenames.
parent = mozpath.dirname(base)
same_prefix = [
p for p in os.listdir(parent) if p.startswith(mozpath.basename(base))
if not os.path.isdir(base) or len(same_prefix) > 1:
base = parent
if base in paths:
return [base]
covered = set()
full = set()
for name in os.listdir(base):
if not dotfiles and name[0] == ".":
path = mozpath.join(base, name)
if path in paths:
# This path was explicitly specified, so just bubble it back up
# without recursing down into it (if it was a directory).
elif os.path.isdir(path):
new_paths = [p for p in paths if p.startswith(path)]
covered.update(collapse(new_paths, base=path, dotfiles=dotfiles))
if full == covered:
# Every file under this base was covered, so we can collapse them all
# up into the base path.
return [base]
return list(covered)
def filterpaths(root, paths, include, exclude=None, extensions=None):
"""Filters a list of paths.
Given a list of paths and some filtering rules, return the set of paths
that should be linted.
:param paths: A starting list of paths to possibly lint.
:param include: A list of paths that should be included (required).
:param exclude: A list of paths that should be excluded (optional).
:param extensions: A list of file extensions which should be considered (optional).
:returns: A tuple containing a list of file paths to lint and a list of
paths to exclude.
def normalize(path):
if "*" not in path and not os.path.isabs(path):
path = os.path.join(root, path)
return FilterPath(path)
# Includes are always paths and should always exist.
include = list(map(normalize, include))
# Exclude paths with and without globs will be handled separately,
# pull them apart now.
exclude = list(map(normalize, exclude or []))
excludepaths = [p for p in exclude if p.exists]
excludeglobs = [p.path for p in exclude if not p.exists]
keep = set()
discard = set()
for path in list(map(normalize, paths)):
# Exclude bad file extensions
if extensions and path.isfile and path.ext not in extensions:
if path.match(excludeglobs):
# First handle include/exclude directives
# that exist (i.e don't have globs)
for inc in include:
# Only excludes that are subdirectories of the include
# path matter.
excs = [e for e in excludepaths if inc.contains(e)]
if path.contains(inc):
# If specified path is an ancestor of include path,
# then lint the include path.
# We can't apply these exclude paths without explicitly
# including every sibling file. Rather than do that,
# just return them and hope the underlying linter will
# deal with them.
elif inc.contains(path):
# If the include path is an ancestor of the specified
# path, then add the specified path only if there are
# no exclude paths in-between them.
if not any(e.contains(path) for e in excs):
discard.update([e for e in excs if path.contains(e)])
# Next expand excludes with globs in them so we can add them to
# the set of files to discard.
for pattern in excludeglobs:
for p, f in path.finder.find(pattern):
return (
[f.path for f in keep if f.exists],
collapse([f.path for f in discard if f.exists]),
def findobject(path):
Find a Python object given a path of the form <modulepath>:<objectpath>.
Conceptually equivalent to
def find_object(modulepath, objectpath):
import <modulepath> as mod
return mod.<objectpath>
if path.count(":") != 1:
raise ValueError(
'python path {!r} does not have the form "module:object"'.format(path)
modulepath, objectpath = path.split(":")
obj = __import__(modulepath)
for a in modulepath.split(".")[1:]:
obj = getattr(obj, a)
for a in objectpath.split("."):
obj = getattr(obj, a)
return obj
def ancestors(path):
while path:
yield path
(path, child) = os.path.split(path)
if child == "":
def get_ancestors_by_name(name, path, root):
"""Returns a list of files called `name` in `path`'s ancestors,
sorted from closest->furthest. This can be useful for finding
relevant configuration files.
configs = []
for path in ancestors(path):
config = os.path.join(path, name)
if os.path.isfile(config):
if path == root:
return configs
def expand_exclusions(paths, config, root):
"""Returns all files that match patterns and aren't excluded.
This is used by some external linters who receive 'batch' files (e.g dirs)
but aren't capable of applying their own exclusions. There is an argument
to be made that this step should just apply to all linters no matter what.
paths (list): List of candidate paths to lint.
config (dict): Linter's config object.
root (str): Root of the repository.
Generator which generates list of paths that weren't excluded.
extensions = [e.lstrip(".") for e in config.get("extensions", [])]
find_dotfiles = config.get("find-dotfiles", False)
def normalize(path):
path = mozpath.normpath(path)
if os.path.isabs(path):
return path
return mozpath.join(root, path)
exclude = list(map(normalize, config.get("exclude", [])))
for path in paths:
path = mozpath.normsep(path)
if os.path.isfile(path):
if any(path.startswith(e) for e in exclude if "*" not in e):
if any(mozpath.match(path, e) for e in exclude if "*" in e):
yield path
ignore = [
e[len(path) :].lstrip("/")
for e in exclude
if mozpath.commonprefix((path, e)) == path
finder = FileFinder(path, ignore=ignore, find_dotfiles=find_dotfiles)
_, ext = os.path.splitext(path)
for ext in extensions:
for p, f in finder.find("**/*.{}".format(ext)):
yield os.path.join(path, p)