Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import re
from collections import defaultdict
from fluent.syntax import ast as ftl
from fluent.syntax.serializer import serialize_variant_key
from fluent.syntax.visitor import Visitor
from .base import Checker, CSSCheckMixin
from compare_locales import plurals
MSGS = {
'missing-msg-ref': 'Missing message reference: {ref}',
'missing-term-ref': 'Missing term reference: {ref}',
'obsolete-msg-ref': 'Obsolete message reference: {ref}',
'obsolete-term-ref': 'Obsolete term reference: {ref}',
'duplicate-attribute': 'Attribute "{name}" is duplicated',
'missing-value': 'Missing value',
'obsolete-value': 'Obsolete value',
'missing-attribute': 'Missing attribute: {name}',
'obsolete-attribute': 'Obsolete attribute: {name}',
'duplicate-variant': 'Variant key "{name}" is duplicated',
'missing-plural': 'Plural categories missing: {categories}',
'plain-message': '{message}',
}
def pattern_variants(pattern):
"""Get variants of plain text of a pattern.
For now, just return simple text patterns.
This can be improved to allow for SelectExpressions
of simple text patterns, or even nested expressions, and Literals.
Variants with Variable-, Message-, or TermReferences should be ignored.
"""
elements = pattern.elements
if len(elements) == 1:
if isinstance(elements[0], ftl.TextElement):
return [elements[0].value]
return []
class ReferenceMessageVisitor(Visitor, CSSCheckMixin):
def __init__(self):
# References to Messages, their Attributes, and Terms
# Store reference name and type
self.entry_refs = defaultdict(dict)
# The currently active references
self.refs = {}
# Start with the Entry value (associated with None)
self.entry_refs[None] = self.refs
# If we're a messsage, store if there was a value
self.message_has_value = False
# Map attribute names to positions
self.attribute_positions = {}
# Map of CSS style attribute properties and units
self.css_styles = None
self.css_errors = None
def generic_visit(self, node):
if isinstance(
node,
(ftl.Span, ftl.Annotation, ftl.BaseComment)
):
return
super().generic_visit(node)
def visit_Message(self, node):
if node.value is not None:
self.message_has_value = True
super().generic_visit(node)
def visit_Attribute(self, node):
self.attribute_positions[node.id.name] = node.span.start
old_refs = self.refs
self.refs = self.entry_refs[node.id.name]
super().generic_visit(node)
self.refs = old_refs
if node.id.name != 'style':
return
text_values = pattern_variants(node.value)
if not text_values:
self.css_styles = 'skip'
return
# right now, there's just one possible text value
self.css_styles, self.css_errors = self.parse_css_spec(text_values[0])
def visit_SelectExpression(self, node):
# optimize select expressions to only go through the variants
self.visit(node.variants)
def visit_MessageReference(self, node):
ref = node.id.name
if node.attribute:
ref += '.' + node.attribute.name
self.refs[ref] = 'msg-ref'
def visit_TermReference(self, node):
# only collect term references, but not attributes of terms
if node.attribute:
return
self.refs['-' + node.id.name] = 'term-ref'
class GenericL10nChecks:
'''Helper Mixin for checks shared between Terms and Messages.'''
def check_duplicate_attributes(self, node):
warned = set()
for left in range(len(node.attributes) - 1):
if left in warned:
continue
left_attr = node.attributes[left]
warned_left = False
for right in range(left+1, len(node.attributes)):
right_attr = node.attributes[right]
if left_attr.id.name == right_attr.id.name:
if not warned_left:
warned_left = True
self.messages.append(
(
'warning', left_attr.span.start,
MSGS['duplicate-attribute'].format(
name=left_attr.id.name
)
)
)
warned.add(right)
self.messages.append(
(
'warning', right_attr.span.start,
MSGS['duplicate-attribute'].format(
name=left_attr.id.name
)
)
)
def check_variants(self, variants):
# Check for duplicate variants
warned = set()
for left in range(len(variants) - 1):
if left in warned:
continue
left_key = variants[left].key
key_string = None
for right in range(left+1, len(variants)):
if left_key.equals(variants[right].key):
if key_string is None:
key_string = serialize_variant_key(left_key)
self.messages.append(
(
'warning', left_key.span.start,
MSGS['duplicate-variant'].format(
name=key_string
)
)
)
warned.add(right)
self.messages.append(
(
'warning', variants[right].key.span.start,
MSGS['duplicate-variant'].format(
name=key_string
)
)
)
# Check for plural categories
known_plurals = plurals.get_plural(self.locale)
if known_plurals:
known_plurals = set(known_plurals)
# Ask for known plurals, but check for plurals w/out `other`.
# `other` is used for all kinds of things.
check_plurals = known_plurals.copy()
check_plurals.discard('other')
given_plurals = {serialize_variant_key(v.key) for v in variants}
if given_plurals & check_plurals:
missing_plurals = sorted(known_plurals - given_plurals)
if missing_plurals:
self.messages.append(
(
'warning', variants[0].key.span.start,
MSGS['missing-plural'].format(
categories=', '.join(missing_plurals)
)
)
)
class L10nMessageVisitor(GenericL10nChecks, ReferenceMessageVisitor):
def __init__(self, locale, reference):
super().__init__()
self.locale = locale
# Overload refs to map to sets, just store what we found
# References to Messages, their Attributes, and Terms
# Store reference name and type
self.entry_refs = defaultdict(set)
# The currently active references
self.refs = set()
# Start with the Entry value (associated with None)
self.entry_refs[None] = self.refs
self.reference = reference
self.reference_refs = reference.entry_refs[None]
self.messages = []
def visit_Message(self, node):
self.check_duplicate_attributes(node)
super().visit_Message(node)
if self.message_has_value and not self.reference.message_has_value:
self.messages.append(
('error', node.value.span.start, MSGS['obsolete-value'])
)
if not self.message_has_value and self.reference.message_has_value:
self.messages.append(
('error', 0, MSGS['missing-value'])
)
ref_attrs = set(self.reference.attribute_positions)
l10n_attrs = set(self.attribute_positions)
for missing_attr in ref_attrs - l10n_attrs:
self.messages.append(
(
'error', 0,
MSGS['missing-attribute'].format(name=missing_attr)
)
)
for obs_attr in l10n_attrs - ref_attrs:
self.messages.append(
(
'error', self.attribute_positions[obs_attr],
MSGS['obsolete-attribute'].format(name=obs_attr)
)
)
def visit_Term(self, node):
raise RuntimeError("Should not use L10nMessageVisitor for Terms")
def visit_Attribute(self, node):
old_reference_refs = self.reference_refs
self.reference_refs = self.reference.entry_refs[node.id.name]
super().visit_Attribute(node)
self.reference_refs = old_reference_refs
if node.id.name != 'style' or self.css_styles == 'skip':
return
ref_styles = self.reference.css_styles
if ref_styles in ('skip', None):
# Reference is complex, l10n isn't.
# Let's still validate the css spec.
ref_styles = {}
for cat, msg, pos, _ in self.check_style(
ref_styles,
self.css_styles,
self.css_errors
):
self.messages.append((cat, msg, pos))
def visit_SelectExpression(self, node):
super().visit_SelectExpression(node)
self.check_variants(node.variants)
def visit_MessageReference(self, node):
ref = node.id.name
if node.attribute:
ref += '.' + node.attribute.name
self.refs.add(ref)
self.check_obsolete_ref(node, ref, 'msg-ref')
def visit_TermReference(self, node):
if node.attribute:
return
ref = '-' + node.id.name
self.refs.add(ref)
self.check_obsolete_ref(node, ref, 'term-ref')
def check_obsolete_ref(self, node, ref, ref_type):
if ref not in self.reference_refs:
self.messages.append(
(
'warning', node.span.start,
MSGS['obsolete-' + ref_type].format(ref=ref),
)
)
class TermVisitor(GenericL10nChecks, Visitor):
def __init__(self, locale):
super().__init__()
self.locale = locale
self.messages = []
def generic_visit(self, node):
if isinstance(
node,
(ftl.Span, ftl.Annotation, ftl.BaseComment)
):
return
super().generic_visit(node)
def visit_Message(self, node):
raise RuntimeError("Should not use TermVisitor for Messages")
def visit_Term(self, node):
self.check_duplicate_attributes(node)
super().generic_visit(node)
def visit_SelectExpression(self, node):
super().generic_visit(node)
self.check_variants(node.variants)
class FluentChecker(Checker):
'''Tests to run on Fluent (FTL) files.
'''
pattern = re.compile(r'.*\.ftl')
def check_message(self, ref_entry, l10n_entry):
'''Run checks on localized messages against reference message.'''
ref_data = ReferenceMessageVisitor()
ref_data.visit(ref_entry)
l10n_data = L10nMessageVisitor(self.locale, ref_data)
l10n_data.visit(l10n_entry)
messages = l10n_data.messages
for attr_or_val, refs in ref_data.entry_refs.items():
for ref, ref_type in refs.items():
if ref not in l10n_data.entry_refs[attr_or_val]:
msg = MSGS['missing-' + ref_type].format(ref=ref)
messages.append(('warning', 0, msg))
return messages
def check_term(self, l10n_entry):
'''Check localized terms.'''
l10n_data = TermVisitor(self.locale)
l10n_data.visit(l10n_entry)
return l10n_data.messages
def check(self, refEnt, l10nEnt):
yield from super().check(refEnt, l10nEnt)
l10n_entry = l10nEnt.entry
if isinstance(l10n_entry, ftl.Message):
ref_entry = refEnt.entry
messages = self.check_message(ref_entry, l10n_entry)
elif isinstance(l10n_entry, ftl.Term):
messages = self.check_term(l10n_entry)
messages.sort(key=lambda t: t[1])
for cat, pos, msg in messages:
if pos:
pos = pos - l10n_entry.span.start
yield (cat, pos, msg, 'fluent')