Source code
Revision control
Copy as Markdown
Other Tools
//! JavaScript lexer.
use crate::numeric_value::{parse_float, parse_int, NumericLiteralBase};
use crate::parser::Parser;
use crate::unicode::{is_id_continue, is_id_start};
use ast::arena;
use ast::source_atom_set::{CommonSourceAtomSetIndices, SourceAtomSet};
use ast::source_slice_list::SourceSliceList;
use ast::SourceLocation;
use bumpalo::{collections::String, Bump};
use generated_parser::{ParseError, Result, TerminalId, Token, TokenValue};
use std::cell::RefCell;
use std::convert::TryFrom;
use std::rc::Rc;
use std::str::Chars;
pub struct Lexer<'alloc> {
allocator: &'alloc Bump,
/// Next token to be returned.
token: arena::Box<'alloc, Token>,
/// Length of the input text, in UTF-8 bytes.
source_length: usize,
/// Iterator over the remaining not-yet-parsed input.
chars: Chars<'alloc>,
atoms: Rc<RefCell<SourceAtomSet<'alloc>>>,
slices: Rc<RefCell<SourceSliceList<'alloc>>>,
}
enum NumericResult {
Int {
base: NumericLiteralBase,
},
Float,
BigInt {
#[allow(dead_code)]
base: NumericLiteralBase,
},
}
impl<'alloc> Lexer<'alloc> {
pub fn new(
allocator: &'alloc Bump,
chars: Chars<'alloc>,
atoms: Rc<RefCell<SourceAtomSet<'alloc>>>,
slices: Rc<RefCell<SourceSliceList<'alloc>>>,
) -> Lexer<'alloc> {
Self::with_offset(allocator, chars, 0, atoms, slices)
}
/// Create a lexer for a part of a JS script or module. `offset` is the
/// total length of all previous parts, in bytes; source locations for
/// tokens created by the new lexer start counting from this number.
pub fn with_offset(
allocator: &'alloc Bump,
chars: Chars<'alloc>,
offset: usize,
atoms: Rc<RefCell<SourceAtomSet<'alloc>>>,
slices: Rc<RefCell<SourceSliceList<'alloc>>>,
) -> Lexer<'alloc> {
let source_length = offset + chars.as_str().len();
let mut token = arena::alloc(allocator, new_token());
token.is_on_new_line = true;
Lexer {
allocator,
token,
source_length,
chars,
atoms,
slices,
}
}
fn is_looking_at(&self, s: &str) -> bool {
self.chars.as_str().starts_with(s)
}
pub fn offset(&self) -> usize {
self.source_length - self.chars.as_str().len()
}
fn peek(&self) -> Option<char> {
self.chars.as_str().chars().next()
}
fn double_peek(&self) -> Option<char> {
let mut chars = self.chars.as_str().chars();
chars.next();
chars.next()
}
fn set_result(
&mut self,
terminal_id: TerminalId,
loc: SourceLocation,
value: TokenValue,
) -> Result<'alloc, ()> {
self.token.terminal_id = terminal_id;
self.token.loc = loc;
self.token.value = value;
Ok(())
}
#[inline]
pub fn next<'parser>(
&mut self,
parser: &Parser<'parser>,
) -> Result<'alloc, arena::Box<'alloc, Token>> {
let mut next_token = arena::alloc_with(self.allocator, || new_token());
self.advance_impl(parser)?;
std::mem::swap(&mut self.token, &mut next_token);
Ok(next_token)
}
fn unexpected_err(&mut self) -> ParseError<'alloc> {
if let Some(ch) = self.peek() {
ParseError::IllegalCharacter(ch)
} else {
ParseError::UnexpectedEnd
}
}
}
/// Returns an empty token which is meant as a place holder to be mutated later.
fn new_token() -> Token {
Token::basic_token(TerminalId::End, SourceLocation::default())
}
// ----------------------------------------------------------------------------
// 11.1 Unicode Format-Control Characters
/// U+200C ZERO WIDTH NON-JOINER, abbreviated in the spec as <ZWNJ>.
/// Specially permitted in identifiers.
const ZWNJ: char = '\u{200c}';
/// U+200D ZERO WIDTH JOINER, abbreviated as <ZWJ>.
/// Specially permitted in identifiers.
const ZWJ: char = '\u{200d}';
/// U+FEFF ZERO WIDTH NO-BREAK SPACE, abbreviated <ZWNBSP>.
/// Considered a whitespace character in JS.
const ZWNBSP: char = '\u{feff}';
// ----------------------------------------------------------------------------
// 11.2 White Space
/// U+0009 CHARACTER TABULATION, abbreviated <TAB>.
const TAB: char = '\u{9}';
/// U+000B VERTICAL TAB, abbreviated <VT>.
const VT: char = '\u{b}';
/// U+000C FORM FEED, abbreviated <FF>.
const FF: char = '\u{c}';
/// U+0020 SPACE, abbreviated <SP>.
const SP: char = '\u{20}';
/// U+00A0 NON-BREAKING SPACE, abbreviated <NBSP>.
const NBSP: char = '\u{a0}';
// ----------------------------------------------------------------------------
// 11.3 Line Terminators
/// U+000A LINE FEED, abbreviated in the spec as <LF>.
const LF: char = '\u{a}';
/// U+000D CARRIAGE RETURN, abbreviated in the spec as <CR>.
const CR: char = '\u{d}';
/// U+2028 LINE SEPARATOR, abbreviated <LS>.
const LS: char = '\u{2028}';
/// U+2029 PARAGRAPH SEPARATOR, abbreviated <PS>.
const PS: char = '\u{2029}';
// ----------------------------------------------------------------------------
// 11.4 Comments
//
// Comment::
// MultiLineComment
// SingleLineComment
impl<'alloc> Lexer<'alloc> {
/// Skip a *MultiLineComment*.
///
/// ```text
/// MultiLineComment ::
/// `/*` MultiLineCommentChars? `*/`
///
/// MultiLineCommentChars ::
/// MultiLineNotAsteriskChar MultiLineCommentChars?
/// `*` PostAsteriskCommentChars?
///
/// PostAsteriskCommentChars ::
/// MultiLineNotForwardSlashOrAsteriskChar MultiLineCommentChars?
/// `*` PostAsteriskCommentChars?
///
/// MultiLineNotAsteriskChar ::
/// SourceCharacter but not `*`
///
/// MultiLineNotForwardSlashOrAsteriskChar ::
/// SourceCharacter but not one of `/` or `*`
/// ```
///
/// (B.1.3 splits MultiLineComment into two nonterminals: MultiLineComment
/// and SingleLineDelimitedComment. The point of that is to help specify
/// that a SingleLineHTMLCloseComment must occur at the start of a line. We
/// use `is_on_new_line` for that.)
///
fn skip_multi_line_comment(&mut self, builder: &mut AutoCow<'alloc>) -> Result<'alloc, ()> {
while let Some(ch) = self.chars.next() {
match ch {
'*' if self.peek() == Some('/') => {
self.chars.next();
*builder = AutoCow::new(&self);
return Ok(());
}
CR | LF | PS | LS => {
self.token.is_on_new_line = true;
}
_ => {}
}
}
Err(ParseError::UnterminatedMultiLineComment.into())
}
/// Skip a *SingleLineComment* and the following *LineTerminatorSequence*,
/// if any.
///
/// ```text
/// SingleLineComment ::
/// `//` SingleLineCommentChars?
///
/// SingleLineCommentChars ::
/// SingleLineCommentChar SingleLineCommentChars?
///
/// SingleLineCommentChar ::
/// SourceCharacter but not LineTerminator
/// ```
fn skip_single_line_comment(&mut self, builder: &mut AutoCow<'alloc>) {
while let Some(ch) = self.chars.next() {
match ch {
CR | LF | LS | PS => break,
_ => continue,
}
}
*builder = AutoCow::new(&self);
self.token.is_on_new_line = true;
}
}
// ----------------------------------------------------------------------------
// 11.6 Names and Keywords
/// True if `c` is a one-character *IdentifierStart*.
///
/// ```text
/// IdentifierStart ::
/// UnicodeIDStart
/// `$`
/// `_`
/// `\` UnicodeEscapeSequence
///
/// UnicodeIDStart ::
/// > any Unicode code point with the Unicode property "ID_Start"
/// ```
fn is_identifier_start(c: char) -> bool {
// Escaped case is handled separately.
if c.is_ascii() {
c == '$' || c == '_' || c.is_ascii_alphabetic()
} else {
is_id_start(c)
}
}
/// True if `c` is a one-character *IdentifierPart*.
///
/// ```text
/// IdentifierPart ::
/// UnicodeIDContinue
/// `$`
/// `\` UnicodeEscapeSequence
/// <ZWNJ>
/// <ZWJ>
///
/// UnicodeIDContinue ::
/// > any Unicode code point with the Unicode property "ID_Continue"
/// ```
fn is_identifier_part(c: char) -> bool {
// Escaped case is handled separately.
if c.is_ascii() {
c == '$' || c == '_' || c.is_ascii_alphanumeric()
} else {
is_id_continue(c) || c == ZWNJ || c == ZWJ
}
}
impl<'alloc> Lexer<'alloc> {
/// Scan the rest of an IdentifierName, having already parsed the initial
/// IdentifierStart and stored it in `builder`.
///
/// On success, this returns `Ok((has_escapes, str))`, where `has_escapes`
/// is true if the identifier contained any UnicodeEscapeSequences, and
/// `str` is the un-escaped IdentifierName, including the IdentifierStart,
/// on success.
///
/// ```text
/// IdentifierName ::
/// IdentifierStart
/// IdentifierName IdentifierPart
/// ```
fn identifier_name_tail(
&mut self,
mut builder: AutoCow<'alloc>,
) -> Result<'alloc, (bool, &'alloc str)> {
while let Some(ch) = self.peek() {
if !is_identifier_part(ch) {
if ch == '\\' {
self.chars.next();
builder.force_allocation_without_current_ascii_char(&self);
let value = self.unicode_escape_sequence_after_backslash()?;
if !is_identifier_part(value) {
return Err(ParseError::InvalidEscapeSequence.into());
}
builder.push_different(value);
continue;
}
break;
}
self.chars.next();
builder.push_matching(ch);
}
let has_different = builder.has_different();
Ok((has_different, builder.finish(&self)))
}
fn identifier_name(&mut self, mut builder: AutoCow<'alloc>) -> Result<'alloc, &'alloc str> {
match self.chars.next() {
None => {
return Err(ParseError::UnexpectedEnd.into());
}
Some(c) => {
match c {
'$' | '_' | 'a'..='z' | 'A'..='Z' => {
builder.push_matching(c);
}
'\\' => {
builder.force_allocation_without_current_ascii_char(&self);
let value = self.unicode_escape_sequence_after_backslash()?;
if !is_identifier_start(value) {
return Err(ParseError::IllegalCharacter(value).into());
}
builder.push_different(value);
}
other if is_identifier_start(other) => {
builder.push_matching(other);
}
other => {
return Err(ParseError::IllegalCharacter(other).into());
}
}
self.identifier_name_tail(builder)
.map(|(_has_escapes, name)| name)
}
}
}
/// Finish scanning an *IdentifierName* or keyword, having already scanned
/// the *IdentifierStart* and pushed it to `builder`.
///
/// `start` is the offset of the *IdentifierStart*.
///
/// The lexer doesn't know the syntactic context, so it always identifies
/// possible keywords. It's up to the parser to understand that, for
/// example, `TerminalId::If` is not a keyword when it's used as a property
/// or method name.
///
/// If the source string contains no escape and it matches to possible
/// keywords (including contextual keywords), the result is corresponding
/// `TerminalId`. For example, if the source string is "yield", the result
/// is `TerminalId::Yield`.
///
/// If the source string contains no escape sequence and also it doesn't
/// match to any possible keywords, the result is `TerminalId::Name`.
///
/// If the source string contains at least one escape sequence,
/// the result is always `TerminalId::NameWithEscape`, regardless of the
/// StringValue of it. For example, if the source string is "\u{79}ield",
/// the result is `TerminalId::NameWithEscape`, and the StringValue is
/// "yield".
fn identifier_tail(&mut self, start: usize, builder: AutoCow<'alloc>) -> Result<'alloc, ()> {
let (has_different, text) = self.identifier_name_tail(builder)?;
//
// keywords in the grammar match literal sequences of specific
// SourceCharacter elements. A code point in a keyword cannot be
// expressed by a `\` UnicodeEscapeSequence.
let (id, value) = if has_different {
// Always return `NameWithEscape`.
//
// Error check against reserved word should be handled in the
// consumer.
(TerminalId::NameWithEscape, self.string_to_token_value(text))
} else {
match &text as &str {
"as" => (
TerminalId::As,
TokenValue::Atom(CommonSourceAtomSetIndices::as_()),
),
"async" => {
/*
(
TerminalId::Async,
TokenValue::Atom(CommonSourceAtomSetIndices::async_()),
),
*/
return Err(ParseError::NotImplemented(
"async cannot be handled in parser due to multiple lookahead",
)
.into());
}
"await" => {
/*
(
TerminalId::Await,
TokenValue::Atom(CommonSourceAtomSetIndices::await_()),
),
*/
return Err(
ParseError::NotImplemented("await cannot be handled in parser").into(),
);
}
"break" => (
TerminalId::Break,
TokenValue::Atom(CommonSourceAtomSetIndices::break_()),
),
"case" => (
TerminalId::Case,
TokenValue::Atom(CommonSourceAtomSetIndices::case()),
),
"catch" => (
TerminalId::Catch,
TokenValue::Atom(CommonSourceAtomSetIndices::catch()),
),
"class" => (
TerminalId::Class,
TokenValue::Atom(CommonSourceAtomSetIndices::class()),
),
"const" => (
TerminalId::Const,
TokenValue::Atom(CommonSourceAtomSetIndices::const_()),
),
"continue" => (
TerminalId::Continue,
TokenValue::Atom(CommonSourceAtomSetIndices::continue_()),
),
"debugger" => (
TerminalId::Debugger,
TokenValue::Atom(CommonSourceAtomSetIndices::debugger()),
),
"default" => (
TerminalId::Default,
TokenValue::Atom(CommonSourceAtomSetIndices::default()),
),
"delete" => (
TerminalId::Delete,
TokenValue::Atom(CommonSourceAtomSetIndices::delete()),
),
"do" => (
TerminalId::Do,
TokenValue::Atom(CommonSourceAtomSetIndices::do_()),
),
"else" => (
TerminalId::Else,
TokenValue::Atom(CommonSourceAtomSetIndices::else_()),
),
"enum" => (
TerminalId::Enum,
TokenValue::Atom(CommonSourceAtomSetIndices::enum_()),
),
"export" => (
TerminalId::Export,
TokenValue::Atom(CommonSourceAtomSetIndices::export()),
),
"extends" => (
TerminalId::Extends,
TokenValue::Atom(CommonSourceAtomSetIndices::extends()),
),
"finally" => (
TerminalId::Finally,
TokenValue::Atom(CommonSourceAtomSetIndices::finally()),
),
"for" => (
TerminalId::For,
TokenValue::Atom(CommonSourceAtomSetIndices::for_()),
),
"from" => (
TerminalId::From,
TokenValue::Atom(CommonSourceAtomSetIndices::from()),
),
"function" => (
TerminalId::Function,
TokenValue::Atom(CommonSourceAtomSetIndices::function()),
),
"get" => (
TerminalId::Get,
TokenValue::Atom(CommonSourceAtomSetIndices::get()),
),
"if" => (
TerminalId::If,
TokenValue::Atom(CommonSourceAtomSetIndices::if_()),
),
"implements" => (
TerminalId::Implements,
TokenValue::Atom(CommonSourceAtomSetIndices::implements()),
),
"import" => (
TerminalId::Import,
TokenValue::Atom(CommonSourceAtomSetIndices::import()),
),
"in" => (
TerminalId::In,
TokenValue::Atom(CommonSourceAtomSetIndices::in_()),
),
"instanceof" => (
TerminalId::Instanceof,
TokenValue::Atom(CommonSourceAtomSetIndices::instanceof()),
),
"interface" => (
TerminalId::Interface,
TokenValue::Atom(CommonSourceAtomSetIndices::interface()),
),
"let" => {
/*
(
TerminalId::Let,
TokenValue::Atom(CommonSourceAtomSetIndices::let_()),
),
*/
return Err(ParseError::NotImplemented(
"let cannot be handled in parser due to multiple lookahead",
)
.into());
}
"new" => (
TerminalId::New,
TokenValue::Atom(CommonSourceAtomSetIndices::new_()),
),
"of" => (
TerminalId::Of,
TokenValue::Atom(CommonSourceAtomSetIndices::of()),
),
"package" => (
TerminalId::Package,
TokenValue::Atom(CommonSourceAtomSetIndices::package()),
),
"private" => (
TerminalId::Private,
TokenValue::Atom(CommonSourceAtomSetIndices::private()),
),
"protected" => (
TerminalId::Protected,
TokenValue::Atom(CommonSourceAtomSetIndices::protected()),
),
"public" => (
TerminalId::Public,
TokenValue::Atom(CommonSourceAtomSetIndices::public()),
),
"return" => (
TerminalId::Return,
TokenValue::Atom(CommonSourceAtomSetIndices::return_()),
),
"set" => (
TerminalId::Set,
TokenValue::Atom(CommonSourceAtomSetIndices::set()),
),
"static" => (
TerminalId::Static,
TokenValue::Atom(CommonSourceAtomSetIndices::static_()),
),
"super" => (
TerminalId::Super,
TokenValue::Atom(CommonSourceAtomSetIndices::super_()),
),
"switch" => (
TerminalId::Switch,
TokenValue::Atom(CommonSourceAtomSetIndices::switch()),
),
"target" => (
TerminalId::Target,
TokenValue::Atom(CommonSourceAtomSetIndices::target()),
),
"this" => (
TerminalId::This,
TokenValue::Atom(CommonSourceAtomSetIndices::this()),
),
"throw" => (
TerminalId::Throw,
TokenValue::Atom(CommonSourceAtomSetIndices::throw()),
),
"try" => (
TerminalId::Try,
TokenValue::Atom(CommonSourceAtomSetIndices::try_()),
),
"typeof" => (
TerminalId::Typeof,
TokenValue::Atom(CommonSourceAtomSetIndices::typeof_()),
),
"var" => (
TerminalId::Var,
TokenValue::Atom(CommonSourceAtomSetIndices::var()),
),
"void" => (
TerminalId::Void,
TokenValue::Atom(CommonSourceAtomSetIndices::void()),
),
"while" => (
TerminalId::While,
TokenValue::Atom(CommonSourceAtomSetIndices::while_()),
),
"with" => (
TerminalId::With,
TokenValue::Atom(CommonSourceAtomSetIndices::with()),
),
"yield" => {
/*
(
TerminalId::Yield,
TokenValue::Atom(CommonSourceAtomSetIndices::yield_()),
),
*/
return Err(
ParseError::NotImplemented("yield cannot be handled in parser").into(),
);
}
"null" => (
TerminalId::NullLiteral,
TokenValue::Atom(CommonSourceAtomSetIndices::null()),
),
"true" => (
TerminalId::BooleanLiteral,
TokenValue::Atom(CommonSourceAtomSetIndices::true_()),
),
"false" => (
TerminalId::BooleanLiteral,
TokenValue::Atom(CommonSourceAtomSetIndices::false_()),
),
_ => (TerminalId::Name, self.string_to_token_value(text)),
}
};
self.set_result(id, SourceLocation::new(start, self.offset()), value)
}
/// ```text
/// PrivateIdentifier::
/// `#` IdentifierName
/// ```
fn private_identifier(&mut self, start: usize, builder: AutoCow<'alloc>) -> Result<'alloc, ()> {
let name = self.identifier_name(builder)?;
let value = self.string_to_token_value(name);
self.set_result(
TerminalId::PrivateIdentifier,
SourceLocation::new(start, self.offset()),
value,
)
}
/// ```text
/// UnicodeEscapeSequence::
/// `u` Hex4Digits
/// `u{` CodePoint `}`
/// ```
fn unicode_escape_sequence_after_backslash(&mut self) -> Result<'alloc, char> {
match self.chars.next() {
Some('u') => {}
_ => {
return Err(ParseError::InvalidEscapeSequence.into());
}
}
self.unicode_escape_sequence_after_backslash_and_u()
}
fn unicode_escape_sequence_after_backslash_and_u(&mut self) -> Result<'alloc, char> {
let value = match self.peek() {
Some('{') => {
self.chars.next();
let value = self.code_point()?;
match self.chars.next() {
Some('}') => {}
_ => {
return Err(ParseError::InvalidEscapeSequence.into());
}
}
value
}
_ => self.hex_4_digits()?,
};
Ok(value)
}
}
impl<'alloc> Lexer<'alloc> {
// ------------------------------------------------------------------------
// 11.8.3 Numeric Literals
/// Advance over decimal digits in the input.
///
/// ```text
/// NumericLiteralSeparator::
/// `_`
///
/// DecimalDigits ::
/// DecimalDigit
/// DecimalDigits NumericLiteralSeparator? DecimalDigit
///
/// DecimalDigit :: one of
/// `0` `1` `2` `3` `4` `5` `6` `7` `8` `9`
/// ```
fn decimal_digits(&mut self) -> Result<'alloc, ()> {
if let Some('0'..='9') = self.peek() {
self.chars.next();
} else {
return Err(self.unexpected_err().into());
}
self.decimal_digits_after_first_digit()?;
Ok(())
}
fn optional_decimal_digits(&mut self) -> Result<'alloc, ()> {
if let Some('0'..='9') = self.peek() {
self.chars.next();
} else {
return Ok(());
}
self.decimal_digits_after_first_digit()?;
Ok(())
}
fn decimal_digits_after_first_digit(&mut self) -> Result<'alloc, ()> {
while let Some(next) = self.peek() {
match next {
'_' => {
self.chars.next();
if let Some('0'..='9') = self.peek() {
self.chars.next();
} else {
return Err(self.unexpected_err().into());
}
}
'0'..='9' => {
self.chars.next();
}
_ => break,
}
}
Ok(())
}
/// Skip an ExponentPart, if present.
///
/// ```text
/// ExponentPart ::
/// ExponentIndicator SignedInteger
///
/// ExponentIndicator :: one of
/// `e` `E`
///
/// SignedInteger ::
/// DecimalDigits
/// `+` DecimalDigits
/// `-` DecimalDigits
/// ```
fn optional_exponent(&mut self) -> Result<'alloc, bool> {
if let Some('e') | Some('E') = self.peek() {
self.chars.next();
self.decimal_exponent()?;
return Ok(true);
}
Ok(false)
}
fn decimal_exponent(&mut self) -> Result<'alloc, ()> {
if let Some('+') | Some('-') = self.peek() {
self.chars.next();
}
self.decimal_digits()?;
Ok(())
}
/// ```text
/// HexDigit :: one of
/// `0` `1` `2` `3` `4` `5` `6` `7` `8` `9` `a` `b` `c` `d` `e` `f` `A` `B` `C` `D` `E` `F`
/// ```
fn hex_digit(&mut self) -> Result<'alloc, u32> {
match self.chars.next() {
None => Err(ParseError::InvalidEscapeSequence.into()),
Some(c @ '0'..='9') => Ok(c as u32 - '0' as u32),
Some(c @ 'a'..='f') => Ok(10 + (c as u32 - 'a' as u32)),
Some(c @ 'A'..='F') => Ok(10 + (c as u32 - 'A' as u32)),
Some(other) => Err(ParseError::IllegalCharacter(other).into()),
}
}
fn code_point_to_char(value: u32) -> Result<'alloc, char> {
if 0xd800 <= value && value <= 0xdfff {
Err(ParseError::NotImplemented("unicode escape sequences (surrogates)").into())
} else {
char::try_from(value).map_err(|_| ParseError::InvalidEscapeSequence.into())
}
}
/// ```text
/// Hex4Digits ::
/// HexDigit HexDigit HexDigit HexDigit
/// ```
fn hex_4_digits(&mut self) -> Result<'alloc, char> {
let mut value = 0;
for _ in 0..4 {
value = (value << 4) | self.hex_digit()?;
}
Self::code_point_to_char(value)
}
/// ```text
/// CodePoint ::
/// HexDigits but only if MV of HexDigits ≤ 0x10FFFF
///
/// HexDigits ::
/// HexDigit
/// HexDigits HexDigit
/// ```
fn code_point(&mut self) -> Result<'alloc, char> {
let mut value = self.hex_digit()?;
loop {
let next = match self.peek() {
None => {
return Err(ParseError::InvalidEscapeSequence.into());
}
Some(c @ '0'..='9') => c as u32 - '0' as u32,
Some(c @ 'a'..='f') => 10 + (c as u32 - 'a' as u32),
Some(c @ 'A'..='F') => 10 + (c as u32 - 'A' as u32),
Some(_) => break,
};
self.chars.next();
value = (value << 4) | next;
if value > 0x10FFFF {
return Err(ParseError::InvalidEscapeSequence.into());
}
}
Self::code_point_to_char(value)
}
/// Scan a NumericLiteral (defined in 11.8.3, extended by B.1.1) after
/// having already consumed the first character, which was `0`.
///
/// ```text
/// NumericLiteral ::
/// DecimalLiteral
/// DecimalBigIntegerLiteral
/// NonDecimalIntegerLiteral
/// NonDecimalIntegerLiteral BigIntLiteralSuffix
///
/// DecimalBigIntegerLiteral ::
/// `0` BigIntLiteralSuffix
/// NonZeroDigit DecimalDigits? BigIntLiteralSuffix
///
/// NonDecimalIntegerLiteral ::
/// BinaryIntegerLiteral
/// OctalIntegerLiteral
/// HexIntegerLiteral
///
/// BigIntLiteralSuffix ::
/// `n`
/// ```
fn numeric_literal_starting_with_zero(&mut self) -> Result<'alloc, NumericResult> {
let mut base = NumericLiteralBase::Decimal;
match self.peek() {
// BinaryIntegerLiteral ::
// `0b` BinaryDigits
// `0B` BinaryDigits
//
// BinaryDigits ::
// BinaryDigit
// BinaryDigits NumericLiteralSeparator? BinaryDigit
//
// BinaryDigit :: one of
// `0` `1`
Some('b') | Some('B') => {
self.chars.next();
base = NumericLiteralBase::Binary;
if let Some('0'..='1') = self.peek() {
self.chars.next();
} else {
return Err(self.unexpected_err().into());
}
while let Some(next) = self.peek() {
match next {
'_' => {
self.chars.next();
if let Some('0'..='1') = self.peek() {
self.chars.next();
} else {
return Err(self.unexpected_err().into());
}
}
'0'..='1' => {
self.chars.next();
}
_ => break,
}
}
if let Some('n') = self.peek() {
self.chars.next();
self.check_after_numeric_literal()?;
return Ok(NumericResult::BigInt { base });
}
}
// OctalIntegerLiteral ::
// `0o` OctalDigits
// `0O` OctalDigits
//
// OctalDigits ::
// OctalDigit
// OctalDigits NumericLiteralSeparator? OctalDigit
//
// OctalDigit :: one of
// `0` `1` `2` `3` `4` `5` `6` `7`
//
Some('o') | Some('O') => {
self.chars.next();
base = NumericLiteralBase::Octal;
if let Some('0'..='7') = self.peek() {
self.chars.next();
} else {
return Err(self.unexpected_err().into());
}
while let Some(next) = self.peek() {
match next {
'_' => {
self.chars.next();
if let Some('0'..='7') = self.peek() {
self.chars.next();
} else {
return Err(self.unexpected_err().into());
}
}
'0'..='7' => {
self.chars.next();
}
_ => break,
}
}
if let Some('n') = self.peek() {
self.chars.next();
self.check_after_numeric_literal()?;
return Ok(NumericResult::BigInt { base });
}
}
// HexIntegerLiteral ::
// `0x` HexDigits
// `0X` HexDigits
//
// HexDigits ::
// HexDigit
// HexDigits NumericLiteralSeparator? HexDigit
//
// HexDigit :: one of
// `0` `1` `2` `3` `4` `5` `6` `7` `8` `9` `a` `b` `c` `d` `e` `f` `A` `B` `C` `D` `E` `F`
Some('x') | Some('X') => {
self.chars.next();
base = NumericLiteralBase::Hex;
if let Some('0'..='9') | Some('a'..='f') | Some('A'..='F') = self.peek() {
self.chars.next();
} else {
return Err(self.unexpected_err().into());
}
while let Some(next) = self.peek() {
match next {
'_' => {
self.chars.next();
if let Some('0'..='9') | Some('a'..='f') | Some('A'..='F') = self.peek()
{
self.chars.next();
} else {
return Err(self.unexpected_err().into());
}
}
'0'..='9' | 'a'..='f' | 'A'..='F' => {
self.chars.next();
}
_ => break,
}
}
if let Some('n') = self.peek() {
self.chars.next();
self.check_after_numeric_literal()?;
return Ok(NumericResult::BigInt { base });
}
}
Some('.') => {
self.chars.next();
return self.decimal_literal_after_decimal_point_after_digits();
}
Some('e') | Some('E') => {
self.chars.next();
self.decimal_exponent()?;
return Ok(NumericResult::Float);
}
Some('n') => {
self.chars.next();
self.check_after_numeric_literal()?;
return Ok(NumericResult::BigInt { base });
}
Some('0'..='9') => {
// This is almost always the token `0` in practice.
//
// In nonstrict code, as a legacy feature, other numbers
// starting with `0` are allowed. If /0[0-7]+/ matches, it's a
// LegacyOctalIntegerLiteral; but if we see an `8` or `9` in
// the number, it's decimal. Decimal numbers can have a decimal
// point and/or ExponentPart; octals can't.
//
// Neither is allowed with a BigIntLiteralSuffix `n`.
//
// LegacyOctalIntegerLiteral ::
// `0` OctalDigit
// LegacyOctalIntegerLiteral OctalDigit
//
// NonOctalDecimalIntegerLiteral ::
// `0` NonOctalDigit
// LegacyOctalLikeDecimalIntegerLiteral NonOctalDigit
// NonOctalDecimalIntegerLiteral DecimalDigit
//
// LegacyOctalLikeDecimalIntegerLiteral ::
// `0` OctalDigit
// LegacyOctalLikeDecimalIntegerLiteral OctalDigit
//
// NonOctalDigit :: one of
// `8` `9`
//
// TODO: implement `strict_mode` check
// let strict_mode = true;
// if !strict_mode {
// // TODO: Distinguish between Octal and NonOctalDecimal.
// // TODO: Support NonOctalDecimal followed by a decimal
// // point and/or ExponentPart.
// self.decimal_digits()?;
// }
return Err(ParseError::NotImplemented("LegacyOctalIntegerLiteral").into());
}
_ => {}
}
self.check_after_numeric_literal()?;
Ok(NumericResult::Int { base })
}
/// Scan a NumericLiteral (defined in 11.8.3, extended by B.1.1) after
/// having already consumed the first character, which is a decimal digit.
fn decimal_literal_after_first_digit(&mut self) -> Result<'alloc, NumericResult> {
// DecimalLiteral ::
// DecimalIntegerLiteral `.` DecimalDigits? ExponentPart?
// `.` DecimalDigits ExponentPart?
// DecimalIntegerLiteral ExponentPart?
//
// DecimalIntegerLiteral ::
// `0` #see `numeric_literal_starting_with_zero`
// NonZeroDigit
// NonZeroDigit NumericLiteralSeparator? DecimalDigits
// NonOctalDecimalIntegerLiteral #see `numeric_literal_
// # starting_with_zero`
//
// NonZeroDigit :: one of
// `1` `2` `3` `4` `5` `6` `7` `8` `9`
self.decimal_digits_after_first_digit()?;
match self.peek() {
Some('.') => {
self.chars.next();
return self.decimal_literal_after_decimal_point_after_digits();
}
Some('n') => {
self.chars.next();
self.check_after_numeric_literal()?;
return Ok(NumericResult::BigInt {
base: NumericLiteralBase::Decimal,
});
}
_ => {}
}
let has_exponent = self.optional_exponent()?;
self.check_after_numeric_literal()?;
let result = if has_exponent {
NumericResult::Float
} else {
NumericResult::Int {
base: NumericLiteralBase::Decimal,
}
};
Ok(result)
}
fn decimal_literal_after_decimal_point(&mut self) -> Result<'alloc, NumericResult> {
// The parts after `.` in
//
// `.` DecimalDigits ExponentPart?
self.decimal_digits()?;
self.optional_exponent()?;
self.check_after_numeric_literal()?;
Ok(NumericResult::Float)
}
fn decimal_literal_after_decimal_point_after_digits(
&mut self,
) -> Result<'alloc, NumericResult> {
// The parts after `.` in
//
// DecimalLiteral ::
// DecimalIntegerLiteral `.` DecimalDigits? ExponentPart?
self.optional_decimal_digits()?;
self.optional_exponent()?;
self.check_after_numeric_literal()?;
Ok(NumericResult::Float)
}
fn check_after_numeric_literal(&self) -> Result<'alloc, ()> {
// The SourceCharacter immediately following a
// NumericLiteral must not be an IdentifierStart or
// DecimalDigit. (11.8.3)
if let Some(ch) = self.peek() {
if is_identifier_start(ch) || ch.is_digit(10) {
return Err(ParseError::IllegalCharacter(ch).into());
}
}
Ok(())
}
// ------------------------------------------------------------------------
// 11.8.4 String Literals (as extended by B.1.2)
/// Scan an LineContinuation or EscapeSequence in a string literal, having
/// already consumed the initial backslash character.
///
/// ```text
/// LineContinuation ::
/// `\` LineTerminatorSequence
///
/// EscapeSequence ::
/// CharacterEscapeSequence
/// (in strict mode code) `0` [lookahead ∉ DecimalDigit]
/// (in non-strict code) LegacyOctalEscapeSequence
/// HexEscapeSequence
/// UnicodeEscapeSequence
///
/// CharacterEscapeSequence ::
/// SingleEscapeCharacter
/// NonEscapeCharacter
///
/// SingleEscapeCharacter :: one of
/// `'` `"` `\` `b` `f` `n` `r` `t` `v`
///
/// LegacyOctalEscapeSequence ::
/// OctalDigit [lookahead ∉ OctalDigit]
/// ZeroToThree OctalDigit [lookahead ∉ OctalDigit]
/// FourToSeven OctalDigit
/// ZeroToThree OctalDigit OctalDigit
///
/// ZeroToThree :: one of
/// `0` `1` `2` `3`
///
/// FourToSeven :: one of
/// `4` `5` `6` `7`
/// ```
fn escape_sequence(&mut self, text: &mut String<'alloc>) -> Result<'alloc, ()> {
match self.chars.next() {
None => {
return Err(ParseError::UnterminatedString.into());
}
Some(c) => match c {
LF | LS | PS => {
// LineContinuation. Ignore it.
//
// Don't set is_on_new_line because this LineContinuation
// has no bearing on whether the current string literal was
// the first token on the line where it started.
}
CR => {
// LineContinuation. Check for the sequence \r\n; otherwise
// ignore it.
if self.peek() == Some(LF) {
self.chars.next();
}
}
'\'' | '"' | '\\' => {
text.push(c);
}
'b' => {
text.push('\u{8}');
}
'f' => {
text.push(FF);
}
'n' => {
text.push(LF);
}
'r' => {
text.push(CR);
}
't' => {
text.push(TAB);
}
'v' => {
text.push(VT);
}
'x' => {
// HexEscapeSequence ::
// `x` HexDigit HexDigit
let mut value = self.hex_digit()?;
value = (value << 4) | self.hex_digit()?;
match char::try_from(value) {
Err(_) => {
return Err(ParseError::InvalidEscapeSequence.into());
}
Ok(c) => {
text.push(c);
}
}
}
'u' => {
let c = self.unicode_escape_sequence_after_backslash_and_u()?;
text.push(c);
}
'0' => {
// In strict mode code and in template literals, the
// relevant production is
//
// EscapeSequence ::
// `0` [lookahead <! DecimalDigit]
//
// In non-strict StringLiterals, `\0` begins a
// LegacyOctalEscapeSequence which may contain more digits.
match self.peek() {
Some('0'..='7') => {
return Err(ParseError::NotImplemented(
"legacy octal escape sequence in string",
)
.into());
}
Some('8'..='9') => {
return Err(ParseError::NotImplemented(
"digit immediately following \\0 escape sequence",
)
.into());
}
_ => {}
}
text.push('\0');
}
'1'..='7' => {
return Err(ParseError::NotImplemented(
"legacy octal escape sequence in string",
)
.into());
}
other => {
// "\8" and "\9" are invalid per spec, but SpiderMonkey and
// V8 accept them, and JSC accepts them in non-strict mode.
// "\8" is "8" and "\9" is "9".
text.push(other);
}
},
}
Ok(())
}
/// Scan a string literal, having already consumed the starting quote
/// character `delimiter`.
///
/// ```text
/// StringLiteral ::
/// `"` DoubleStringCharacters? `"`
/// `'` SingleStringCharacters? `'`
///
/// DoubleStringCharacters ::
/// DoubleStringCharacter DoubleStringCharacters?
///
/// SingleStringCharacters ::
/// SingleStringCharacter SingleStringCharacters?
///
/// DoubleStringCharacter ::
/// SourceCharacter but not one of `"` or `\` or LineTerminator
/// <LS>
/// <PS>
/// `\` EscapeSequence
/// LineContinuation
///
/// SingleStringCharacter ::
/// SourceCharacter but not one of `'` or `\` or LineTerminator
/// <LS>
/// <PS>
/// `\` EscapeSequence
/// LineContinuation
/// ```
fn string_literal(&mut self, delimiter: char) -> Result<'alloc, ()> {
let offset = self.offset() - 1;
let mut builder = AutoCow::new(&self);
loop {
match self.chars.next() {
None | Some('\r') | Some('\n') => {
return Err(ParseError::UnterminatedString.into());
}
Some(c @ '"') | Some(c @ '\'') => {
if c == delimiter {
let value = self.string_to_token_value(builder.finish_without_push(&self));
return self.set_result(
TerminalId::StringLiteral,
SourceLocation::new(offset, self.offset()),
value,
);
} else {
builder.push_matching(c);
}
}
Some('\\') => {
let text = builder.get_mut_string_without_current_ascii_char(&self);
self.escape_sequence(text)?;
}
Some(other) => {
// NonEscapeCharacter ::
// SourceCharacter but not one of EscapeCharacter or LineTerminator
//
// EscapeCharacter ::
// SingleEscapeCharacter
// DecimalDigit
// `x`
// `u`
builder.push_matching(other);
}
}
}
}
// ------------------------------------------------------------------------
// 11.8.5 Regular Expression Literals
fn regular_expression_backslash_sequence(&mut self) -> Result<'alloc, ()> {
match self.chars.next() {
None | Some(CR) | Some(LF) | Some(LS) | Some(PS) => {
Err(ParseError::UnterminatedRegExp.into())
}
Some(_) => Ok(()),
}
}
// See 12.2.8 and 11.8.5 sections.
fn regular_expression_literal(&mut self, builder: &mut AutoCow<'alloc>) -> Result<'alloc, ()> {
let offset = self.offset();
loop {
match self.chars.next() {
None | Some(CR) | Some(LF) | Some(LS) | Some(PS) => {
return Err(ParseError::UnterminatedRegExp.into());
}
Some('/') => {
break;
}
Some('[') => {
// RegularExpressionClass.
loop {
match self.chars.next() {
None | Some(CR) | Some(LF) | Some(LS) | Some(PS) => {
return Err(ParseError::UnterminatedRegExp.into());
}
Some(']') => {
break;
}
Some('\\') => {
self.regular_expression_backslash_sequence()?;
}
Some(_) => {}
}
}
}
Some('\\') => {
self.regular_expression_backslash_sequence()?;
}
Some(_) => {}
}
}
let mut flag_text = AutoCow::new(&self);
while let Some(ch) = self.peek() {
match ch {
'$' | '_' | 'a'..='z' | 'A'..='Z' | '0'..='9' => {
self.chars.next();
flag_text.push_matching(ch);
}
_ => break,
}
}
// 12.2.8.2.1 Assert literal is a RegularExpressionLiteral.
let literal = builder.finish(&self);
// 12.2.8.2.2 Check that only gimsuy flags are mentioned at most once.
let gimsuy_mask: u32 = ['g', 'i', 'm', 's', 'u', 'y']
.iter()
.map(|x| 1 << ((*x as u8) - ('a' as u8)))
.sum();
let mut flag_text_set: u32 = 0;
for ch in flag_text.finish(&self).chars() {
if !ch.is_ascii_lowercase() {
return Err(ParseError::NotImplemented(
"Unexpected flag in regular expression literal",
)
.into());
}
let ch_mask = 1 << ((ch as u8) - ('a' as u8));
if ch_mask & gimsuy_mask == 0 {
return Err(ParseError::NotImplemented(
"Unexpected flag in regular expression literal",
)
.into());
}
if flag_text_set & ch_mask != 0 {
return Err(ParseError::NotImplemented(
"Flag is mentioned twice in regular expression literal",
)
.into());
}
flag_text_set |= ch_mask;
}
// TODO: 12.2.8.2.4 and 12.2.8.2.5 Check that the body matches the
// grammar defined in 21.2.1.
let value = self.slice_to_token_value(literal);
self.set_result(
TerminalId::RegularExpressionLiteral,
SourceLocation::new(offset, self.offset()),
value,
)
}
// ------------------------------------------------------------------------
// 11.8.6 Template Literal Lexical Components
/// Parse a template literal component token, having already consumed the
/// starting `` ` `` or `}` character. On success, the `id` of the returned
/// `Token` is `subst` (if the token ends with `${`) or `tail` (if the
/// token ends with `` ` ``).
///
/// ```text
/// NoSubstitutionTemplate ::
/// ``` TemplateCharacters? ```
///
/// TemplateHead ::
/// ``` TemplateCharacters? `${`
///
/// TemplateMiddle ::
/// `}` TemplateCharacters? `${`
///
/// TemplateTail ::
/// `}` TemplateCharacters? ```
///
/// TemplateCharacters ::
/// TemplateCharacter TemplateCharacters?
/// ```
fn template_part(
&mut self,
start: usize,
subst: TerminalId,
tail: TerminalId,
) -> Result<'alloc, ()> {
let mut builder = AutoCow::new(&self);
while let Some(ch) = self.chars.next() {
// TemplateCharacter ::
// `$` [lookahead != `{` ]
// `\` EscapeSequence
// `\` NotEscapeSequence
// LineContinuation
// LineTerminatorSequence
// SourceCharacter but not one of ``` or `\` or `$` or LineTerminator
//
// NotEscapeSequence ::
// `0` DecimalDigit
// DecimalDigit but not `0`
// `x` [lookahead <! HexDigit]
// `x` HexDigit [lookahead <! HexDigit]
// `u` [lookahead <! HexDigit] [lookahead != `{`]
// `u` HexDigit [lookahead <! HexDigit]
// `u` HexDigit HexDigit [lookahead <! HexDigit]
// `u` HexDigit HexDigit HexDigit [lookahead <! HexDigit]
// `u` `{` [lookahead <! HexDigit]
// `u` `{` NotCodePoint [lookahead <! HexDigit]
// `u` `{` CodePoint [lookahead <! HexDigit] [lookahead != `}`]
//
// NotCodePoint ::
// HexDigits [> but only if MV of |HexDigits| > 0x10FFFF ]
//
// CodePoint ::
// HexDigits [> but only if MV of |HexDigits| ≤ 0x10FFFF ]
if ch == '$' && self.peek() == Some('{') {
self.chars.next();
let value = self.string_to_token_value(builder.finish_without_push(&self));
return self.set_result(subst, SourceLocation::new(start, self.offset()), value);
}
if ch == '`' {
let value = self.string_to_token_value(builder.finish_without_push(&self));
return self.set_result(tail, SourceLocation::new(start, self.offset()), value);
}
// TODO: Support escape sequences.
if ch == '\\' {
let text = builder.get_mut_string_without_current_ascii_char(&self);
self.escape_sequence(text)?;
} else {
builder.push_matching(ch);
}
}
Err(ParseError::UnterminatedString.into())
}
fn advance_impl<'parser>(&mut self, parser: &Parser<'parser>) -> Result<'alloc, ()> {
let mut builder = AutoCow::new(&self);
let mut start = self.offset();
while let Some(c) = self.chars.next() {
match c {
// 11.2 White Space
//
// WhiteSpace ::
// <TAB>
// <VT>
// <FF>
// <SP>
// <NBSP>
// <ZWNBSP>
// <USP>
TAB |
VT |
FF |
SP |
NBSP |
ZWNBSP |
'\u{1680}' | // Ogham space mark (in <USP>)
'\u{2000}' ..= '\u{200a}' | // typesetting spaces (in <USP>)
'\u{202f}' | // Narrow no-break space (in <USP>)
'\u{205f}' | // Medium mathematical space (in <USP>)
'\u{3000}' // Ideographic space (in <USP>)
=> {
// TODO - The spec uses <USP> to stand for any character
// with category "Space_Separator" (Zs). New Unicode
// standards may add characters to this set. This should therefore be
// implemented using the Unicode database somehow.
builder = AutoCow::new(&self);
start = self.offset();
continue;
}
// 11.3 Line Terminators
//
// LineTerminator ::
// <LF>
// <CR>
// <LS>
// <PS>
LF | CR | LS | PS => {
self.token.is_on_new_line = true;
builder = AutoCow::new(&self);
start = self.offset();
continue;
}
'0' => {
let result = self.numeric_literal_starting_with_zero()?;
return Ok(self.numeric_result_to_advance_result(builder.finish(&self), start, result)?);
}
'1'..='9' => {
let result = self.decimal_literal_after_first_digit()?;
return Ok(self.numeric_result_to_advance_result(builder.finish(&self), start, result)?);
}
'"' | '\'' => {
return self.string_literal(c);
}
'`' => {
return self.template_part(start, TerminalId::TemplateHead, TerminalId::NoSubstitutionTemplate);
}
'!' => match self.peek() {
Some('=') => {
self.chars.next();
match self.peek() {
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::StrictNotEqual,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::LaxNotEqual,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
}
}
_ => return self.set_result(
TerminalId::LogicalNot,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
},
'%' => match self.peek() {
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::RemainderAssign,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::Remainder,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
},
'&' => match self.peek() {
Some('&') => {
self.chars.next();
match self.peek() {
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::LogicalAndAssign,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::LogicalAnd,
SourceLocation::new(start, self.offset()),
TokenValue::None,
)
}
}
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::BitwiseAndAssign,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::BitwiseAnd,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
},
'*' => match self.peek() {
Some('*') => {
self.chars.next();
match self.peek() {
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::ExponentiateAssign,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::Exponentiate,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
}
}
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::MultiplyAssign,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::Star,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
},
'+' => match self.peek() {
Some('+') => {
self.chars.next();
return self.set_result(
TerminalId::Increment,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::AddAssign,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::Plus,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
},
'-' => match self.peek() {
Some('-') => {
self.chars.next();
match self.peek() {
Some('>') if self.token.is_on_new_line => {
// B.1.3 SingleLineHTMLCloseComment
// TODO: Limit this to Script (not Module).
self.skip_single_line_comment(&mut builder);
continue;
}
_ => return self.set_result(
TerminalId::Decrement,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
}
}
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::SubtractAssign,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::Minus,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
},
'.' => match self.peek() {
Some('.') => {
self.chars.next();
match self.peek() {
Some('.') => {
self.chars.next();
return self.set_result(
TerminalId::Ellipsis,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return Err(ParseError::IllegalCharacter('.').into()),
}
}
Some('0'..='9') => {
let result = self.decimal_literal_after_decimal_point()?;
return Ok(self.numeric_result_to_advance_result(builder.finish(&self), start, result)?);
}
_ => return self.set_result(
TerminalId::Dot,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
},
'/' => match self.peek() {
Some('/') => {
// SingleLineComment :: `//` SingleLineCommentChars?
self.chars.next();
self.skip_single_line_comment(&mut builder);
start = self.offset();
continue;
}
Some('*') => {
self.chars.next();
self.skip_multi_line_comment(&mut builder)?;
start = self.offset();
continue;
}
_ => {
if parser.can_accept_terminal(TerminalId::Divide) {
match self.peek() {
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::DivideAssign,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::Divide,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
}
}
return self.regular_expression_literal(&mut builder);
}
},
'}' => {
if parser.can_accept_terminal(TerminalId::TemplateMiddle) {
return self.template_part(start, TerminalId::TemplateMiddle, TerminalId::TemplateTail);
}
return self.set_result(
TerminalId::CloseBrace,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
'<' => match self.peek() {
Some('<') => {
self.chars.next();
match self.peek() {
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::LeftShiftAssign,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::LeftShift,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
}
}
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::LessThanOrEqualTo,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
Some('!') if self.is_looking_at("!--") => {
// B.1.3 SingleLineHTMLOpenComment. Note that the above
// `is_looking_at` test peeked ahead at the next three
// characters of input. This lookahead is necessary
// because `x<!--` has a comment but `x<!-y` does not.
//
// TODO: Limit this to Script (not Module).
self.skip_single_line_comment(&mut builder);
start = self.offset();
continue;
}
_ => return self.set_result(
TerminalId::LessThan,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
},
'=' => match self.peek() {
Some('=') => {
self.chars.next();
match self.peek() {
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::StrictEqual,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::LaxEqual,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
}
}
Some('>') => {
self.chars.next();
return self.set_result(
TerminalId::Arrow,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::EqualSign,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
},
'>' => match self.peek() {
Some('>') => {
self.chars.next();
match self.peek() {
Some('>') => {
self.chars.next();
match self.peek() {
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::UnsignedRightShiftAssign,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::UnsignedRightShift,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
}
}
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::SignedRightShiftAssign,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::SignedRightShift,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
}
}
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::GreaterThanOrEqualTo,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::GreaterThan,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
},
'^' => match self.peek() {
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::BitwiseXorAssign,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::BitwiseXor,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
},
'|' => match self.peek() {
Some('|') => {
self.chars.next();
match self.peek() {
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::LogicalOrAssign,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::LogicalOr,
SourceLocation::new(start, self.offset()),
TokenValue::None,
)
}
}
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::BitwiseOrAssign,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::BitwiseOr,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
},
'?' => match self.peek() {
Some('?') => {
self.chars.next();
match self.peek() {
Some('=') => {
self.chars.next();
return self.set_result(
TerminalId::CoalesceAssign,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::Coalesce,
SourceLocation::new(start, self.offset()),
TokenValue::None,
)
}
}
Some('.') => {
if let Some('0'..='9') = self.double_peek() {
return self.set_result(
TerminalId::QuestionMark,
SourceLocation::new(start, self.offset()),
TokenValue::None,
)
}
self.chars.next();
return self.set_result(
TerminalId::OptionalChain,
SourceLocation::new(start, self.offset()),
TokenValue::None,
);
}
_ => return self.set_result(
TerminalId::QuestionMark,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
}
'(' => return self.set_result(
TerminalId::OpenParenthesis,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
')' => return self.set_result(
TerminalId::CloseParenthesis,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
',' => return self.set_result(
TerminalId::Comma,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
':' => return self.set_result(
TerminalId::Colon,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
';' => return self.set_result(
TerminalId::Semicolon,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
'[' => return self.set_result(
TerminalId::OpenBracket,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
']' => return self.set_result(
TerminalId::CloseBracket,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
'{' => return self.set_result(
TerminalId::OpenBrace,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
'~' => return self.set_result(
TerminalId::BitwiseNot,
SourceLocation::new(start, self.offset()),
TokenValue::None,
),
// Idents
'$' | '_' | 'a'..='z' | 'A'..='Z' => {
builder.push_matching(c);
return self.identifier_tail(start, builder);
}
'\\' => {
builder.force_allocation_without_current_ascii_char(&self);
let value = self.unicode_escape_sequence_after_backslash()?;
if !is_identifier_start(value) {
return Err(ParseError::IllegalCharacter(value).into());
}
builder.push_different(value);
return self.identifier_tail(start, builder);
}
'#' => {
if start == 0 {
// HashbangComment ::
// `#!` SingleLineCommentChars?
if let Some('!') = self.peek() {
self.skip_single_line_comment(&mut builder);
start = self.offset();
continue;
}
}
builder.push_matching(c);
return self.private_identifier(start, builder);
}
other if is_identifier_start(other) => {
builder.push_matching(other);
return self.identifier_tail(start, builder);
}
other => {
return Err(ParseError::IllegalCharacter(other).into());
}
}
}
self.set_result(
TerminalId::End,
SourceLocation::new(start, self.offset()),
TokenValue::None,
)
}
fn string_to_token_value(&mut self, s: &'alloc str) -> TokenValue {
let index = self.atoms.borrow_mut().insert(s);
TokenValue::Atom(index)
}
fn slice_to_token_value(&mut self, s: &'alloc str) -> TokenValue {
let index = self.slices.borrow_mut().push(s);
TokenValue::Slice(index)
}
fn numeric_result_to_advance_result(
&mut self,
s: &'alloc str,
start: usize,
result: NumericResult,
) -> Result<'alloc, ()> {
let (terminal_id, value) = match result {
NumericResult::Int { base } => {
let n = parse_int(s, base).map_err(|s| ParseError::NotImplemented(s))?;
(TerminalId::NumericLiteral, TokenValue::Number(n))
}
NumericResult::Float => {
let n = parse_float(s).map_err(|s| ParseError::NotImplemented(s))?;
(TerminalId::NumericLiteral, TokenValue::Number(n))
}
NumericResult::BigInt { .. } => {
// FIXME
(TerminalId::BigIntLiteral, self.string_to_token_value(s))
}
};
self.set_result(
terminal_id,
SourceLocation::new(start, self.offset()),
value,
)
}
}
struct AutoCow<'alloc> {
start: &'alloc str,
value: Option<String<'alloc>>,
}
impl<'alloc> AutoCow<'alloc> {
fn new(lexer: &Lexer<'alloc>) -> Self {
AutoCow {
start: lexer.chars.as_str(),
value: None,
}
}
// Push a char that matches lexer.chars.next()
fn push_matching(&mut self, c: char) {
if let Some(text) = &mut self.value {
text.push(c);
}
}
// Push a different character than lexer.chars.next().
// force_allocation_without_current_ascii_char must be called before this.
fn push_different(&mut self, c: char) {
debug_assert!(self.value.is_some());
self.value.as_mut().unwrap().push(c)
}
// Force allocation of a String, excluding the current ASCII character,
// and return the reference to it
fn get_mut_string_without_current_ascii_char<'b>(
&'b mut self,
lexer: &'_ Lexer<'alloc>,
) -> &'b mut String<'alloc> {
self.force_allocation_without_current_ascii_char(lexer);
self.value.as_mut().unwrap()
}
// Force allocation of a String, excluding the current ASCII character.
fn force_allocation_without_current_ascii_char(&mut self, lexer: &'_ Lexer<'alloc>) {
if self.value.is_some() {
return;
}
self.value = Some(String::from_str_in(
&self.start[..self.start.len() - lexer.chars.as_str().len() - 1],
lexer.allocator,
));
}
// Check if the string contains a different character, such as an escape
// sequence
fn has_different(&self) -> bool {
self.value.is_some()
}
fn finish(&mut self, lexer: &Lexer<'alloc>) -> &'alloc str {
match self.value.take() {
Some(arena_string) => arena_string.into_bump_str(),
None => &self.start[..self.start.len() - lexer.chars.as_str().len()],
}
}
// Just like finish, but without pushing current char.
fn finish_without_push(&mut self, lexer: &Lexer<'alloc>) -> &'alloc str {
match self.value.take() {
Some(arena_string) => arena_string.into_bump_str(),
None => &self.start[..self.start.len() - lexer.chars.as_str().len() - 1],
}
}
}