4fb96afafc
https://tc39.es/ecma262/#sec-additional-syntax-string-literals The syntax and semantics of 11.8.4 is extended as follows except that this extension is not allowed for strict mode code: Syntax EscapeSequence:: CharacterEscapeSequence LegacyOctalEscapeSequence NonOctalDecimalEscapeSequence HexEscapeSequence UnicodeEscapeSequence LegacyOctalEscapeSequence:: OctalDigit [lookahead ∉ OctalDigit] ZeroToThree OctalDigit [lookahead ∉ OctalDigit] FourToSeven OctalDigit ZeroToThree OctalDigit OctalDigit ZeroToThree :: one of 0 1 2 3 FourToSeven :: one of 4 5 6 7 NonOctalDecimalEscapeSequence :: one of 8 9 This definition of EscapeSequence is not used in strict mode or when parsing TemplateCharacter. Note It is possible for string literals to precede a Use Strict Directive that places the enclosing code in strict mode, and implementations must take care to not use this extended definition of EscapeSequence with such literals. For example, attempting to parse the following source text must fail: function invalid() { "\7"; "use strict"; }
313 lines
12 KiB
C++
313 lines
12 KiB
C++
/*
|
|
* Copyright (c) 2020, Stephan Unverwerth <s.unverwerth@gmx.de>
|
|
* Copyright (c) 2020, Linus Groh <mail@linusgroh.de>
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are met:
|
|
*
|
|
* 1. Redistributions of source code must retain the above copyright notice, this
|
|
* list of conditions and the following disclaimer.
|
|
*
|
|
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
* this list of conditions and the following disclaimer in the documentation
|
|
* and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
#include "Token.h"
|
|
#include <AK/Assertions.h>
|
|
#include <AK/StringBuilder.h>
|
|
#include <AK/Utf32View.h>
|
|
#include <ctype.h>
|
|
|
|
namespace JS {
|
|
|
|
const char* Token::name(TokenType type)
|
|
{
|
|
switch (type) {
|
|
#define __ENUMERATE_JS_TOKEN(type, category) \
|
|
case TokenType::type: \
|
|
return #type;
|
|
ENUMERATE_JS_TOKENS
|
|
#undef __ENUMERATE_JS_TOKEN
|
|
default:
|
|
ASSERT_NOT_REACHED();
|
|
return "<Unknown>";
|
|
}
|
|
}
|
|
|
|
const char* Token::name() const
|
|
{
|
|
return name(m_type);
|
|
}
|
|
|
|
TokenCategory Token::category(TokenType type)
|
|
{
|
|
switch (type) {
|
|
#define __ENUMERATE_JS_TOKEN(type, category) \
|
|
case TokenType::type: \
|
|
return TokenCategory::category;
|
|
ENUMERATE_JS_TOKENS
|
|
#undef __ENUMERATE_JS_TOKEN
|
|
default:
|
|
ASSERT_NOT_REACHED();
|
|
}
|
|
}
|
|
|
|
TokenCategory Token::category() const
|
|
{
|
|
return category(m_type);
|
|
}
|
|
|
|
double Token::double_value() const
|
|
{
|
|
ASSERT(type() == TokenType::NumericLiteral);
|
|
String value_string(m_value);
|
|
if (value_string[0] == '0' && value_string.length() >= 2) {
|
|
if (value_string[1] == 'x' || value_string[1] == 'X') {
|
|
// hexadecimal
|
|
return static_cast<double>(strtoul(value_string.characters() + 2, nullptr, 16));
|
|
} else if (value_string[1] == 'o' || value_string[1] == 'O') {
|
|
// octal
|
|
return static_cast<double>(strtoul(value_string.characters() + 2, nullptr, 8));
|
|
} else if (value_string[1] == 'b' || value_string[1] == 'B') {
|
|
// binary
|
|
return static_cast<double>(strtoul(value_string.characters() + 2, nullptr, 2));
|
|
} else if (isdigit(value_string[1])) {
|
|
// also octal, but syntax error in strict mode
|
|
return static_cast<double>(strtoul(value_string.characters() + 1, nullptr, 8));
|
|
}
|
|
}
|
|
return strtod(value_string.characters(), nullptr);
|
|
}
|
|
|
|
static u32 hex2int(char x)
|
|
{
|
|
ASSERT(isxdigit(x));
|
|
if (x >= '0' && x <= '9')
|
|
return x - '0';
|
|
return 10u + (tolower(x) - 'a');
|
|
}
|
|
|
|
String Token::string_value(StringValueStatus& status) const
|
|
{
|
|
ASSERT(type() == TokenType::StringLiteral || type() == TokenType::TemplateLiteralString);
|
|
auto is_template = type() == TokenType::TemplateLiteralString;
|
|
auto offset = is_template ? 0 : 1;
|
|
|
|
size_t i;
|
|
|
|
auto lookahead = [&]<typename T>(T fn, size_t distance = 1) -> bool {
|
|
if (i + distance >= m_value.length() - offset)
|
|
return false;
|
|
return fn(m_value[i + distance]);
|
|
};
|
|
|
|
auto is_octal_digit = [](char c) {
|
|
return c >= '0' && c <= '7';
|
|
};
|
|
|
|
auto encoding_failure = [&status](StringValueStatus parse_status) -> String {
|
|
status = parse_status;
|
|
return {};
|
|
};
|
|
|
|
StringBuilder builder;
|
|
for (i = offset; i < m_value.length() - offset; ++i) {
|
|
if (m_value[i] == '\\' && i + 1 < m_value.length() - offset) {
|
|
i++;
|
|
switch (m_value[i]) {
|
|
case 'b':
|
|
builder.append('\b');
|
|
break;
|
|
case 'f':
|
|
builder.append('\f');
|
|
break;
|
|
case 'n':
|
|
builder.append('\n');
|
|
break;
|
|
case 'r':
|
|
builder.append('\r');
|
|
break;
|
|
case 't':
|
|
builder.append('\t');
|
|
break;
|
|
case 'v':
|
|
builder.append('\v');
|
|
break;
|
|
case '\'':
|
|
builder.append('\'');
|
|
break;
|
|
case '"':
|
|
builder.append('"');
|
|
break;
|
|
case '\\':
|
|
builder.append('\\');
|
|
break;
|
|
case 'x': {
|
|
if (i + 2 >= m_value.length() - offset)
|
|
return encoding_failure(StringValueStatus::MalformedHexEscape);
|
|
|
|
auto digit1 = m_value[++i];
|
|
auto digit2 = m_value[++i];
|
|
if (!isxdigit(digit1) || !isxdigit(digit2))
|
|
return encoding_failure(StringValueStatus::MalformedHexEscape);
|
|
builder.append_code_point(hex2int(digit1) * 16 + hex2int(digit2));
|
|
break;
|
|
}
|
|
case 'u': {
|
|
if (i + 1 >= m_value.length() - offset)
|
|
return encoding_failure(StringValueStatus::MalformedUnicodeEscape);
|
|
u32 code_point = m_value[++i];
|
|
|
|
if (code_point == '{') {
|
|
code_point = 0;
|
|
while (true) {
|
|
if (i + 1 >= m_value.length() - offset)
|
|
return encoding_failure(StringValueStatus::MalformedUnicodeEscape);
|
|
|
|
auto ch = m_value[++i];
|
|
if (ch == '}')
|
|
break;
|
|
if (!isxdigit(ch))
|
|
return encoding_failure(StringValueStatus::MalformedUnicodeEscape);
|
|
|
|
auto new_code_point = (code_point << 4u) | hex2int(ch);
|
|
if (new_code_point < code_point)
|
|
return encoding_failure(StringValueStatus::UnicodeEscapeOverflow);
|
|
code_point = new_code_point;
|
|
}
|
|
} else {
|
|
if (i + 3 >= m_value.length() - offset || !isxdigit(code_point))
|
|
return encoding_failure(StringValueStatus::MalformedUnicodeEscape);
|
|
|
|
code_point = hex2int(code_point);
|
|
for (int j = 0; j < 3; ++j) {
|
|
auto ch = m_value[++i];
|
|
if (!isxdigit(ch))
|
|
return encoding_failure(StringValueStatus::MalformedUnicodeEscape);
|
|
code_point = (code_point << 4u) | hex2int(ch);
|
|
}
|
|
}
|
|
|
|
builder.append_code_point(code_point);
|
|
break;
|
|
}
|
|
default:
|
|
if (is_template && (m_value[i] == '$' || m_value[i] == '`')) {
|
|
builder.append(m_value[i]);
|
|
break;
|
|
}
|
|
if (m_value[i] == '0' && !lookahead(isdigit)) {
|
|
builder.append((char)0);
|
|
break;
|
|
}
|
|
|
|
// In non-strict mode LegacyOctalEscapeSequence is allowed in strings:
|
|
// https://tc39.es/ecma262/#sec-additional-syntax-string-literals
|
|
String octal_str;
|
|
|
|
// OctalDigit [lookahead ∉ OctalDigit]
|
|
if (is_octal_digit(m_value[i]) && !lookahead(is_octal_digit)) {
|
|
status = StringValueStatus::LegacyOctalEscapeSequence;
|
|
octal_str = String(&m_value[i], 1);
|
|
}
|
|
// ZeroToThree OctalDigit [lookahead ∉ OctalDigit]
|
|
else if (m_value[i] >= '0' && m_value[i] <= '3' && lookahead(is_octal_digit) && !lookahead(is_octal_digit, 2)) {
|
|
status = StringValueStatus::LegacyOctalEscapeSequence;
|
|
octal_str = String(m_value.substring_view(i, 2));
|
|
i++;
|
|
}
|
|
// FourToSeven OctalDigit
|
|
else if (m_value[i] >= '4' && m_value[i] <= '7' && lookahead(is_octal_digit)) {
|
|
status = StringValueStatus::LegacyOctalEscapeSequence;
|
|
octal_str = String(m_value.substring_view(i, 2));
|
|
i++;
|
|
}
|
|
// ZeroToThree OctalDigit OctalDigit
|
|
else if (m_value[i] >= '0' && m_value[i] <= '3' && lookahead(is_octal_digit) && lookahead(is_octal_digit, 2)) {
|
|
status = StringValueStatus::LegacyOctalEscapeSequence;
|
|
octal_str = String(m_value.substring_view(i, 3));
|
|
i += 2;
|
|
}
|
|
|
|
if (status == StringValueStatus::LegacyOctalEscapeSequence)
|
|
builder.append_code_point(strtoul(octal_str.characters(), nullptr, 8));
|
|
else
|
|
builder.append(m_value[i]);
|
|
}
|
|
} else {
|
|
builder.append(m_value[i]);
|
|
}
|
|
}
|
|
return builder.to_string();
|
|
}
|
|
|
|
bool Token::bool_value() const
|
|
{
|
|
ASSERT(type() == TokenType::BoolLiteral);
|
|
return m_value == "true";
|
|
}
|
|
|
|
bool Token::is_identifier_name() const
|
|
{
|
|
// IdentifierNames are Identifiers + ReservedWords
|
|
// The standard defines this reversed: Identifiers are IdentifierNames except reserved words
|
|
// https://www.ecma-international.org/ecma-262/5.1/#sec-7.6
|
|
return m_type == TokenType::Identifier
|
|
|| m_type == TokenType::Await
|
|
|| m_type == TokenType::BoolLiteral
|
|
|| m_type == TokenType::Break
|
|
|| m_type == TokenType::Case
|
|
|| m_type == TokenType::Catch
|
|
|| m_type == TokenType::Class
|
|
|| m_type == TokenType::Const
|
|
|| m_type == TokenType::Continue
|
|
|| m_type == TokenType::Default
|
|
|| m_type == TokenType::Delete
|
|
|| m_type == TokenType::Do
|
|
|| m_type == TokenType::Else
|
|
|| m_type == TokenType::Enum
|
|
|| m_type == TokenType::Export
|
|
|| m_type == TokenType::Extends
|
|
|| m_type == TokenType::Finally
|
|
|| m_type == TokenType::For
|
|
|| m_type == TokenType::Function
|
|
|| m_type == TokenType::If
|
|
|| m_type == TokenType::Import
|
|
|| m_type == TokenType::In
|
|
|| m_type == TokenType::Instanceof
|
|
|| m_type == TokenType::Interface
|
|
|| m_type == TokenType::Let
|
|
|| m_type == TokenType::New
|
|
|| m_type == TokenType::NullLiteral
|
|
|| m_type == TokenType::Return
|
|
|| m_type == TokenType::Super
|
|
|| m_type == TokenType::Switch
|
|
|| m_type == TokenType::This
|
|
|| m_type == TokenType::Throw
|
|
|| m_type == TokenType::Try
|
|
|| m_type == TokenType::Typeof
|
|
|| m_type == TokenType::Var
|
|
|| m_type == TokenType::Void
|
|
|| m_type == TokenType::While
|
|
|| m_type == TokenType::Yield;
|
|
}
|
|
|
|
bool Token::trivia_contains_line_terminator() const
|
|
{
|
|
return m_trivia.contains('\n') || m_trivia.contains('\r') || m_trivia.contains(LINE_SEPARATOR) || m_trivia.contains(PARAGRAPH_SEPARATOR);
|
|
}
|
|
|
|
}
|