LibJS: Add operator precedence parsing

Obey precedence and associativity rules when parsing expressions
with chained operators.

Base/home/anon/js/operator-precedence.js

@@ -0,0 +1,2 @@
+var foo = 1 + 2 * 3 - 4 / 5;
+foo.bar();

Libraries/LibJS/AST.cpp

@@ -54,7 +54,7 @@ Value ExpressionStatement::execute(Interpreter& interpreter) const
 
 Value CallExpression::execute(Interpreter& interpreter) const
 {
-    auto callee = interpreter.get_variable(name());
+    auto callee = m_callee->execute(interpreter);
     ASSERT(callee.is_object());
     ASSERT(callee.as_object()->is_function());
     auto* function = static_cast<Function*>(callee.as_object());
@@ -304,9 +304,8 @@ void UnaryExpression::dump(int indent) const
 
 void CallExpression::dump(int indent) const
 {
-    print_indent(indent);
-    printf("%s '%s'\n", class_name(), name().characters());
-
+    ASTNode::dump(indent);
+    m_callee->dump(indent + 1);
     for (auto& argument : m_arguments)
         argument.dump(indent + 1);
 }

Libraries/LibJS/AST.h

@@ -398,8 +398,8 @@ private:
 
 class CallExpression : public Expression {
 public:
-    explicit CallExpression(String name, NonnullOwnPtrVector<Expression> arguments = {})
-        : m_name(move(name))
+    explicit CallExpression(NonnullOwnPtr<Expression> callee, NonnullOwnPtrVector<Expression> arguments = {})
+        : m_callee(move(callee))
         , m_arguments(move(arguments))
     {
     }
@@ -407,12 +407,10 @@ public:
     virtual Value execute(Interpreter&) const override;
     virtual void dump(int indent) const override;
 
-    const String& name() const { return m_name; }
-
 private:
     virtual const char* class_name() const override { return "CallExpression"; }
 
-    String m_name;
+    NonnullOwnPtr<Expression> m_callee;
     const NonnullOwnPtrVector<Expression> m_arguments;
 };
 

Libraries/LibJS/Lexer.cpp

@@ -42,31 +42,41 @@ Lexer::Lexer(StringView source)
     , m_current_token(TokenType::Eof, StringView(nullptr), StringView(nullptr))
 {
     if (s_keywords.is_empty()) {
-        s_keywords.set("true", TokenType::BoolLiteral);
-        s_keywords.set("false", TokenType::BoolLiteral);
+        s_keywords.set("await", TokenType::Await);
         s_keywords.set("catch", TokenType::Catch);
         s_keywords.set("class", TokenType::Class);
         s_keywords.set("const", TokenType::Const);
         s_keywords.set("delete", TokenType::Delete);
         s_keywords.set("do", TokenType::Do);
         s_keywords.set("else", TokenType::Else);
+        s_keywords.set("false", TokenType::BoolLiteral);
         s_keywords.set("finally", TokenType::Finally);
         s_keywords.set("for", TokenType::For);
         s_keywords.set("function", TokenType::Function);
         s_keywords.set("if", TokenType::If);
+        s_keywords.set("in", TokenType::In);
+        s_keywords.set("instanceof", TokenType::Instanceof);
         s_keywords.set("interface", TokenType::Interface);
         s_keywords.set("let", TokenType::Let);
         s_keywords.set("new", TokenType::New);
         s_keywords.set("null", TokenType::NullLiteral);
         s_keywords.set("return", TokenType::Return);
+        s_keywords.set("true", TokenType::BoolLiteral);
         s_keywords.set("try", TokenType::Try);
+        s_keywords.set("typeof", TokenType::Typeof);
         s_keywords.set("var", TokenType::Var);
+        s_keywords.set("void", TokenType::Void);
         s_keywords.set("while", TokenType::While);
+        s_keywords.set("yield", TokenType::Yield);
     }
 
     if (s_three_char_tokens.is_empty()) {
         s_three_char_tokens.set("===", TokenType::EqualsEqualsEquals);
         s_three_char_tokens.set("!==", TokenType::ExclamationMarkEqualsEquals);
+        s_three_char_tokens.set("**=", TokenType::AsteriskAsteriskEquals);
+        s_three_char_tokens.set("<<=", TokenType::ShiftLeftEquals);
+        s_three_char_tokens.set(">>=", TokenType::ShiftRightEquals);
+        s_three_char_tokens.set(">>>", TokenType::UnsignedShiftRight);
     }
 
     if (s_two_char_tokens.is_empty()) {
@@ -79,12 +89,15 @@ Lexer::Lexer(StringView source)
         s_two_char_tokens.set("|=", TokenType::PipeEquals);
         s_two_char_tokens.set("&&", TokenType::DoubleAmpersand);
         s_two_char_tokens.set("||", TokenType::DoublePipe);
+        s_two_char_tokens.set("??", TokenType::DoubleQuestionMark);
+        s_two_char_tokens.set("**", TokenType::DoubleAsterisk);
         s_two_char_tokens.set("==", TokenType::EqualsEquals);
         s_two_char_tokens.set("!=", TokenType::ExclamationMarkEquals);
         s_two_char_tokens.set("--", TokenType::MinusMinus);
         s_two_char_tokens.set("++", TokenType::PlusPlus);
         s_two_char_tokens.set("<<", TokenType::ShiftLeft);
         s_two_char_tokens.set(">>", TokenType::ShiftRight);
+        s_two_char_tokens.set("?.", TokenType::QuestionMarkPeriod);
     }
 
     if (s_single_char_tokens.is_empty()) {
@@ -92,6 +105,7 @@ Lexer::Lexer(StringView source)
         s_single_char_tokens.set('*', TokenType::Asterisk);
         s_single_char_tokens.set('[', TokenType::BracketOpen);
         s_single_char_tokens.set(']', TokenType::BracketClose);
+        s_single_char_tokens.set('^', TokenType::Caret);
         s_single_char_tokens.set(',', TokenType::Comma);
         s_single_char_tokens.set('{', TokenType::CurlyOpen);
         s_single_char_tokens.set('}', TokenType::CurlyClose);
@@ -107,6 +121,7 @@ Lexer::Lexer(StringView source)
         s_single_char_tokens.set('?', TokenType::QuestionMark);
         s_single_char_tokens.set(';', TokenType::Semicolon);
         s_single_char_tokens.set('/', TokenType::Slash);
+        s_single_char_tokens.set('~', TokenType::Tilde);
         s_single_char_tokens.set('<', TokenType::LessThan);
         s_single_char_tokens.set('>', TokenType::GreaterThan);
     }
@@ -213,8 +228,25 @@ Token Lexer::next()
     } else if (m_current_char == EOF) {
         token_type = TokenType::Eof;
     } else {
+        // There is only one four-char operator: >>>=
+        bool found_four_char_token = false;
+        if (m_position + 2 < m_source.length()) {
+            if (m_current_char == '>'
+                && m_source[m_position] == '>'
+                && m_source[m_position + 1] == '>'
+                && m_source[m_position + 2] == '=') {
+
+                found_four_char_token = true;
+                consume();
+                consume();
+                consume();
+                consume();
+                token_type = TokenType::UnsignedShiftRightEquals;
+            }
+        }
+
         bool found_three_char_token = false;
-        if (m_position + 1 < m_source.length()) {
+        if (!found_four_char_token && m_position + 1 < m_source.length()) {
             char second_char = m_source[m_position];
             char third_char = m_source[m_position + 1];
             char three_chars[] { (char)m_current_char, second_char, third_char, 0 };
@@ -229,7 +261,7 @@ Token Lexer::next()
         }
 
         bool found_two_char_token = false;
-        if (!found_three_char_token && m_position < m_source.length()) {
+        if (!found_four_char_token && !found_three_char_token && m_position < m_source.length()) {
             char second_char = m_source[m_position];
             char two_chars[] { (char)m_current_char, second_char, 0 };
             auto it = s_two_char_tokens.find(two_chars);
@@ -242,7 +274,7 @@ Token Lexer::next()
         }
 
         bool found_one_char_token = false;
-        if (!found_three_char_token && !found_two_char_token) {
+        if (!found_four_char_token && !found_three_char_token && !found_two_char_token) {
             auto it = s_single_char_tokens.find(m_current_char);
             if (it != s_single_char_tokens.end()) {
                 found_one_char_token = true;
@@ -251,7 +283,7 @@ Token Lexer::next()
             }
         }
 
-        if (!found_three_char_token && !found_two_char_token && !found_one_char_token) {
+        if (!found_four_char_token && !found_three_char_token && !found_two_char_token && !found_one_char_token) {
             consume();
             token_type = TokenType::Invalid;
         }

Libraries/LibJS/Parser.cpp

@@ -25,14 +25,142 @@
  */
 
 #include "Parser.h"
+#include <AK/HashMap.h>
 #include <AK/StdLibExtras.h>
 #include <stdio.h>
 
 namespace JS {
+
+static HashMap<TokenType, int> g_operator_precedence;
+
 Parser::Parser(Lexer lexer)
     : m_lexer(move(lexer))
     , m_current_token(m_lexer.next())
 {
+    if (g_operator_precedence.is_empty()) {
+        // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_Precedence
+        g_operator_precedence.set(TokenType::Period, 20);
+        g_operator_precedence.set(TokenType::BracketOpen, 20);
+        g_operator_precedence.set(TokenType::ParenOpen, 20);
+        g_operator_precedence.set(TokenType::QuestionMarkPeriod, 20);
+
+        g_operator_precedence.set(TokenType::New, 19);
+
+        g_operator_precedence.set(TokenType::PlusPlus, 18);
+        g_operator_precedence.set(TokenType::MinusMinus, 18);
+
+        g_operator_precedence.set(TokenType::ExclamationMark, 17);
+        g_operator_precedence.set(TokenType::Tilde, 17);
+        g_operator_precedence.set(TokenType::Typeof, 17);
+        g_operator_precedence.set(TokenType::Void, 17);
+        g_operator_precedence.set(TokenType::Delete, 17);
+        g_operator_precedence.set(TokenType::Await, 17);
+
+        g_operator_precedence.set(TokenType::DoubleAsterisk, 16);
+
+        g_operator_precedence.set(TokenType::Asterisk, 15);
+        g_operator_precedence.set(TokenType::Slash, 15);
+        g_operator_precedence.set(TokenType::Percent, 15);
+
+        g_operator_precedence.set(TokenType::Plus, 14);
+        g_operator_precedence.set(TokenType::Minus, 14);
+
+        g_operator_precedence.set(TokenType::ShiftLeft, 13);
+        g_operator_precedence.set(TokenType::ShiftRight, 13);
+        g_operator_precedence.set(TokenType::UnsignedShiftRight, 13);
+
+        g_operator_precedence.set(TokenType::LessThan, 12);
+        g_operator_precedence.set(TokenType::LessThanEquals, 12);
+        g_operator_precedence.set(TokenType::GreaterThan, 12);
+        g_operator_precedence.set(TokenType::GreaterThanEquals, 12);
+        g_operator_precedence.set(TokenType::In, 12);
+        g_operator_precedence.set(TokenType::Instanceof, 12);
+
+        g_operator_precedence.set(TokenType::EqualsEquals, 11);
+        g_operator_precedence.set(TokenType::ExclamationMarkEquals, 11);
+        g_operator_precedence.set(TokenType::EqualsEqualsEquals, 11);
+        g_operator_precedence.set(TokenType::ExclamationMarkEqualsEquals, 11);
+
+        g_operator_precedence.set(TokenType::Ampersand, 10);
+
+        g_operator_precedence.set(TokenType::Caret, 9);
+
+        g_operator_precedence.set(TokenType::Pipe, 8);
+
+        g_operator_precedence.set(TokenType::DoubleQuestionMark, 7);
+
+        g_operator_precedence.set(TokenType::DoubleAmpersand, 6);
+
+        g_operator_precedence.set(TokenType::DoublePipe, 5);
+
+        g_operator_precedence.set(TokenType::QuestionMark, 4);
+
+        g_operator_precedence.set(TokenType::Equals, 3);
+        g_operator_precedence.set(TokenType::PlusEquals, 3);
+        g_operator_precedence.set(TokenType::MinusEquals, 3);
+        g_operator_precedence.set(TokenType::AsteriskAsteriskEquals, 3);
+        g_operator_precedence.set(TokenType::AsteriskEquals, 3);
+        g_operator_precedence.set(TokenType::SlashEquals, 3);
+        g_operator_precedence.set(TokenType::PercentEquals, 3);
+        g_operator_precedence.set(TokenType::ShiftLeftEquals, 3);
+        g_operator_precedence.set(TokenType::ShiftRightEquals, 3);
+        g_operator_precedence.set(TokenType::UnsignedShiftRightEquals, 3);
+        g_operator_precedence.set(TokenType::PipeEquals, 3);
+
+        g_operator_precedence.set(TokenType::Yield, 2);
+
+        g_operator_precedence.set(TokenType::Comma, 1);
+    }
+}
+
+int Parser::operator_precedence(TokenType type) const
+{
+    auto it = g_operator_precedence.find(type);
+    if (it == g_operator_precedence.end()) {
+        fprintf(stderr, "No precedence for operator %s\n", Token::name(type));
+        ASSERT_NOT_REACHED();
+        return -1;
+    }
+
+    return it->value;
+}
+
+Associativity Parser::operator_associativity(TokenType type) const
+{
+    switch (type) {
+    case TokenType::Period:
+    case TokenType::BracketOpen:
+    case TokenType::ParenOpen:
+    case TokenType::QuestionMarkPeriod:
+    case TokenType::Asterisk:
+    case TokenType::Slash:
+    case TokenType::Percent:
+    case TokenType::Plus:
+    case TokenType::Minus:
+    case TokenType::ShiftLeft:
+    case TokenType::ShiftRight:
+    case TokenType::UnsignedShiftRight:
+    case TokenType::LessThan:
+    case TokenType::LessThanEquals:
+    case TokenType::GreaterThan:
+    case TokenType::GreaterThanEquals:
+    case TokenType::In:
+    case TokenType::Instanceof:
+    case TokenType::EqualsEquals:
+    case TokenType::ExclamationMarkEquals:
+    case TokenType::EqualsEqualsEquals:
+    case TokenType::ExclamationMarkEqualsEquals:
+    case TokenType::Ampersand:
+    case TokenType::Caret:
+    case TokenType::Pipe:
+    case TokenType::DoubleQuestionMark:
+    case TokenType::DoubleAmpersand:
+    case TokenType::DoublePipe:
+    case TokenType::Comma:
+        return Associativity::Left;
+    default:
+        return Associativity::Right;
+    }
 }
 
 NonnullOwnPtr<Program> Parser::parse_program()
@@ -54,7 +182,7 @@ NonnullOwnPtr<Program> Parser::parse_program()
 NonnullOwnPtr<Statement> Parser::parse_statement()
 {
     if (match_expression()) {
-        return make<JS::ExpressionStatement>(parse_expression());
+        return make<JS::ExpressionStatement>(parse_expression(0));
     }
 
     switch (m_current_token.type()) {
@@ -83,7 +211,7 @@ NonnullOwnPtr<Expression> Parser::parse_primary_expression()
     switch (m_current_token.type()) {
     case TokenType::ParenOpen: {
         consume(TokenType::ParenOpen);
-        auto expression = parse_expression();
+        auto expression = parse_expression(0);
         consume(TokenType::ParenClose);
         return expression;
     }
@@ -113,68 +241,75 @@ NonnullOwnPtr<ObjectExpression> Parser::parse_object_expression()
     return make<ObjectExpression>();
 }
 
-NonnullOwnPtr<Expression> Parser::parse_expression()
+NonnullOwnPtr<Expression> Parser::parse_expression(int min_precedence, Associativity associativity)
 {
     auto expression = parse_primary_expression();
     while (match_secondary_expression()) {
-        expression = parse_secondary_expression(move(expression));
+        int new_precedence = operator_precedence(m_current_token.type());
+        if (new_precedence < min_precedence)
+            break;
+        if (new_precedence == min_precedence && associativity == Associativity::Left)
+            break;
+
+        Associativity new_associativity = operator_associativity(m_current_token.type());
+        expression = parse_secondary_expression(move(expression), new_precedence, new_associativity);
     }
     return expression;
 }
 
-NonnullOwnPtr<Expression> Parser::parse_secondary_expression(NonnullOwnPtr<Expression> lhs)
+NonnullOwnPtr<Expression> Parser::parse_secondary_expression(NonnullOwnPtr<Expression> lhs, int min_precedence, Associativity associativity)
 {
     switch (m_current_token.type()) {
     case TokenType::Plus:
         consume();
-        return make<BinaryExpression>(BinaryOp::Plus, move(lhs), parse_expression());
+        return make<BinaryExpression>(BinaryOp::Plus, move(lhs), parse_expression(min_precedence, associativity));
     case TokenType::PlusEquals:
         consume();
-        return make<AssignmentExpression>(AssignmentOp::AdditionAssignment, move(lhs), parse_expression());
+        return make<AssignmentExpression>(AssignmentOp::AdditionAssignment, move(lhs), parse_expression(min_precedence, associativity));
     case TokenType::Minus:
         consume();
-        return make<BinaryExpression>(BinaryOp::Minus, move(lhs), parse_expression());
+        return make<BinaryExpression>(BinaryOp::Minus, move(lhs), parse_expression(min_precedence, associativity));
     case TokenType::MinusEquals:
         consume();
-        return make<AssignmentExpression>(AssignmentOp::SubtractionAssignment, move(lhs), parse_expression());
+        return make<AssignmentExpression>(AssignmentOp::SubtractionAssignment, move(lhs), parse_expression(min_precedence, associativity));
     case TokenType::Asterisk:
         consume();
-        return make<BinaryExpression>(BinaryOp::Asterisk, move(lhs), parse_expression());
+        return make<BinaryExpression>(BinaryOp::Asterisk, move(lhs), parse_expression(min_precedence, associativity));
     case TokenType::AsteriskEquals:
         consume();
-        return make<AssignmentExpression>(AssignmentOp::MultiplicationAssignment, move(lhs), parse_expression());
+        return make<AssignmentExpression>(AssignmentOp::MultiplicationAssignment, move(lhs), parse_expression(min_precedence, associativity));
     case TokenType::Slash:
         consume();
-        return make<BinaryExpression>(BinaryOp::Slash, move(lhs), parse_expression());
+        return make<BinaryExpression>(BinaryOp::Slash, move(lhs), parse_expression(min_precedence, associativity));
     case TokenType::SlashEquals:
         consume();
-        return make<AssignmentExpression>(AssignmentOp::DivisionAssignment, move(lhs), parse_expression());
+        return make<AssignmentExpression>(AssignmentOp::DivisionAssignment, move(lhs), parse_expression(min_precedence, associativity));
     case TokenType::GreaterThan:
         consume();
-        return make<BinaryExpression>(BinaryOp::GreaterThan, move(lhs), parse_expression());
+        return make<BinaryExpression>(BinaryOp::GreaterThan, move(lhs), parse_expression(min_precedence, associativity));
     case TokenType::GreaterThanEquals:
         consume();
-        return make<BinaryExpression>(BinaryOp::GreaterThanEquals, move(lhs), parse_expression());
+        return make<BinaryExpression>(BinaryOp::GreaterThanEquals, move(lhs), parse_expression(min_precedence, associativity));
     case TokenType::LessThan:
         consume();
-        return make<BinaryExpression>(BinaryOp::LessThan, move(lhs), parse_expression());
+        return make<BinaryExpression>(BinaryOp::LessThan, move(lhs), parse_expression(min_precedence, associativity));
     case TokenType::LessThanEquals:
         consume();
-        return make<BinaryExpression>(BinaryOp::LessThanEquals, move(lhs), parse_expression());
+        return make<BinaryExpression>(BinaryOp::LessThanEquals, move(lhs), parse_expression(min_precedence, associativity));
     case TokenType::EqualsEqualsEquals:
         consume();
-        return make<BinaryExpression>(BinaryOp::TypedEquals, move(lhs), parse_expression());
+        return make<BinaryExpression>(BinaryOp::TypedEquals, move(lhs), parse_expression(min_precedence, associativity));
     case TokenType::ExclamationMarkEqualsEquals:
         consume();
-        return make<BinaryExpression>(BinaryOp::TypedInequals, move(lhs), parse_expression());
+        return make<BinaryExpression>(BinaryOp::TypedInequals, move(lhs), parse_expression(min_precedence, associativity));
     case TokenType::ParenOpen:
         return parse_call_expression(move(lhs));
     case TokenType::Equals:
         consume();
-        return make<AssignmentExpression>(AssignmentOp::Assignment, move(lhs), parse_expression());
+        return make<AssignmentExpression>(AssignmentOp::Assignment, move(lhs), parse_expression(min_precedence, associativity));
     case TokenType::Period:
         consume();
-        return make<MemberExpression>(move(lhs), parse_expression());
+        return make<MemberExpression>(move(lhs), parse_expression(min_precedence, associativity));
     case TokenType::PlusPlus:
         consume();
         return make<UpdateExpression>(UpdateOp::Increment, move(lhs));
@@ -196,7 +331,7 @@ NonnullOwnPtr<CallExpression> Parser::parse_call_expression(NonnullOwnPtr<Expres
     NonnullOwnPtrVector<Expression> arguments;
 
     while (match_expression()) {
-        arguments.append(parse_expression());
+        arguments.append(parse_expression(0));
         if (!match(TokenType::Comma))
             break;
         consume();
@@ -204,20 +339,14 @@ NonnullOwnPtr<CallExpression> Parser::parse_call_expression(NonnullOwnPtr<Expres
 
     consume(TokenType::ParenClose);
 
-    // FIXME: Allow lhs expression instead of just a string
-    if (lhs->is_identifier()) {
-        return make<CallExpression>(static_cast<Identifier*>(lhs.ptr())->string(), move(arguments));
-    }
-
-    m_has_errors = true;
-    return make<CallExpression>("***ERROR***");
+    return make<CallExpression>(move(lhs), move(arguments));
 }
 
 NonnullOwnPtr<ReturnStatement> Parser::parse_return_statement()
 {
     consume(TokenType::Return);
     if (match_expression()) {
-        return make<ReturnStatement>(parse_expression());
+        return make<ReturnStatement>(parse_expression(0));
     }
     return make<ReturnStatement>(nullptr);
 }
@@ -283,7 +412,7 @@ NonnullOwnPtr<VariableDeclaration> Parser::parse_variable_declaration()
     OwnPtr<Expression> initializer;
     if (match(TokenType::Equals)) {
         consume();
-        initializer = parse_expression();
+        initializer = parse_expression(0);
     }
     return make<VariableDeclaration>(make<Identifier>(name), move(initializer), declaration_type);
 }
@@ -313,7 +442,7 @@ NonnullOwnPtr<ForStatement> Parser::parse_for_statement()
     case TokenType::Semicolon:
         break;
     default:
-        test = parse_expression();
+        test = parse_expression(0);
         break;
     }
 
@@ -324,7 +453,7 @@ NonnullOwnPtr<ForStatement> Parser::parse_for_statement()
     case TokenType::Semicolon:
         break;
     default:
-        update = parse_expression();
+        update = parse_expression(0);
         break;
     }
 
@@ -405,9 +534,9 @@ bool Parser::done() const
 
 Token Parser::consume()
 {
-    auto oldToken = m_current_token;
+    auto old_token = m_current_token;
     m_current_token = m_lexer.next();
-    return oldToken;
+    return old_token;
 }
 
 Token Parser::consume(TokenType type)

Libraries/LibJS/Parser.h

@@ -31,6 +31,12 @@
 #include <AK/NonnullOwnPtr.h>
 
 namespace JS {
+
+enum class Associativity {
+    Left,
+    Right
+};
+
 class Parser {
 public:
     explicit Parser(Lexer lexer);
@@ -44,15 +50,17 @@ public:
     NonnullOwnPtr<VariableDeclaration> parse_variable_declaration();
     NonnullOwnPtr<ForStatement> parse_for_statement();
 
-    NonnullOwnPtr<Expression> parse_expression();
+    NonnullOwnPtr<Expression> parse_expression(int min_precedence, Associativity associate = Associativity::Right);
     NonnullOwnPtr<Expression> parse_primary_expression();
     NonnullOwnPtr<ObjectExpression> parse_object_expression();
-    NonnullOwnPtr<Expression> parse_secondary_expression(NonnullOwnPtr<Expression>);
+    NonnullOwnPtr<Expression> parse_secondary_expression(NonnullOwnPtr<Expression>, int min_precedence, Associativity associate = Associativity::Right);
     NonnullOwnPtr<CallExpression> parse_call_expression(NonnullOwnPtr<Expression>);
 
     bool has_errors() const { return m_has_errors; }
 
 private:
+    int operator_precedence(TokenType) const;
+    Associativity operator_associativity(TokenType) const;
     bool match_expression() const;
     bool match_secondary_expression() const;
     bool match_statement() const;

Libraries/LibJS/Token.cpp

@@ -25,6 +25,7 @@
  */
 
 #include "Token.h"
+#include <AK/Assertions.h>
 
 namespace JS {
 
@@ -37,14 +38,20 @@ const char* Token::name(TokenType type)
         return "AmpersandEquals";
     case TokenType::Asterisk:
         return "Asterisk";
+    case TokenType::AsteriskAsteriskEquals:
+        return "AsteriskAsteriskEquals";
     case TokenType::AsteriskEquals:
         return "AsteriskEquals";
+    case TokenType::Await:
+        return "Await";
     case TokenType::BoolLiteral:
         return "BoolLiteral";
     case TokenType::BracketOpen:
         return "BracketOpen";
     case TokenType::BracketClose:
         return "BracketClose";
+    case TokenType::Caret:
+        return "Caret";
     case TokenType::Catch:
         return "Catch";
     case TokenType::Class:
@@ -63,8 +70,12 @@ const char* Token::name(TokenType type)
         return "Do";
     case TokenType::DoubleAmpersand:
         return "DoubleAmpersand";
+    case TokenType::DoubleAsterisk:
+        return "DoubleAsterisk";
     case TokenType::DoublePipe:
         return "DoublePipe";
+    case TokenType::DoubleQuestionMark:
+        return "DoubleQuestionMark";
     case TokenType::Else:
         return "Else";
     case TokenType::Eof:
@@ -77,6 +88,8 @@ const char* Token::name(TokenType type)
         return "ExclamationMark";
     case TokenType::ExclamationMarkEquals:
         return "ExclamationMarkEquals";
+    case TokenType::ExclamationMarkEqualsEquals:
+        return "ExclamationMarkEqualsEquals";
     case TokenType::Finally:
         return "Finally";
     case TokenType::Function:
@@ -87,6 +100,10 @@ const char* Token::name(TokenType type)
         return "Identifier";
     case TokenType::If:
         return "If";
+    case TokenType::In:
+        return "In";
+    case TokenType::Instanceof:
+        return "Instanceof";
     case TokenType::Interface:
         return "Interface";
     case TokenType::Invalid:
@@ -129,6 +146,8 @@ const char* Token::name(TokenType type)
         return "PlusPlus";
     case TokenType::QuestionMark:
         return "QuestionMark";
+    case TokenType::QuestionMarkPeriod:
+        return "QuestionMarkPeriod";
     case TokenType::RegexLiteral:
         return "RegexLiteral";
     case TokenType::Return:
@@ -137,21 +156,38 @@ const char* Token::name(TokenType type)
         return "Semicolon";
     case TokenType::ShiftLeft:
         return "ShiftLeft";
+    case TokenType::ShiftLeftEquals:
+        return "ShiftLeftEquals";
     case TokenType::ShiftRight:
         return "ShiftRight";
+    case TokenType::ShiftRightEquals:
+        return "ShiftRightEquals";
     case TokenType::Slash:
         return "Slash";
     case TokenType::SlashEquals:
         return "SlashEquals";
     case TokenType::StringLiteral:
         return "StringLiteral";
+    case TokenType::Tilde:
+        return "Tilde";
     case TokenType::Try:
         return "Try";
+    case TokenType::Typeof:
+        return "Typeof";
+    case TokenType::UnsignedShiftRight:
+        return "UnsignedShiftRight";
+    case TokenType::UnsignedShiftRightEquals:
+        return "UnsignedShiftRightEquals";
     case TokenType::Var:
         return "Var";
+    case TokenType::Void:
+        return "Void";
     case TokenType::While:
         return "While";
+    case TokenType::Yield:
+        return "Yield";
     default:
+        ASSERT_NOT_REACHED();
         return "<Unknown>";
     }
 }

Libraries/LibJS/Token.h

@@ -35,10 +35,13 @@ enum class TokenType {
     Ampersand,
     AmpersandEquals,
     Asterisk,
+    AsteriskAsteriskEquals,
     AsteriskEquals,
+    Await,
     BoolLiteral,
     BracketClose,
     BracketOpen,
+    Caret,
     Catch,
     Class,
     Comma,
@@ -48,7 +51,9 @@ enum class TokenType {
     Delete,
     Do,
     DoubleAmpersand,
+    DoubleAsterisk,
     DoublePipe,
+    DoubleQuestionMark,
     Else,
     Eof,
     Equals,
@@ -64,6 +69,8 @@ enum class TokenType {
     GreaterThanEquals,
     Identifier,
     If,
+    In,
+    Instanceof,
     Interface,
     Invalid,
     LessThan,
@@ -86,17 +93,26 @@ enum class TokenType {
     PlusEquals,
     PlusPlus,
     QuestionMark,
+    QuestionMarkPeriod,
     RegexLiteral,
     Return,
     Semicolon,
     ShiftLeft,
+    ShiftLeftEquals,
     ShiftRight,
+    ShiftRightEquals,
     Slash,
     SlashEquals,
     StringLiteral,
+    Tilde,
     Try,
+    Typeof,
+    UnsignedShiftRight,
+    UnsignedShiftRightEquals,
     Var,
-    While
+    Void,
+    While,
+    Yield,
 };
 
 class Token {
@@ -125,3 +141,11 @@ private:
 };
 
 }
+
+namespace AK {
+template<>
+struct Traits<JS::TokenType> : public GenericTraits<JS::TokenType> {
+    static constexpr bool is_trivial() { return true; }
+    static unsigned hash(JS::TokenType t) { return int_hash((int)t); }
+};
+}