LibJS: Add message string to Token

This allows us to communicate details about invalid tokens to the parser
without having to invent a bunch of specific invalid tokens like
TokenType::InvalidNumericLiteral.

Libraries/LibJS/Lexer.cpp

@@ -42,7 +42,7 @@ HashMap<char, TokenType> Lexer::s_single_char_tokens;
 
 Lexer::Lexer(StringView source)
     : m_source(source)
-    , m_current_token(TokenType::Eof, StringView(nullptr), StringView(nullptr), 0, 0)
+    , m_current_token(TokenType::Eof, {}, StringView(nullptr), StringView(nullptr), 0, 0)
 {
     if (s_keywords.is_empty()) {
         s_keywords.set("await", TokenType::Await);
@@ -392,6 +392,10 @@ Token Lexer::next()
     size_t value_start_line_number = m_line_number;
     size_t value_start_column_number = m_line_column;
     auto token_type = TokenType::Invalid;
+    // This is being used to communicate info about invalid tokens to the parser, which then
+    // can turn that into more specific error messages - instead of us having to make up a
+    // bunch of Invalid* tokens (bad numeric literals, unterminated comments etc.)
+    String token_message;
 
     if (m_current_token.type() == TokenType::RegexLiteral && !is_eof() && isalpha(m_current_char)) {
         token_type = TokenType::RegexFlags;
@@ -602,6 +606,7 @@ Token Lexer::next()
 
     m_current_token = Token(
         token_type,
+        token_message,
         m_source.substring_view(trivia_start - 1, value_start - trivia_start),
         m_source.substring_view(value_start - 1, m_position - value_start),
         value_start_line_number,

Libraries/LibJS/Token.h

@@ -181,8 +181,9 @@ enum class TokenCategory {
 
 class Token {
 public:
-    Token(TokenType type, StringView trivia, StringView value, size_t line_number, size_t line_column)
+    Token(TokenType type, String message, StringView trivia, StringView value, size_t line_number, size_t line_column)
         : m_type(type)
+        , m_message(message)
         , m_trivia(trivia)
         , m_value(value)
         , m_line_number(line_number)
@@ -196,6 +197,7 @@ public:
     const char* name() const;
     static const char* name(TokenType);
 
+    const String& message() const { return m_message; }
     const StringView& trivia() const { return m_trivia; }
     const StringView& value() const { return m_value; }
     size_t line_number() const { return m_line_number; }
@@ -217,6 +219,7 @@ public:
 
 private:
     TokenType m_type;
+    String m_message;
     StringView m_trivia;
     StringView m_value;
     size_t m_line_number;