Lexer.cpp 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651
  1. /*
  2. * Copyright (c) 2020, Stephan Unverwerth <s.unverwerth@serenityos.org>
  3. * Copyright (c) 2020-2021, Linus Groh <linusg@serenityos.org>
  4. *
  5. * SPDX-License-Identifier: BSD-2-Clause
  6. */
  7. #include "Lexer.h"
  8. #include <AK/CharacterTypes.h>
  9. #include <AK/Debug.h>
  10. #include <AK/HashMap.h>
  11. #include <stdio.h>
  12. namespace JS {
  13. HashMap<String, TokenType> Lexer::s_keywords;
  14. HashMap<String, TokenType> Lexer::s_three_char_tokens;
  15. HashMap<String, TokenType> Lexer::s_two_char_tokens;
  16. HashMap<char, TokenType> Lexer::s_single_char_tokens;
  17. Lexer::Lexer(StringView source, StringView filename, size_t line_number, size_t line_column)
  18. : m_source(source)
  19. , m_current_token(TokenType::Eof, {}, StringView(nullptr), StringView(nullptr), filename, 0, 0)
  20. , m_filename(filename)
  21. , m_line_number(line_number)
  22. , m_line_column(line_column)
  23. {
  24. if (s_keywords.is_empty()) {
  25. s_keywords.set("await", TokenType::Await);
  26. s_keywords.set("break", TokenType::Break);
  27. s_keywords.set("case", TokenType::Case);
  28. s_keywords.set("catch", TokenType::Catch);
  29. s_keywords.set("class", TokenType::Class);
  30. s_keywords.set("const", TokenType::Const);
  31. s_keywords.set("continue", TokenType::Continue);
  32. s_keywords.set("debugger", TokenType::Debugger);
  33. s_keywords.set("default", TokenType::Default);
  34. s_keywords.set("delete", TokenType::Delete);
  35. s_keywords.set("do", TokenType::Do);
  36. s_keywords.set("else", TokenType::Else);
  37. s_keywords.set("enum", TokenType::Enum);
  38. s_keywords.set("export", TokenType::Export);
  39. s_keywords.set("extends", TokenType::Extends);
  40. s_keywords.set("false", TokenType::BoolLiteral);
  41. s_keywords.set("finally", TokenType::Finally);
  42. s_keywords.set("for", TokenType::For);
  43. s_keywords.set("function", TokenType::Function);
  44. s_keywords.set("if", TokenType::If);
  45. s_keywords.set("import", TokenType::Import);
  46. s_keywords.set("in", TokenType::In);
  47. s_keywords.set("instanceof", TokenType::Instanceof);
  48. s_keywords.set("let", TokenType::Let);
  49. s_keywords.set("new", TokenType::New);
  50. s_keywords.set("null", TokenType::NullLiteral);
  51. s_keywords.set("return", TokenType::Return);
  52. s_keywords.set("super", TokenType::Super);
  53. s_keywords.set("switch", TokenType::Switch);
  54. s_keywords.set("this", TokenType::This);
  55. s_keywords.set("throw", TokenType::Throw);
  56. s_keywords.set("true", TokenType::BoolLiteral);
  57. s_keywords.set("try", TokenType::Try);
  58. s_keywords.set("typeof", TokenType::Typeof);
  59. s_keywords.set("var", TokenType::Var);
  60. s_keywords.set("void", TokenType::Void);
  61. s_keywords.set("while", TokenType::While);
  62. s_keywords.set("with", TokenType::With);
  63. s_keywords.set("yield", TokenType::Yield);
  64. }
  65. if (s_three_char_tokens.is_empty()) {
  66. s_three_char_tokens.set("===", TokenType::EqualsEqualsEquals);
  67. s_three_char_tokens.set("!==", TokenType::ExclamationMarkEqualsEquals);
  68. s_three_char_tokens.set("**=", TokenType::DoubleAsteriskEquals);
  69. s_three_char_tokens.set("<<=", TokenType::ShiftLeftEquals);
  70. s_three_char_tokens.set(">>=", TokenType::ShiftRightEquals);
  71. s_three_char_tokens.set("&&=", TokenType::DoubleAmpersandEquals);
  72. s_three_char_tokens.set("||=", TokenType::DoublePipeEquals);
  73. s_three_char_tokens.set("\?\?=", TokenType::DoubleQuestionMarkEquals);
  74. s_three_char_tokens.set(">>>", TokenType::UnsignedShiftRight);
  75. s_three_char_tokens.set("...", TokenType::TripleDot);
  76. }
  77. if (s_two_char_tokens.is_empty()) {
  78. s_two_char_tokens.set("=>", TokenType::Arrow);
  79. s_two_char_tokens.set("+=", TokenType::PlusEquals);
  80. s_two_char_tokens.set("-=", TokenType::MinusEquals);
  81. s_two_char_tokens.set("*=", TokenType::AsteriskEquals);
  82. s_two_char_tokens.set("/=", TokenType::SlashEquals);
  83. s_two_char_tokens.set("%=", TokenType::PercentEquals);
  84. s_two_char_tokens.set("&=", TokenType::AmpersandEquals);
  85. s_two_char_tokens.set("|=", TokenType::PipeEquals);
  86. s_two_char_tokens.set("^=", TokenType::CaretEquals);
  87. s_two_char_tokens.set("&&", TokenType::DoubleAmpersand);
  88. s_two_char_tokens.set("||", TokenType::DoublePipe);
  89. s_two_char_tokens.set("??", TokenType::DoubleQuestionMark);
  90. s_two_char_tokens.set("**", TokenType::DoubleAsterisk);
  91. s_two_char_tokens.set("==", TokenType::EqualsEquals);
  92. s_two_char_tokens.set("<=", TokenType::LessThanEquals);
  93. s_two_char_tokens.set(">=", TokenType::GreaterThanEquals);
  94. s_two_char_tokens.set("!=", TokenType::ExclamationMarkEquals);
  95. s_two_char_tokens.set("--", TokenType::MinusMinus);
  96. s_two_char_tokens.set("++", TokenType::PlusPlus);
  97. s_two_char_tokens.set("<<", TokenType::ShiftLeft);
  98. s_two_char_tokens.set(">>", TokenType::ShiftRight);
  99. s_two_char_tokens.set("?.", TokenType::QuestionMarkPeriod);
  100. }
  101. if (s_single_char_tokens.is_empty()) {
  102. s_single_char_tokens.set('&', TokenType::Ampersand);
  103. s_single_char_tokens.set('*', TokenType::Asterisk);
  104. s_single_char_tokens.set('[', TokenType::BracketOpen);
  105. s_single_char_tokens.set(']', TokenType::BracketClose);
  106. s_single_char_tokens.set('^', TokenType::Caret);
  107. s_single_char_tokens.set(':', TokenType::Colon);
  108. s_single_char_tokens.set(',', TokenType::Comma);
  109. s_single_char_tokens.set('{', TokenType::CurlyOpen);
  110. s_single_char_tokens.set('}', TokenType::CurlyClose);
  111. s_single_char_tokens.set('=', TokenType::Equals);
  112. s_single_char_tokens.set('!', TokenType::ExclamationMark);
  113. s_single_char_tokens.set('-', TokenType::Minus);
  114. s_single_char_tokens.set('(', TokenType::ParenOpen);
  115. s_single_char_tokens.set(')', TokenType::ParenClose);
  116. s_single_char_tokens.set('%', TokenType::Percent);
  117. s_single_char_tokens.set('.', TokenType::Period);
  118. s_single_char_tokens.set('|', TokenType::Pipe);
  119. s_single_char_tokens.set('+', TokenType::Plus);
  120. s_single_char_tokens.set('?', TokenType::QuestionMark);
  121. s_single_char_tokens.set(';', TokenType::Semicolon);
  122. s_single_char_tokens.set('/', TokenType::Slash);
  123. s_single_char_tokens.set('~', TokenType::Tilde);
  124. s_single_char_tokens.set('<', TokenType::LessThan);
  125. s_single_char_tokens.set('>', TokenType::GreaterThan);
  126. }
  127. consume();
  128. }
  129. void Lexer::consume()
  130. {
  131. auto did_reach_eof = [this] {
  132. if (m_position != m_source.length())
  133. return false;
  134. m_eof = true;
  135. m_current_char = '\0';
  136. m_position++;
  137. m_line_column++;
  138. return true;
  139. };
  140. if (m_position > m_source.length())
  141. return;
  142. if (did_reach_eof())
  143. return;
  144. if (is_line_terminator()) {
  145. if constexpr (LEXER_DEBUG) {
  146. String type;
  147. if (m_current_char == '\n')
  148. type = "LINE FEED";
  149. else if (m_current_char == '\r')
  150. type = "CARRIAGE RETURN";
  151. else if (m_source[m_position + 1] == (char)0xa8)
  152. type = "LINE SEPARATOR";
  153. else
  154. type = "PARAGRAPH SEPARATOR";
  155. dbgln("Found a line terminator: {}", type);
  156. }
  157. // This is a three-char line terminator, we need to increase m_position some more.
  158. // We might reach EOF and need to check again.
  159. if (m_current_char != '\n' && m_current_char != '\r') {
  160. m_position += 2;
  161. if (did_reach_eof())
  162. return;
  163. }
  164. // If the previous character is \r and the current one \n we already updated line number
  165. // and column - don't do it again. From https://tc39.es/ecma262/#sec-line-terminators:
  166. // The sequence <CR><LF> is commonly used as a line terminator.
  167. // It should be considered a single SourceCharacter for the purpose of reporting line numbers.
  168. auto second_char_of_crlf = m_position > 1 && m_source[m_position - 2] == '\r' && m_current_char == '\n';
  169. if (!second_char_of_crlf) {
  170. m_line_number++;
  171. m_line_column = 1;
  172. dbgln_if(LEXER_DEBUG, "Incremented line number, now at: line {}, column 1", m_line_number);
  173. } else {
  174. dbgln_if(LEXER_DEBUG, "Previous was CR, this is LF - not incrementing line number again.");
  175. }
  176. } else {
  177. m_line_column++;
  178. }
  179. m_current_char = m_source[m_position++];
  180. }
  181. bool Lexer::consume_exponent()
  182. {
  183. consume();
  184. if (m_current_char == '-' || m_current_char == '+')
  185. consume();
  186. if (!is_ascii_digit(m_current_char))
  187. return false;
  188. while (is_ascii_digit(m_current_char)) {
  189. consume();
  190. }
  191. return true;
  192. }
  193. bool Lexer::consume_octal_number()
  194. {
  195. consume();
  196. if (!(m_current_char >= '0' && m_current_char <= '7'))
  197. return false;
  198. while (m_current_char >= '0' && m_current_char <= '7') {
  199. consume();
  200. }
  201. return true;
  202. }
  203. bool Lexer::consume_hexadecimal_number()
  204. {
  205. consume();
  206. if (!is_ascii_hex_digit(m_current_char))
  207. return false;
  208. while (is_ascii_hex_digit(m_current_char))
  209. consume();
  210. return true;
  211. }
  212. bool Lexer::consume_binary_number()
  213. {
  214. consume();
  215. if (!(m_current_char == '0' || m_current_char == '1'))
  216. return false;
  217. while (m_current_char == '0' || m_current_char == '1')
  218. consume();
  219. return true;
  220. }
  221. bool Lexer::match(char a, char b) const
  222. {
  223. if (m_position >= m_source.length())
  224. return false;
  225. return m_current_char == a
  226. && m_source[m_position] == b;
  227. }
  228. bool Lexer::match(char a, char b, char c) const
  229. {
  230. if (m_position + 1 >= m_source.length())
  231. return false;
  232. return m_current_char == a
  233. && m_source[m_position] == b
  234. && m_source[m_position + 1] == c;
  235. }
  236. bool Lexer::match(char a, char b, char c, char d) const
  237. {
  238. if (m_position + 2 >= m_source.length())
  239. return false;
  240. return m_current_char == a
  241. && m_source[m_position] == b
  242. && m_source[m_position + 1] == c
  243. && m_source[m_position + 2] == d;
  244. }
  245. bool Lexer::is_eof() const
  246. {
  247. return m_eof;
  248. }
  249. bool Lexer::is_line_terminator() const
  250. {
  251. if (m_current_char == '\n' || m_current_char == '\r')
  252. return true;
  253. if (m_position > 0 && m_position + 1 < m_source.length()) {
  254. auto three_chars_view = m_source.substring_view(m_position - 1, 3);
  255. return (three_chars_view == LINE_SEPARATOR) || (three_chars_view == PARAGRAPH_SEPARATOR);
  256. }
  257. return false;
  258. }
  259. bool Lexer::is_identifier_start() const
  260. {
  261. return is_ascii_alpha(m_current_char) || m_current_char == '_' || m_current_char == '$';
  262. }
  263. bool Lexer::is_identifier_middle() const
  264. {
  265. return is_identifier_start() || is_ascii_digit(m_current_char);
  266. }
  267. bool Lexer::is_line_comment_start(bool line_has_token_yet) const
  268. {
  269. return match('/', '/')
  270. || match('<', '!', '-', '-')
  271. // "-->" is considered a line comment start if the current line is only whitespace and/or
  272. // other block comment(s); or in other words: the current line does not have a token or
  273. // ongoing line comment yet
  274. || (match('-', '-', '>') && !line_has_token_yet)
  275. // https://tc39.es/proposal-hashbang/out.html#sec-updated-syntax
  276. || (match('#', '!') && m_position == 1);
  277. }
  278. bool Lexer::is_block_comment_start() const
  279. {
  280. return match('/', '*');
  281. }
  282. bool Lexer::is_block_comment_end() const
  283. {
  284. return match('*', '/');
  285. }
  286. bool Lexer::is_numeric_literal_start() const
  287. {
  288. return is_ascii_digit(m_current_char) || (m_current_char == '.' && m_position < m_source.length() && is_ascii_digit(m_source[m_position]));
  289. }
  290. bool Lexer::slash_means_division() const
  291. {
  292. auto type = m_current_token.type();
  293. return type == TokenType::BigIntLiteral
  294. || type == TokenType::BoolLiteral
  295. || type == TokenType::BracketClose
  296. || type == TokenType::CurlyClose
  297. || type == TokenType::Identifier
  298. || type == TokenType::NullLiteral
  299. || type == TokenType::NumericLiteral
  300. || type == TokenType::ParenClose
  301. || type == TokenType::RegexLiteral
  302. || type == TokenType::StringLiteral
  303. || type == TokenType::TemplateLiteralEnd
  304. || type == TokenType::This;
  305. }
  306. Token Lexer::next()
  307. {
  308. size_t trivia_start = m_position;
  309. auto in_template = !m_template_states.is_empty();
  310. bool line_has_token_yet = m_line_column > 1;
  311. bool unterminated_comment = false;
  312. if (!in_template || m_template_states.last().in_expr) {
  313. // consume whitespace and comments
  314. while (true) {
  315. if (is_line_terminator()) {
  316. line_has_token_yet = false;
  317. do {
  318. consume();
  319. } while (is_line_terminator());
  320. } else if (is_ascii_space(m_current_char)) {
  321. do {
  322. consume();
  323. } while (is_ascii_space(m_current_char));
  324. } else if (is_line_comment_start(line_has_token_yet)) {
  325. consume();
  326. do {
  327. consume();
  328. } while (!is_eof() && !is_line_terminator());
  329. } else if (is_block_comment_start()) {
  330. consume();
  331. do {
  332. consume();
  333. } while (!is_eof() && !is_block_comment_end());
  334. if (is_eof())
  335. unterminated_comment = true;
  336. consume(); // consume *
  337. if (is_eof())
  338. unterminated_comment = true;
  339. consume(); // consume /
  340. } else {
  341. break;
  342. }
  343. }
  344. }
  345. size_t value_start = m_position;
  346. size_t value_start_line_number = m_line_number;
  347. size_t value_start_column_number = m_line_column;
  348. auto token_type = TokenType::Invalid;
  349. auto did_consume_whitespace_or_comments = trivia_start != value_start;
  350. // This is being used to communicate info about invalid tokens to the parser, which then
  351. // can turn that into more specific error messages - instead of us having to make up a
  352. // bunch of Invalid* tokens (bad numeric literals, unterminated comments etc.)
  353. String token_message;
  354. if (m_current_token.type() == TokenType::RegexLiteral && !is_eof() && is_ascii_alpha(m_current_char) && !did_consume_whitespace_or_comments) {
  355. token_type = TokenType::RegexFlags;
  356. while (!is_eof() && is_ascii_alpha(m_current_char))
  357. consume();
  358. } else if (m_current_char == '`') {
  359. consume();
  360. if (!in_template) {
  361. token_type = TokenType::TemplateLiteralStart;
  362. m_template_states.append({ false, 0 });
  363. } else {
  364. if (m_template_states.last().in_expr) {
  365. m_template_states.append({ false, 0 });
  366. token_type = TokenType::TemplateLiteralStart;
  367. } else {
  368. m_template_states.take_last();
  369. token_type = TokenType::TemplateLiteralEnd;
  370. }
  371. }
  372. } else if (in_template && m_template_states.last().in_expr && m_template_states.last().open_bracket_count == 0 && m_current_char == '}') {
  373. consume();
  374. token_type = TokenType::TemplateLiteralExprEnd;
  375. m_template_states.last().in_expr = false;
  376. } else if (in_template && !m_template_states.last().in_expr) {
  377. if (is_eof()) {
  378. token_type = TokenType::UnterminatedTemplateLiteral;
  379. m_template_states.take_last();
  380. } else if (match('$', '{')) {
  381. token_type = TokenType::TemplateLiteralExprStart;
  382. consume();
  383. consume();
  384. m_template_states.last().in_expr = true;
  385. } else {
  386. while (!match('$', '{') && m_current_char != '`' && !is_eof()) {
  387. if (match('\\', '$') || match('\\', '`'))
  388. consume();
  389. consume();
  390. }
  391. if (is_eof() && !m_template_states.is_empty())
  392. token_type = TokenType::UnterminatedTemplateLiteral;
  393. else
  394. token_type = TokenType::TemplateLiteralString;
  395. }
  396. } else if (is_identifier_start()) {
  397. // identifier or keyword
  398. do {
  399. consume();
  400. } while (is_identifier_middle());
  401. StringView value = m_source.substring_view(value_start - 1, m_position - value_start);
  402. auto it = s_keywords.find(value.hash(), [&](auto& entry) { return entry.key == value; });
  403. if (it == s_keywords.end()) {
  404. token_type = TokenType::Identifier;
  405. } else {
  406. token_type = it->value;
  407. }
  408. } else if (is_numeric_literal_start()) {
  409. token_type = TokenType::NumericLiteral;
  410. bool is_invalid_numeric_literal = false;
  411. if (m_current_char == '0') {
  412. consume();
  413. if (m_current_char == '.') {
  414. // decimal
  415. consume();
  416. while (is_ascii_digit(m_current_char))
  417. consume();
  418. if (m_current_char == 'e' || m_current_char == 'E')
  419. is_invalid_numeric_literal = !consume_exponent();
  420. } else if (m_current_char == 'e' || m_current_char == 'E') {
  421. is_invalid_numeric_literal = !consume_exponent();
  422. } else if (m_current_char == 'o' || m_current_char == 'O') {
  423. // octal
  424. is_invalid_numeric_literal = !consume_octal_number();
  425. if (m_current_char == 'n') {
  426. consume();
  427. token_type = TokenType::BigIntLiteral;
  428. }
  429. } else if (m_current_char == 'b' || m_current_char == 'B') {
  430. // binary
  431. is_invalid_numeric_literal = !consume_binary_number();
  432. if (m_current_char == 'n') {
  433. consume();
  434. token_type = TokenType::BigIntLiteral;
  435. }
  436. } else if (m_current_char == 'x' || m_current_char == 'X') {
  437. // hexadecimal
  438. is_invalid_numeric_literal = !consume_hexadecimal_number();
  439. if (m_current_char == 'n') {
  440. consume();
  441. token_type = TokenType::BigIntLiteral;
  442. }
  443. } else if (m_current_char == 'n') {
  444. consume();
  445. token_type = TokenType::BigIntLiteral;
  446. } else if (is_ascii_digit(m_current_char)) {
  447. // octal without '0o' prefix. Forbidden in 'strict mode'
  448. do {
  449. consume();
  450. } while (is_ascii_digit(m_current_char));
  451. }
  452. } else {
  453. // 1...9 or period
  454. while (is_ascii_digit(m_current_char))
  455. consume();
  456. if (m_current_char == 'n') {
  457. consume();
  458. token_type = TokenType::BigIntLiteral;
  459. } else {
  460. if (m_current_char == '.') {
  461. consume();
  462. while (is_ascii_digit(m_current_char))
  463. consume();
  464. }
  465. if (m_current_char == 'e' || m_current_char == 'E')
  466. is_invalid_numeric_literal = !consume_exponent();
  467. }
  468. }
  469. if (is_invalid_numeric_literal) {
  470. token_type = TokenType::Invalid;
  471. token_message = "Invalid numeric literal";
  472. }
  473. } else if (m_current_char == '"' || m_current_char == '\'') {
  474. char stop_char = m_current_char;
  475. consume();
  476. // Note: LS/PS line terminators are allowed in string literals.
  477. while (m_current_char != stop_char && m_current_char != '\r' && m_current_char != '\n' && !is_eof()) {
  478. if (m_current_char == '\\') {
  479. consume();
  480. }
  481. consume();
  482. }
  483. if (m_current_char != stop_char) {
  484. token_type = TokenType::UnterminatedStringLiteral;
  485. } else {
  486. consume();
  487. token_type = TokenType::StringLiteral;
  488. }
  489. } else if (m_current_char == '/' && !slash_means_division()) {
  490. consume();
  491. token_type = TokenType::RegexLiteral;
  492. while (!is_eof()) {
  493. if (m_current_char == '[') {
  494. m_regex_is_in_character_class = true;
  495. } else if (m_current_char == ']') {
  496. m_regex_is_in_character_class = false;
  497. } else if (!m_regex_is_in_character_class && m_current_char == '/') {
  498. break;
  499. }
  500. if (match('\\', '/') || match('\\', '[') || match('\\', '\\') || (m_regex_is_in_character_class && match('\\', ']')))
  501. consume();
  502. consume();
  503. }
  504. if (is_eof()) {
  505. token_type = TokenType::UnterminatedRegexLiteral;
  506. } else {
  507. consume();
  508. }
  509. } else if (m_eof) {
  510. if (unterminated_comment) {
  511. token_type = TokenType::Invalid;
  512. token_message = "Unterminated multi-line comment";
  513. } else {
  514. token_type = TokenType::Eof;
  515. }
  516. } else {
  517. // There is only one four-char operator: >>>=
  518. bool found_four_char_token = false;
  519. if (match('>', '>', '>', '=')) {
  520. found_four_char_token = true;
  521. consume();
  522. consume();
  523. consume();
  524. consume();
  525. token_type = TokenType::UnsignedShiftRightEquals;
  526. }
  527. bool found_three_char_token = false;
  528. if (!found_four_char_token && m_position + 1 < m_source.length()) {
  529. auto three_chars_view = m_source.substring_view(m_position - 1, 3);
  530. auto it = s_three_char_tokens.find(three_chars_view.hash(), [&](auto& entry) { return entry.key == three_chars_view; });
  531. if (it != s_three_char_tokens.end()) {
  532. found_three_char_token = true;
  533. consume();
  534. consume();
  535. consume();
  536. token_type = it->value;
  537. }
  538. }
  539. bool found_two_char_token = false;
  540. if (!found_four_char_token && !found_three_char_token && m_position < m_source.length()) {
  541. auto two_chars_view = m_source.substring_view(m_position - 1, 2);
  542. auto it = s_two_char_tokens.find(two_chars_view.hash(), [&](auto& entry) { return entry.key == two_chars_view; });
  543. if (it != s_two_char_tokens.end()) {
  544. // OptionalChainingPunctuator :: ?. [lookahead ∉ DecimalDigit]
  545. if (!(it->value == TokenType::QuestionMarkPeriod && m_position + 1 < m_source.length() && is_ascii_digit(m_source[m_position + 1]))) {
  546. found_two_char_token = true;
  547. consume();
  548. consume();
  549. token_type = it->value;
  550. }
  551. }
  552. }
  553. bool found_one_char_token = false;
  554. if (!found_four_char_token && !found_three_char_token && !found_two_char_token) {
  555. auto it = s_single_char_tokens.find(m_current_char);
  556. if (it != s_single_char_tokens.end()) {
  557. found_one_char_token = true;
  558. consume();
  559. token_type = it->value;
  560. }
  561. }
  562. if (!found_four_char_token && !found_three_char_token && !found_two_char_token && !found_one_char_token) {
  563. consume();
  564. token_type = TokenType::Invalid;
  565. }
  566. }
  567. if (!m_template_states.is_empty() && m_template_states.last().in_expr) {
  568. if (token_type == TokenType::CurlyOpen) {
  569. m_template_states.last().open_bracket_count++;
  570. } else if (token_type == TokenType::CurlyClose) {
  571. m_template_states.last().open_bracket_count--;
  572. }
  573. }
  574. m_current_token = Token(
  575. token_type,
  576. token_message,
  577. m_source.substring_view(trivia_start - 1, value_start - trivia_start),
  578. m_source.substring_view(value_start - 1, m_position - value_start),
  579. m_filename,
  580. value_start_line_number,
  581. value_start_column_number);
  582. if constexpr (LEXER_DEBUG) {
  583. dbgln("------------------------------");
  584. dbgln("Token: {}", m_current_token.name());
  585. dbgln("Trivia: _{}_", m_current_token.trivia());
  586. dbgln("Value: _{}_", m_current_token.value());
  587. dbgln("Line: {}, Column: {}", m_current_token.line_number(), m_current_token.line_column());
  588. dbgln("------------------------------");
  589. }
  590. return m_current_token;
  591. }
  592. }