Tokenizer.cpp 43 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287
  1. /*
  2. * Copyright (c) 2020-2022, the SerenityOS developers.
  3. * Copyright (c) 2021-2022, Sam Atkins <atkinssj@serenityos.org>
  4. *
  5. * SPDX-License-Identifier: BSD-2-Clause
  6. */
  7. #include <AK/CharacterTypes.h>
  8. #include <AK/Debug.h>
  9. #include <AK/FloatingPointStringConversions.h>
  10. #include <AK/SourceLocation.h>
  11. #include <AK/Vector.h>
  12. #include <LibTextCodec/Decoder.h>
  13. #include <LibWeb/CSS/Parser/Tokenizer.h>
  14. #include <math.h>
  15. namespace Web::CSS::Parser {
  16. // U+FFFD REPLACEMENT CHARACTER (�)
  17. #define REPLACEMENT_CHARACTER 0xFFFD
  18. static constexpr u32 TOKENIZER_EOF = 0xFFFFFFFF;
  19. static inline void log_parse_error(SourceLocation const& location = SourceLocation::current())
  20. {
  21. dbgln_if(CSS_TOKENIZER_DEBUG, "Parse error (css tokenization) {} ", location);
  22. }
  23. static inline bool is_eof(u32 code_point)
  24. {
  25. return code_point == TOKENIZER_EOF;
  26. }
  27. static inline bool is_quotation_mark(u32 code_point)
  28. {
  29. return code_point == 0x22;
  30. }
  31. static inline bool is_greater_than_maximum_allowed_code_point(u32 code_point)
  32. {
  33. return code_point > 0x10FFFF;
  34. }
  35. static inline bool is_low_line(u32 code_point)
  36. {
  37. return code_point == 0x5F;
  38. }
  39. // https://www.w3.org/TR/css-syntax-3/#ident-start-code-point
  40. static inline bool is_ident_start_code_point(u32 code_point)
  41. {
  42. // FIXME: We use !is_ascii() for "non-ASCII code point" in the spec, but it's not quite right -
  43. // it treats EOF as a valid! The spec also lacks a definition of code point. For now, the
  44. // !is_eof() check is a hack, but it should work.
  45. return !is_eof(code_point) && (is_ascii_alpha(code_point) || !is_ascii(code_point) || is_low_line(code_point));
  46. }
  47. static inline bool is_hyphen_minus(u32 code_point)
  48. {
  49. return code_point == 0x2D;
  50. }
  51. // https://www.w3.org/TR/css-syntax-3/#ident-code-point
  52. static inline bool is_ident_code_point(u32 code_point)
  53. {
  54. return is_ident_start_code_point(code_point) || is_ascii_digit(code_point) || is_hyphen_minus(code_point);
  55. }
  56. static inline bool is_non_printable(u32 code_point)
  57. {
  58. return code_point <= 0x8 || code_point == 0xB || (code_point >= 0xE && code_point <= 0x1F) || code_point == 0x7F;
  59. }
  60. static inline bool is_number_sign(u32 code_point)
  61. {
  62. return code_point == 0x23;
  63. }
  64. static inline bool is_reverse_solidus(u32 code_point)
  65. {
  66. return code_point == 0x5C;
  67. }
  68. static inline bool is_apostrophe(u32 code_point)
  69. {
  70. return code_point == 0x27;
  71. }
  72. static inline bool is_left_paren(u32 code_point)
  73. {
  74. return code_point == 0x28;
  75. }
  76. static inline bool is_right_paren(u32 code_point)
  77. {
  78. return code_point == 0x29;
  79. }
  80. static inline bool is_plus_sign(u32 code_point)
  81. {
  82. return code_point == 0x2B;
  83. }
  84. static inline bool is_comma(u32 code_point)
  85. {
  86. return code_point == 0x2C;
  87. }
  88. static inline bool is_full_stop(u32 code_point)
  89. {
  90. return code_point == 0x2E;
  91. }
  92. static inline bool is_newline(u32 code_point)
  93. {
  94. return code_point == 0xA;
  95. }
  96. static inline bool is_asterisk(u32 code_point)
  97. {
  98. return code_point == 0x2A;
  99. }
  100. static inline bool is_solidus(u32 code_point)
  101. {
  102. return code_point == 0x2F;
  103. }
  104. static inline bool is_colon(u32 code_point)
  105. {
  106. return code_point == 0x3A;
  107. }
  108. static inline bool is_semicolon(u32 code_point)
  109. {
  110. return code_point == 0x3B;
  111. }
  112. static inline bool is_less_than_sign(u32 code_point)
  113. {
  114. return code_point == 0x3C;
  115. }
  116. static inline bool is_greater_than_sign(u32 code_point)
  117. {
  118. return code_point == 0x3E;
  119. }
  120. static inline bool is_at(u32 code_point)
  121. {
  122. return code_point == 0x40;
  123. }
  124. static inline bool is_open_square_bracket(u32 code_point)
  125. {
  126. return code_point == 0x5B;
  127. }
  128. static inline bool is_closed_square_bracket(u32 code_point)
  129. {
  130. return code_point == 0x5D;
  131. }
  132. static inline bool is_open_curly_bracket(u32 code_point)
  133. {
  134. return code_point == 0x7B;
  135. }
  136. static inline bool is_closed_curly_bracket(u32 code_point)
  137. {
  138. return code_point == 0x7D;
  139. }
  140. static inline bool is_whitespace(u32 code_point)
  141. {
  142. return code_point == 0x9 || code_point == 0xA || code_point == 0x20;
  143. }
  144. static inline bool is_percent(u32 code_point)
  145. {
  146. return code_point == 0x25;
  147. }
  148. static inline bool is_exclamation_mark(u32 code_point)
  149. {
  150. return code_point == 0x21;
  151. }
  152. static inline bool is_e(u32 code_point)
  153. {
  154. return code_point == 0x65;
  155. }
  156. static inline bool is_E(u32 code_point)
  157. {
  158. return code_point == 0x45;
  159. }
  160. Tokenizer::Tokenizer(StringView input, DeprecatedString const& encoding)
  161. {
  162. // https://www.w3.org/TR/css-syntax-3/#css-filter-code-points
  163. auto filter_code_points = [](StringView input, auto const& encoding) -> DeprecatedString {
  164. auto* decoder = TextCodec::decoder_for(encoding);
  165. VERIFY(decoder);
  166. StringBuilder builder { input.length() };
  167. bool last_was_carriage_return = false;
  168. // To filter code points from a stream of (unfiltered) code points input:
  169. decoder->process(input, [&builder, &last_was_carriage_return](u32 code_point) {
  170. // Replace any U+000D CARRIAGE RETURN (CR) code points,
  171. // U+000C FORM FEED (FF) code points,
  172. // or pairs of U+000D CARRIAGE RETURN (CR) followed by U+000A LINE FEED (LF)
  173. // in input by a single U+000A LINE FEED (LF) code point.
  174. if (code_point == '\r') {
  175. if (last_was_carriage_return) {
  176. builder.append('\n');
  177. } else {
  178. last_was_carriage_return = true;
  179. }
  180. } else {
  181. if (last_was_carriage_return)
  182. builder.append('\n');
  183. if (code_point == '\n') {
  184. if (!last_was_carriage_return)
  185. builder.append('\n');
  186. } else if (code_point == '\f') {
  187. builder.append('\n');
  188. // Replace any U+0000 NULL or surrogate code points in input with U+FFFD REPLACEMENT CHARACTER (�).
  189. } else if (code_point == 0x00 || (code_point >= 0xD800 && code_point <= 0xDFFF)) {
  190. builder.append_code_point(REPLACEMENT_CHARACTER);
  191. } else {
  192. builder.append_code_point(code_point);
  193. }
  194. last_was_carriage_return = false;
  195. }
  196. });
  197. return builder.to_deprecated_string();
  198. };
  199. m_decoded_input = filter_code_points(input, encoding);
  200. m_utf8_view = Utf8View(m_decoded_input);
  201. m_utf8_iterator = m_utf8_view.begin();
  202. }
  203. Vector<Token> Tokenizer::parse()
  204. {
  205. Vector<Token> tokens;
  206. for (;;) {
  207. auto token_start = m_position;
  208. auto token = consume_a_token();
  209. token.m_start_position = token_start;
  210. token.m_end_position = m_position;
  211. tokens.append(token);
  212. if (token.is(Token::Type::EndOfFile)) {
  213. return tokens;
  214. }
  215. }
  216. }
  217. u32 Tokenizer::next_code_point()
  218. {
  219. if (m_utf8_iterator == m_utf8_view.end())
  220. return TOKENIZER_EOF;
  221. m_prev_utf8_iterator = m_utf8_iterator;
  222. ++m_utf8_iterator;
  223. auto code_point = *m_prev_utf8_iterator;
  224. m_prev_position = m_position;
  225. if (is_newline(code_point)) {
  226. m_position.line++;
  227. m_position.column = 0;
  228. } else {
  229. m_position.column++;
  230. }
  231. dbgln_if(CSS_TOKENIZER_DEBUG, "(Tokenizer) Next code_point: {:d}", code_point);
  232. return code_point;
  233. }
  234. u32 Tokenizer::peek_code_point(size_t offset) const
  235. {
  236. auto it = m_utf8_iterator;
  237. for (size_t i = 0; i < offset && it != m_utf8_view.end(); ++i)
  238. ++it;
  239. if (it == m_utf8_view.end())
  240. return TOKENIZER_EOF;
  241. dbgln_if(CSS_TOKENIZER_DEBUG, "(Tokenizer) Peek code_point: {:d}", *m_prev_utf8_iterator);
  242. return *it;
  243. }
  244. U32Twin Tokenizer::peek_twin() const
  245. {
  246. U32Twin values { TOKENIZER_EOF, TOKENIZER_EOF };
  247. auto it = m_utf8_iterator;
  248. for (size_t i = 0; i < 2 && it != m_utf8_view.end(); ++i) {
  249. values.set(i, *it);
  250. ++it;
  251. }
  252. dbgln_if(CSS_TOKENIZER_DEBUG, "(Tokenizer) Peek twin: {:d},{:d}", values.first, values.second);
  253. return values;
  254. }
  255. U32Triplet Tokenizer::peek_triplet() const
  256. {
  257. U32Triplet values { TOKENIZER_EOF, TOKENIZER_EOF, TOKENIZER_EOF };
  258. auto it = m_utf8_iterator;
  259. for (size_t i = 0; i < 3 && it != m_utf8_view.end(); ++i) {
  260. values.set(i, *it);
  261. ++it;
  262. }
  263. dbgln_if(CSS_TOKENIZER_DEBUG, "(Tokenizer) Peek triplet: {:d},{:d},{:d}", values.first, values.second, values.third);
  264. return values;
  265. }
  266. U32Twin Tokenizer::start_of_input_stream_twin()
  267. {
  268. U32Twin twin;
  269. // FIXME: Reconsuming just to read the current code point again is weird.
  270. reconsume_current_input_code_point();
  271. twin.first = next_code_point();
  272. twin.second = peek_code_point();
  273. return twin;
  274. }
  275. U32Triplet Tokenizer::start_of_input_stream_triplet()
  276. {
  277. U32Triplet triplet;
  278. // FIXME: Reconsuming just to read the current code point again is weird.
  279. reconsume_current_input_code_point();
  280. triplet.first = next_code_point();
  281. auto next_two = peek_twin();
  282. triplet.second = next_two.first;
  283. triplet.third = next_two.second;
  284. return triplet;
  285. }
  286. Token Tokenizer::create_new_token(Token::Type type)
  287. {
  288. Token token = {};
  289. token.m_type = type;
  290. return token;
  291. }
  292. Token Tokenizer::create_eof_token()
  293. {
  294. return create_new_token(Token::Type::EndOfFile);
  295. }
  296. Token Tokenizer::create_value_token(Token::Type type, DeprecatedString value)
  297. {
  298. Token token;
  299. token.m_type = type;
  300. token.m_value = move(value);
  301. return token;
  302. }
  303. Token Tokenizer::create_value_token(Token::Type type, u32 value)
  304. {
  305. Token token = {};
  306. token.m_type = type;
  307. // FIXME: Avoid temporary StringBuilder here
  308. StringBuilder builder;
  309. builder.append_code_point(value);
  310. token.m_value = builder.to_deprecated_string();
  311. return token;
  312. }
  313. // https://www.w3.org/TR/css-syntax-3/#consume-escaped-code-point
  314. u32 Tokenizer::consume_escaped_code_point()
  315. {
  316. // This section describes how to consume an escaped code point.
  317. // It assumes that the U+005C REVERSE SOLIDUS (\) has already been consumed and that the next
  318. // input code point has already been verified to be part of a valid escape.
  319. // It will return a code point.
  320. // Consume the next input code point.
  321. auto input = next_code_point();
  322. // hex digit
  323. if (is_ascii_hex_digit(input)) {
  324. // Consume as many hex digits as possible, but no more than 5.
  325. // Note that this means 1-6 hex digits have been consumed in total.
  326. StringBuilder builder;
  327. builder.append_code_point(input);
  328. size_t counter = 0;
  329. while (is_ascii_hex_digit(peek_code_point()) && counter++ < 5) {
  330. builder.append_code_point(next_code_point());
  331. }
  332. // If the next input code point is whitespace, consume it as well.
  333. if (is_whitespace(peek_code_point())) {
  334. (void)next_code_point();
  335. }
  336. // Interpret the hex digits as a hexadecimal number.
  337. auto unhexed = strtoul(builder.to_deprecated_string().characters(), nullptr, 16);
  338. // If this number is zero, or is for a surrogate, or is greater than the maximum allowed
  339. // code point, return U+FFFD REPLACEMENT CHARACTER (�).
  340. if (unhexed == 0 || is_unicode_surrogate(unhexed) || is_greater_than_maximum_allowed_code_point(unhexed)) {
  341. return REPLACEMENT_CHARACTER;
  342. }
  343. // Otherwise, return the code point with that value.
  344. return unhexed;
  345. }
  346. // EOF
  347. if (is_eof(input)) {
  348. // This is a parse error. Return U+FFFD REPLACEMENT CHARACTER (�).
  349. log_parse_error();
  350. return REPLACEMENT_CHARACTER;
  351. }
  352. // anything else
  353. // Return the current input code point.
  354. return input;
  355. }
  356. // https://www.w3.org/TR/css-syntax-3/#consume-ident-like-token
  357. Token Tokenizer::consume_an_ident_like_token()
  358. {
  359. // This section describes how to consume an ident-like token from a stream of code points.
  360. // It returns an <ident-token>, <function-token>, <url-token>, or <bad-url-token>.
  361. // Consume an ident sequence, and let string be the result.
  362. auto string = consume_an_ident_sequence();
  363. // If string’s value is an ASCII case-insensitive match for "url", and the next input code
  364. // point is U+0028 LEFT PARENTHESIS ((), consume it.
  365. if (string.equals_ignoring_case("url"sv) && is_left_paren(peek_code_point())) {
  366. (void)next_code_point();
  367. // While the next two input code points are whitespace, consume the next input code point.
  368. for (;;) {
  369. auto maybe_whitespace = peek_twin();
  370. if (!(is_whitespace(maybe_whitespace.first) && is_whitespace(maybe_whitespace.second))) {
  371. break;
  372. }
  373. (void)next_code_point();
  374. }
  375. // If the next one or two input code points are U+0022 QUOTATION MARK ("), U+0027 APOSTROPHE ('),
  376. // or whitespace followed by U+0022 QUOTATION MARK (") or U+0027 APOSTROPHE ('), then create a
  377. // <function-token> with its value set to string and return it.
  378. auto next_two = peek_twin();
  379. if (is_quotation_mark(next_two.first) || is_apostrophe(next_two.first) || (is_whitespace(next_two.first) && (is_quotation_mark(next_two.second) || is_apostrophe(next_two.second)))) {
  380. return create_value_token(Token::Type::Function, string);
  381. }
  382. // Otherwise, consume a url token, and return it.
  383. return consume_a_url_token();
  384. }
  385. // Otherwise, if the next input code point is U+0028 LEFT PARENTHESIS ((), consume it.
  386. if (is_left_paren(peek_code_point())) {
  387. (void)next_code_point();
  388. // Create a <function-token> with its value set to string and return it.
  389. return create_value_token(Token::Type::Function, string);
  390. }
  391. // Otherwise, create an <ident-token> with its value set to string and return it.
  392. return create_value_token(Token::Type::Ident, string);
  393. }
  394. // https://www.w3.org/TR/css-syntax-3/#consume-number
  395. Number Tokenizer::consume_a_number()
  396. {
  397. // This section describes how to consume a number from a stream of code points.
  398. // It returns a numeric value, and a type which is either "integer" or "number".
  399. //
  400. // Note: This algorithm does not do the verification of the first few code points
  401. // that are necessary to ensure a number can be obtained from the stream. Ensure
  402. // that the stream starts with a number before calling this algorithm.
  403. // Execute the following steps in order:
  404. // 1. Initially set type to "integer". Let repr be the empty string.
  405. StringBuilder repr;
  406. Number::Type type = Number::Type::Integer;
  407. // 2. If the next input code point is U+002B PLUS SIGN (+) or U+002D HYPHEN-MINUS (-),
  408. // consume it and append it to repr.
  409. bool has_explicit_sign = false;
  410. auto next_input = peek_code_point();
  411. if (is_plus_sign(next_input) || is_hyphen_minus(next_input)) {
  412. has_explicit_sign = true;
  413. repr.append_code_point(next_code_point());
  414. }
  415. // 3. While the next input code point is a digit, consume it and append it to repr.
  416. for (;;) {
  417. auto digits = peek_code_point();
  418. if (!is_ascii_digit(digits))
  419. break;
  420. repr.append_code_point(next_code_point());
  421. }
  422. // 4. If the next 2 input code points are U+002E FULL STOP (.) followed by a digit, then:
  423. auto maybe_number = peek_twin();
  424. if (is_full_stop(maybe_number.first) && is_ascii_digit(maybe_number.second)) {
  425. // 1. Consume them.
  426. // 2. Append them to repr.
  427. repr.append_code_point(next_code_point());
  428. repr.append_code_point(next_code_point());
  429. // 3. Set type to "number".
  430. type = Number::Type::Number;
  431. // 4. While the next input code point is a digit, consume it and append it to repr.
  432. for (;;) {
  433. auto digit = peek_code_point();
  434. if (!is_ascii_digit(digit))
  435. break;
  436. repr.append_code_point(next_code_point());
  437. }
  438. }
  439. // 5. If the next 2 or 3 input code points are U+0045 LATIN CAPITAL LETTER E (E) or
  440. // U+0065 LATIN SMALL LETTER E (e), optionally followed by U+002D HYPHEN-MINUS (-)
  441. // or U+002B PLUS SIGN (+), followed by a digit, then:
  442. auto maybe_exp = peek_triplet();
  443. if (is_E(maybe_exp.first) || is_e(maybe_exp.first)) {
  444. // 1. Consume them.
  445. // 2. Append them to repr.
  446. // FIXME: These conditions should be part of step 5 above.
  447. if (is_plus_sign(maybe_exp.second) || is_hyphen_minus(maybe_exp.second)) {
  448. if (is_ascii_digit(maybe_exp.third)) {
  449. repr.append_code_point(next_code_point());
  450. repr.append_code_point(next_code_point());
  451. repr.append_code_point(next_code_point());
  452. }
  453. } else if (is_ascii_digit(maybe_exp.second)) {
  454. repr.append_code_point(next_code_point());
  455. repr.append_code_point(next_code_point());
  456. }
  457. // 3. Set type to "number".
  458. type = Number::Type::Number;
  459. // 4. While the next input code point is a digit, consume it and append it to repr.
  460. for (;;) {
  461. auto digits = peek_code_point();
  462. if (!is_ascii_digit(digits))
  463. break;
  464. repr.append_code_point(next_code_point());
  465. }
  466. }
  467. // 6. Convert repr to a number, and set the value to the returned value.
  468. auto value = convert_a_string_to_a_number(repr.string_view());
  469. // 7. Return value and type.
  470. if (type == Number::Type::Integer && has_explicit_sign)
  471. return Number { Number::Type::IntegerWithExplicitSign, value };
  472. return Number { type, value };
  473. }
  474. // https://www.w3.org/TR/css-syntax-3/#convert-string-to-number
  475. float Tokenizer::convert_a_string_to_a_number(StringView string)
  476. {
  477. // FIXME: We already found the whole part, fraction part and exponent during
  478. // validation, we could probably skip
  479. return string.to_float(AK::TrimWhitespace::No).release_value();
  480. }
  481. // https://www.w3.org/TR/css-syntax-3/#consume-name
  482. DeprecatedString Tokenizer::consume_an_ident_sequence()
  483. {
  484. // This section describes how to consume an ident sequence from a stream of code points.
  485. // It returns a string containing the largest name that can be formed from adjacent
  486. // code points in the stream, starting from the first.
  487. //
  488. // Note: This algorithm does not do the verification of the first few code points that
  489. // are necessary to ensure the returned code points would constitute an <ident-token>.
  490. // If that is the intended use, ensure that the stream starts with an ident sequence before
  491. // calling this algorithm.
  492. // Let result initially be an empty string.
  493. StringBuilder result;
  494. // Repeatedly consume the next input code point from the stream:
  495. for (;;) {
  496. auto input = next_code_point();
  497. if (is_eof(input))
  498. break;
  499. // name code point
  500. if (is_ident_code_point(input)) {
  501. // Append the code point to result.
  502. result.append_code_point(input);
  503. continue;
  504. }
  505. // the stream starts with a valid escape
  506. if (is_valid_escape_sequence(start_of_input_stream_twin())) {
  507. // Consume an escaped code point. Append the returned code point to result.
  508. result.append_code_point(consume_escaped_code_point());
  509. continue;
  510. }
  511. // anything else
  512. // Reconsume the current input code point. Return result.
  513. reconsume_current_input_code_point();
  514. break;
  515. }
  516. return result.to_deprecated_string();
  517. }
  518. // https://www.w3.org/TR/css-syntax-3/#consume-url-token
  519. Token Tokenizer::consume_a_url_token()
  520. {
  521. // This section describes how to consume a url token from a stream of code points.
  522. // It returns either a <url-token> or a <bad-url-token>.
  523. //
  524. // Note: This algorithm assumes that the initial "url(" has already been consumed.
  525. // This algorithm also assumes that it’s being called to consume an "unquoted" value,
  526. // like url(foo). A quoted value, like url("foo"), is parsed as a <function-token>.
  527. // Consume an ident-like token automatically handles this distinction; this algorithm
  528. // shouldn’t be called directly otherwise.
  529. // 1. Initially create a <url-token> with its value set to the empty string.
  530. auto token = create_new_token(Token::Type::Url);
  531. StringBuilder builder;
  532. // 2. Consume as much whitespace as possible.
  533. consume_as_much_whitespace_as_possible();
  534. auto make_token = [&]() {
  535. token.m_value = builder.to_deprecated_string();
  536. return token;
  537. };
  538. // 3. Repeatedly consume the next input code point from the stream:
  539. for (;;) {
  540. auto input = next_code_point();
  541. // U+0029 RIGHT PARENTHESIS ())
  542. if (is_right_paren(input)) {
  543. // Return the <url-token>.
  544. return make_token();
  545. }
  546. // EOF
  547. if (is_eof(input)) {
  548. // This is a parse error. Return the <url-token>.
  549. log_parse_error();
  550. return make_token();
  551. }
  552. // whitespace
  553. if (is_whitespace(input)) {
  554. // Consume as much whitespace as possible.
  555. consume_as_much_whitespace_as_possible();
  556. // If the next input code point is U+0029 RIGHT PARENTHESIS ()) or EOF, consume it
  557. // and return the <url-token> (if EOF was encountered, this is a parse error);
  558. input = peek_code_point();
  559. if (is_right_paren(input)) {
  560. (void)next_code_point();
  561. return make_token();
  562. }
  563. if (is_eof(input)) {
  564. (void)next_code_point();
  565. log_parse_error();
  566. return make_token();
  567. }
  568. // otherwise, consume the remnants of a bad url, create a <bad-url-token>, and return it.
  569. consume_the_remnants_of_a_bad_url();
  570. return create_new_token(Token::Type::BadUrl);
  571. }
  572. // U+0022 QUOTATION MARK (")
  573. // U+0027 APOSTROPHE (')
  574. // U+0028 LEFT PARENTHESIS (()
  575. // non-printable code point
  576. if (is_quotation_mark(input) || is_apostrophe(input) || is_left_paren(input) || is_non_printable(input)) {
  577. // This is a parse error. Consume the remnants of a bad url, create a <bad-url-token>, and return it.
  578. log_parse_error();
  579. consume_the_remnants_of_a_bad_url();
  580. return create_new_token(Token::Type::BadUrl);
  581. }
  582. // U+005C REVERSE SOLIDUS (\)
  583. if (is_reverse_solidus(input)) {
  584. // If the stream starts with a valid escape,
  585. if (is_valid_escape_sequence(start_of_input_stream_twin())) {
  586. // consume an escaped code point and append the returned code point to the <url-token>’s value.
  587. builder.append_code_point(consume_escaped_code_point());
  588. continue;
  589. } else {
  590. // Otherwise, this is a parse error.
  591. log_parse_error();
  592. // Consume the remnants of a bad url, create a <bad-url-token>, and return it.
  593. consume_the_remnants_of_a_bad_url();
  594. return create_new_token(Token::Type::BadUrl);
  595. }
  596. }
  597. // anything else
  598. // Append the current input code point to the <url-token>’s value.
  599. builder.append_code_point(input);
  600. }
  601. }
  602. // https://www.w3.org/TR/css-syntax-3/#consume-remnants-of-bad-url
  603. void Tokenizer::consume_the_remnants_of_a_bad_url()
  604. {
  605. // This section describes how to consume the remnants of a bad url from a stream of code points,
  606. // "cleaning up" after the tokenizer realizes that it’s in the middle of a <bad-url-token> rather
  607. // than a <url-token>. It returns nothing; its sole use is to consume enough of the input stream
  608. // to reach a recovery point where normal tokenizing can resume.
  609. // Repeatedly consume the next input code point from the stream:
  610. for (;;) {
  611. auto input = next_code_point();
  612. // U+0029 RIGHT PARENTHESIS ())
  613. // EOF
  614. if (is_eof(input) || is_right_paren(input)) {
  615. // Return.
  616. return;
  617. }
  618. // the input stream starts with a valid escape
  619. if (is_valid_escape_sequence(start_of_input_stream_twin())) {
  620. // Consume an escaped code point.
  621. // This allows an escaped right parenthesis ("\)") to be encountered without ending
  622. // the <bad-url-token>. This is otherwise identical to the "anything else" clause.
  623. (void)consume_escaped_code_point();
  624. }
  625. // anything else
  626. // Do nothing.
  627. }
  628. }
  629. void Tokenizer::consume_as_much_whitespace_as_possible()
  630. {
  631. while (is_whitespace(peek_code_point())) {
  632. (void)next_code_point();
  633. }
  634. }
  635. void Tokenizer::reconsume_current_input_code_point()
  636. {
  637. m_utf8_iterator = m_prev_utf8_iterator;
  638. m_position = m_prev_position;
  639. }
  640. // https://www.w3.org/TR/css-syntax-3/#consume-numeric-token
  641. Token Tokenizer::consume_a_numeric_token()
  642. {
  643. // This section describes how to consume a numeric token from a stream of code points.
  644. // It returns either a <number-token>, <percentage-token>, or <dimension-token>.
  645. // Consume a number and let number be the result.
  646. auto number = consume_a_number();
  647. // If the next 3 input code points would start an ident sequence, then:
  648. if (would_start_an_ident_sequence(peek_triplet())) {
  649. // 1. Create a <dimension-token> with the same value and type flag as number,
  650. // and a unit set initially to the empty string.
  651. auto token = create_new_token(Token::Type::Dimension);
  652. token.m_number_value = number;
  653. // 2. Consume an ident sequence. Set the <dimension-token>’s unit to the returned value.
  654. auto unit = consume_an_ident_sequence();
  655. VERIFY(!unit.is_empty());
  656. // NOTE: We intentionally store this in the `value`, to save space.
  657. token.m_value = move(unit);
  658. // 3. Return the <dimension-token>.
  659. return token;
  660. }
  661. // Otherwise, if the next input code point is U+0025 PERCENTAGE SIGN (%), consume it.
  662. if (is_percent(peek_code_point())) {
  663. (void)next_code_point();
  664. // Create a <percentage-token> with the same value as number, and return it.
  665. auto token = create_new_token(Token::Type::Percentage);
  666. token.m_number_value = number;
  667. return token;
  668. }
  669. // Otherwise, create a <number-token> with the same value and type flag as number, and return it.
  670. auto token = create_new_token(Token::Type::Number);
  671. token.m_number_value = number;
  672. return token;
  673. }
  674. // https://www.w3.org/TR/css-syntax-3/#starts-with-a-number
  675. bool Tokenizer::would_start_a_number(U32Triplet values)
  676. {
  677. // This section describes how to check if three code points would start a number.
  678. // The algorithm described here can be called explicitly with three code points,
  679. // or can be called with the input stream itself. In the latter case, the three
  680. // code points in question are the current input code point and the next two input
  681. // code points, in that order.
  682. //
  683. // Note: This algorithm will not consume any additional code points.
  684. // Look at the first code point:
  685. // U+002B PLUS SIGN (+)
  686. // U+002D HYPHEN-MINUS (-)
  687. if (is_plus_sign(values.first) || is_hyphen_minus(values.first)) {
  688. // If the second code point is a digit, return true.
  689. if (is_ascii_digit(values.second))
  690. return true;
  691. // Otherwise, if the second code point is a U+002E FULL STOP (.) and the third
  692. // code point is a digit, return true.
  693. if (is_full_stop(values.second) && is_ascii_digit(values.third))
  694. return true;
  695. // Otherwise, return false.
  696. return false;
  697. }
  698. // U+002E FULL STOP (.)
  699. if (is_full_stop(values.first))
  700. // If the second code point is a digit, return true. Otherwise, return false.
  701. return is_ascii_digit(values.second);
  702. // digit
  703. if (is_ascii_digit(values.first))
  704. // Return true.
  705. return true;
  706. // anything else
  707. // Return false.
  708. return false;
  709. }
  710. // https://www.w3.org/TR/css-syntax-3/#starts-with-a-valid-escape
  711. bool Tokenizer::is_valid_escape_sequence(U32Twin values)
  712. {
  713. // This section describes how to check if two code points are a valid escape.
  714. // The algorithm described here can be called explicitly with two code points,
  715. // or can be called with the input stream itself. In the latter case, the two
  716. // code points in question are the current input code point and the next input
  717. // code point, in that order.
  718. //
  719. // Note: This algorithm will not consume any additional code point.
  720. // If the first code point is not U+005C REVERSE SOLIDUS (\), return false.
  721. if (!is_reverse_solidus(values.first))
  722. return false;
  723. // Otherwise, if the second code point is a newline, return false.
  724. if (is_newline(values.second))
  725. return false;
  726. // Otherwise, return true.
  727. return true;
  728. }
  729. // https://www.w3.org/TR/css-syntax-3/#would-start-an-identifier
  730. bool Tokenizer::would_start_an_ident_sequence(U32Triplet values)
  731. {
  732. // This section describes how to check if three code points would start an ident sequence.
  733. // The algorithm described here can be called explicitly with three code points, or
  734. // can be called with the input stream itself. In the latter case, the three code
  735. // points in question are the current input code point and the next two input code
  736. // points, in that order.
  737. //
  738. // Note: This algorithm will not consume any additional code points.
  739. // Look at the first code point:
  740. // U+002D HYPHEN-MINUS
  741. if (is_hyphen_minus(values.first)) {
  742. // If the second code point is a name-start code point or a U+002D HYPHEN-MINUS,
  743. // or the second and third code points are a valid escape, return true.
  744. if (is_ident_start_code_point(values.second) || is_hyphen_minus(values.second) || is_valid_escape_sequence(values.to_twin_23()))
  745. return true;
  746. // Otherwise, return false.
  747. return false;
  748. }
  749. // name-start code point
  750. if (is_ident_start_code_point(values.first)) {
  751. // Return true.
  752. return true;
  753. }
  754. // U+005C REVERSE SOLIDUS (\)
  755. if (is_reverse_solidus(values.first)) {
  756. // If the first and second code points are a valid escape, return true.
  757. if (is_valid_escape_sequence(values.to_twin_12()))
  758. return true;
  759. // Otherwise, return false.
  760. return false;
  761. }
  762. // anything else
  763. // Return false.
  764. return false;
  765. }
  766. // https://www.w3.org/TR/css-syntax-3/#consume-string-token
  767. Token Tokenizer::consume_string_token(u32 ending_code_point)
  768. {
  769. // This section describes how to consume a string token from a stream of code points.
  770. // It returns either a <string-token> or <bad-string-token>.
  771. //
  772. // This algorithm may be called with an ending code point, which denotes the code point
  773. // that ends the string. If an ending code point is not specified, the current input
  774. // code point is used.
  775. // Initially create a <string-token> with its value set to the empty string.
  776. auto token = create_new_token(Token::Type::String);
  777. StringBuilder builder;
  778. auto make_token = [&]() {
  779. token.m_value = builder.to_deprecated_string();
  780. return token;
  781. };
  782. // Repeatedly consume the next input code point from the stream:
  783. for (;;) {
  784. auto input = next_code_point();
  785. // ending code point
  786. if (input == ending_code_point)
  787. return make_token();
  788. // EOF
  789. if (is_eof(input)) {
  790. // This is a parse error. Return the <string-token>.
  791. log_parse_error();
  792. return make_token();
  793. }
  794. // newline
  795. if (is_newline(input)) {
  796. // This is a parse error. Reconsume the current input code point, create a
  797. // <bad-string-token>, and return it.
  798. reconsume_current_input_code_point();
  799. return create_new_token(Token::Type::BadString);
  800. }
  801. // U+005C REVERSE SOLIDUS (\)
  802. if (is_reverse_solidus(input)) {
  803. // If the next input code point is EOF, do nothing.
  804. auto next_input = peek_code_point();
  805. if (is_eof(next_input))
  806. continue;
  807. // Otherwise, if the next input code point is a newline, consume it.
  808. if (is_newline(next_input)) {
  809. (void)next_code_point();
  810. continue;
  811. }
  812. // Otherwise, (the stream starts with a valid escape) consume an escaped code
  813. // point and append the returned code point to the <string-token>’s value.
  814. auto escaped = consume_escaped_code_point();
  815. builder.append_code_point(escaped);
  816. continue;
  817. }
  818. // anything else
  819. // Append the current input code point to the <string-token>’s value.
  820. builder.append_code_point(input);
  821. }
  822. }
  823. // https://www.w3.org/TR/css-syntax-3/#consume-comment
  824. void Tokenizer::consume_comments()
  825. {
  826. // This section describes how to consume comments from a stream of code points.
  827. // It returns nothing.
  828. start:
  829. // If the next two input code point are U+002F SOLIDUS (/) followed by a U+002A ASTERISK (*),
  830. // consume them and all following code points up to and including the first U+002A ASTERISK (*)
  831. // followed by a U+002F SOLIDUS (/), or up to an EOF code point. Return to the start of this step.
  832. //
  833. // If the preceding paragraph ended by consuming an EOF code point, this is a parse error.
  834. //
  835. // Return nothing.
  836. auto twin = peek_twin();
  837. if (!(is_solidus(twin.first) && is_asterisk(twin.second)))
  838. return;
  839. (void)next_code_point();
  840. (void)next_code_point();
  841. for (;;) {
  842. auto twin_inner = peek_twin();
  843. if (is_eof(twin_inner.first) || is_eof(twin_inner.second)) {
  844. log_parse_error();
  845. return;
  846. }
  847. if (is_asterisk(twin_inner.first) && is_solidus(twin_inner.second)) {
  848. (void)next_code_point();
  849. (void)next_code_point();
  850. goto start;
  851. }
  852. (void)next_code_point();
  853. }
  854. }
  855. // https://www.w3.org/TR/css-syntax-3/#consume-token
  856. Token Tokenizer::consume_a_token()
  857. {
  858. // This section describes how to consume a token from a stream of code points.
  859. // It will return a single token of any type.
  860. // Consume comments.
  861. consume_comments();
  862. // Consume the next input code point.
  863. auto input = next_code_point();
  864. // whitespace
  865. if (is_whitespace(input)) {
  866. dbgln_if(CSS_TOKENIZER_DEBUG, "is whitespace");
  867. // Consume as much whitespace as possible. Return a <whitespace-token>.
  868. consume_as_much_whitespace_as_possible();
  869. return create_new_token(Token::Type::Whitespace);
  870. }
  871. // U+0022 QUOTATION MARK (")
  872. if (is_quotation_mark(input)) {
  873. dbgln_if(CSS_TOKENIZER_DEBUG, "is quotation mark");
  874. // Consume a string token and return it.
  875. return consume_string_token(input);
  876. }
  877. // U+0023 NUMBER SIGN (#)
  878. if (is_number_sign(input)) {
  879. dbgln_if(CSS_TOKENIZER_DEBUG, "is number sign");
  880. // If the next input code point is an ident code point or the next two input code points
  881. // are a valid escape, then:
  882. auto next_input = peek_code_point();
  883. auto maybe_escape = peek_twin();
  884. if (is_ident_code_point(next_input) || is_valid_escape_sequence(maybe_escape)) {
  885. // 1. Create a <hash-token>.
  886. auto token = create_new_token(Token::Type::Hash);
  887. // 2. If the next 3 input code points would start an ident sequence, set the <hash-token>’s
  888. // type flag to "id".
  889. if (would_start_an_ident_sequence(peek_triplet()))
  890. token.m_hash_type = Token::HashType::Id;
  891. // 3. Consume an ident sequence, and set the <hash-token>’s value to the returned string.
  892. auto name = consume_an_ident_sequence();
  893. token.m_value = move(name);
  894. // 4. Return the <hash-token>.
  895. return token;
  896. }
  897. // Otherwise, return a <delim-token> with its value set to the current input code point.
  898. return create_value_token(Token::Type::Delim, input);
  899. }
  900. // U+0027 APOSTROPHE (')
  901. if (is_apostrophe(input)) {
  902. dbgln_if(CSS_TOKENIZER_DEBUG, "is apostrophe");
  903. // Consume a string token and return it.
  904. return consume_string_token(input);
  905. }
  906. // U+0028 LEFT PARENTHESIS (()
  907. if (is_left_paren(input)) {
  908. dbgln_if(CSS_TOKENIZER_DEBUG, "is left paren");
  909. // Return a <(-token>.
  910. return create_new_token(Token::Type::OpenParen);
  911. }
  912. // U+0029 RIGHT PARENTHESIS ())
  913. if (is_right_paren(input)) {
  914. dbgln_if(CSS_TOKENIZER_DEBUG, "is right paren");
  915. // Return a <)-token>.
  916. return create_new_token(Token::Type::CloseParen);
  917. }
  918. // U+002B PLUS SIGN (+)
  919. if (is_plus_sign(input)) {
  920. dbgln_if(CSS_TOKENIZER_DEBUG, "is plus sign");
  921. // If the input stream starts with a number, reconsume the current input code point,
  922. // consume a numeric token and return it.
  923. if (would_start_a_number(start_of_input_stream_triplet())) {
  924. reconsume_current_input_code_point();
  925. return consume_a_numeric_token();
  926. }
  927. // Otherwise, return a <delim-token> with its value set to the current input code point.
  928. return create_value_token(Token::Type::Delim, input);
  929. }
  930. // U+002C COMMA (,)
  931. if (is_comma(input)) {
  932. dbgln_if(CSS_TOKENIZER_DEBUG, "is comma");
  933. // Return a <comma-token>.
  934. return create_new_token(Token::Type::Comma);
  935. }
  936. // U+002D HYPHEN-MINUS (-)
  937. if (is_hyphen_minus(input)) {
  938. dbgln_if(CSS_TOKENIZER_DEBUG, "is hyphen minus");
  939. // If the input stream starts with a number, reconsume the current input code point,
  940. // consume a numeric token, and return it.
  941. if (would_start_a_number(start_of_input_stream_triplet())) {
  942. reconsume_current_input_code_point();
  943. return consume_a_numeric_token();
  944. }
  945. // Otherwise, if the next 2 input code points are U+002D HYPHEN-MINUS U+003E
  946. // GREATER-THAN SIGN (->), consume them and return a <CDC-token>.
  947. auto next_twin = peek_twin();
  948. if (is_hyphen_minus(next_twin.first) && is_greater_than_sign(next_twin.second)) {
  949. (void)next_code_point();
  950. (void)next_code_point();
  951. return create_new_token(Token::Type::CDC);
  952. }
  953. // Otherwise, if the input stream starts with an identifier, reconsume the current
  954. // input code point, consume an ident-like token, and return it.
  955. if (would_start_an_ident_sequence(start_of_input_stream_triplet())) {
  956. reconsume_current_input_code_point();
  957. return consume_an_ident_like_token();
  958. }
  959. // Otherwise, return a <delim-token> with its value set to the current input code point.
  960. return create_value_token(Token::Type::Delim, input);
  961. }
  962. // U+002E FULL STOP (.)
  963. if (is_full_stop(input)) {
  964. dbgln_if(CSS_TOKENIZER_DEBUG, "is full stop");
  965. // If the input stream starts with a number, reconsume the current input code point,
  966. // consume a numeric token, and return it.
  967. if (would_start_a_number(start_of_input_stream_triplet())) {
  968. reconsume_current_input_code_point();
  969. return consume_a_numeric_token();
  970. }
  971. // Otherwise, return a <delim-token> with its value set to the current input code point.
  972. return create_value_token(Token::Type::Delim, input);
  973. }
  974. // U+003A COLON (:)
  975. if (is_colon(input)) {
  976. dbgln_if(CSS_TOKENIZER_DEBUG, "is colon");
  977. // Return a <colon-token>.
  978. return create_new_token(Token::Type::Colon);
  979. }
  980. // U+003B SEMICOLON (;)
  981. if (is_semicolon(input)) {
  982. dbgln_if(CSS_TOKENIZER_DEBUG, "is semicolon");
  983. // Return a <semicolon-token>.
  984. return create_new_token(Token::Type::Semicolon);
  985. }
  986. // U+003C LESS-THAN SIGN (<)
  987. if (is_less_than_sign(input)) {
  988. dbgln_if(CSS_TOKENIZER_DEBUG, "is less than");
  989. // If the next 3 input code points are U+0021 EXCLAMATION MARK U+002D HYPHEN-MINUS
  990. // U+002D HYPHEN-MINUS (!--), consume them and return a <CDO-token>.
  991. auto maybe_cdo = peek_triplet();
  992. if (is_exclamation_mark(maybe_cdo.first) && is_hyphen_minus(maybe_cdo.second) && is_hyphen_minus(maybe_cdo.third)) {
  993. (void)next_code_point();
  994. (void)next_code_point();
  995. (void)next_code_point();
  996. return create_new_token(Token::Type::CDO);
  997. }
  998. // Otherwise, return a <delim-token> with its value set to the current input code point.
  999. return create_value_token(Token::Type::Delim, input);
  1000. }
  1001. // U+0040 COMMERCIAL AT (@)
  1002. if (is_at(input)) {
  1003. dbgln_if(CSS_TOKENIZER_DEBUG, "is at");
  1004. // If the next 3 input code points would start an ident sequence, consume an ident sequence, create
  1005. // an <at-keyword-token> with its value set to the returned value, and return it.
  1006. if (would_start_an_ident_sequence(peek_triplet())) {
  1007. auto name = consume_an_ident_sequence();
  1008. return create_value_token(Token::Type::AtKeyword, name);
  1009. }
  1010. // Otherwise, return a <delim-token> with its value set to the current input code point.
  1011. return create_value_token(Token::Type::Delim, input);
  1012. }
  1013. // U+005B LEFT SQUARE BRACKET ([)
  1014. if (is_open_square_bracket(input)) {
  1015. dbgln_if(CSS_TOKENIZER_DEBUG, "is open square");
  1016. // Return a <[-token>.
  1017. return create_new_token(Token::Type::OpenSquare);
  1018. }
  1019. // U+005C REVERSE SOLIDUS (\)
  1020. if (is_reverse_solidus(input)) {
  1021. dbgln_if(CSS_TOKENIZER_DEBUG, "is reverse solidus");
  1022. // If the input stream starts with a valid escape, reconsume the current input code point,
  1023. // consume an ident-like token, and return it.
  1024. if (is_valid_escape_sequence(start_of_input_stream_twin())) {
  1025. reconsume_current_input_code_point();
  1026. return consume_an_ident_like_token();
  1027. }
  1028. // Otherwise, this is a parse error. Return a <delim-token> with its value set to the
  1029. // current input code point.
  1030. log_parse_error();
  1031. return create_value_token(Token::Type::Delim, input);
  1032. }
  1033. // U+005D RIGHT SQUARE BRACKET (])
  1034. if (is_closed_square_bracket(input)) {
  1035. dbgln_if(CSS_TOKENIZER_DEBUG, "is closed square");
  1036. // Return a <]-token>.
  1037. return create_new_token(Token::Type::CloseSquare);
  1038. }
  1039. // U+007B LEFT CURLY BRACKET ({)
  1040. if (is_open_curly_bracket(input)) {
  1041. dbgln_if(CSS_TOKENIZER_DEBUG, "is open curly");
  1042. // Return a <{-token>.
  1043. return create_new_token(Token::Type::OpenCurly);
  1044. }
  1045. // U+007D RIGHT CURLY BRACKET (})
  1046. if (is_closed_curly_bracket(input)) {
  1047. dbgln_if(CSS_TOKENIZER_DEBUG, "is closed curly");
  1048. // Return a <}-token>.
  1049. return create_new_token(Token::Type::CloseCurly);
  1050. }
  1051. // digit
  1052. if (is_ascii_digit(input)) {
  1053. dbgln_if(CSS_TOKENIZER_DEBUG, "is digit");
  1054. // Reconsume the current input code point, consume a numeric token, and return it.
  1055. reconsume_current_input_code_point();
  1056. return consume_a_numeric_token();
  1057. }
  1058. // name-start code point
  1059. if (is_ident_start_code_point(input)) {
  1060. dbgln_if(CSS_TOKENIZER_DEBUG, "is name start");
  1061. // Reconsume the current input code point, consume an ident-like token, and return it.
  1062. reconsume_current_input_code_point();
  1063. return consume_an_ident_like_token();
  1064. }
  1065. // EOF
  1066. if (is_eof(input)) {
  1067. // Return an <EOF-token>.
  1068. return create_new_token(Token::Type::EndOfFile);
  1069. }
  1070. // anything else
  1071. dbgln_if(CSS_TOKENIZER_DEBUG, "is delimiter");
  1072. // Return a <delim-token> with its value set to the current input code point.
  1073. return create_value_token(Token::Type::Delim, input);
  1074. }
  1075. }