HTMLTokenizer.cpp 85 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575
  1. /*
  2. * Copyright (c) 2020, Andreas Kling <kling@serenityos.org>
  3. * All rights reserved.
  4. *
  5. * Redistribution and use in source and binary forms, with or without
  6. * modification, are permitted provided that the following conditions are met:
  7. *
  8. * 1. Redistributions of source code must retain the above copyright notice, this
  9. * list of conditions and the following disclaimer.
  10. *
  11. * 2. Redistributions in binary form must reproduce the above copyright notice,
  12. * this list of conditions and the following disclaimer in the documentation
  13. * and/or other materials provided with the distribution.
  14. *
  15. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  16. * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17. * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  18. * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
  19. * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20. * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  21. * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  22. * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  23. * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  24. * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25. */
  26. #include <LibTextCodec/Decoder.h>
  27. #include <LibWeb/Parser/Entities.h>
  28. #include <LibWeb/Parser/HTMLToken.h>
  29. #include <LibWeb/Parser/HTMLTokenizer.h>
  30. #include <ctype.h>
  31. #pragma GCC diagnostic ignored "-Wunused-label"
  32. //#define TOKENIZER_TRACE
  33. #ifdef TOKENIZER_TRACE
  34. # define PARSE_ERROR() \
  35. do { \
  36. dbg() << "Parse error (tokenization)" << __PRETTY_FUNCTION__ << " @ " << __LINE__; \
  37. } while (0)
  38. #else
  39. # define PARSE_ERROR()
  40. #endif
  41. #define CONSUME_NEXT_INPUT_CHARACTER \
  42. current_input_character = next_codepoint();
  43. #define SWITCH_TO(new_state) \
  44. do { \
  45. will_switch_to(State::new_state); \
  46. m_state = State::new_state; \
  47. CONSUME_NEXT_INPUT_CHARACTER; \
  48. goto new_state; \
  49. } while (0)
  50. #define RECONSUME_IN(new_state) \
  51. do { \
  52. will_reconsume_in(State::new_state); \
  53. m_state = State::new_state; \
  54. goto new_state; \
  55. } while (0)
  56. #define SWITCH_TO_RETURN_STATE \
  57. do { \
  58. will_switch_to(m_return_state); \
  59. m_state = m_return_state; \
  60. goto _StartOfFunction; \
  61. } while (0)
  62. #define RECONSUME_IN_RETURN_STATE \
  63. do { \
  64. will_reconsume_in(m_return_state); \
  65. m_state = m_return_state; \
  66. goto _StartOfFunction; \
  67. } while (0)
  68. #define SWITCH_TO_AND_EMIT_CURRENT_TOKEN(new_state) \
  69. do { \
  70. will_switch_to(State::new_state); \
  71. m_state = State::new_state; \
  72. will_emit(m_current_token); \
  73. m_queued_tokens.enqueue(m_current_token); \
  74. return m_queued_tokens.dequeue(); \
  75. } while (0)
  76. #define EMIT_CHARACTER_AND_RECONSUME_IN(codepoint, new_state) \
  77. do { \
  78. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint)); \
  79. will_reconsume_in(State::new_state); \
  80. m_state = State::new_state; \
  81. goto new_state; \
  82. } while (0)
  83. #define FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE \
  84. do { \
  85. for (auto codepoint : m_temporary_buffer) { \
  86. if (consumed_as_part_of_an_attribute()) { \
  87. m_current_token.m_tag.attributes.last().value_builder.append_codepoint(codepoint); \
  88. } else { \
  89. create_new_token(HTMLToken::Type::Character); \
  90. m_current_token.m_comment_or_character.data.append_codepoint(codepoint); \
  91. m_queued_tokens.enqueue(m_current_token); \
  92. } \
  93. } \
  94. } while (0)
  95. #define DONT_CONSUME_NEXT_INPUT_CHARACTER \
  96. do { \
  97. m_utf8_iterator = m_prev_utf8_iterator; \
  98. } while (0)
  99. #define ON(codepoint) \
  100. if (current_input_character.has_value() && current_input_character.value() == codepoint)
  101. #define ON_EOF \
  102. if (!current_input_character.has_value())
  103. #define ON_ASCII_ALPHA \
  104. if (current_input_character.has_value() && isalpha(current_input_character.value()))
  105. #define ON_ASCII_ALPHANUMERIC \
  106. if (current_input_character.has_value() && isalnum(current_input_character.value()))
  107. #define ON_ASCII_UPPER_ALPHA \
  108. if (current_input_character.has_value() && current_input_character.value() >= 'A' && current_input_character.value() <= 'Z')
  109. #define ON_ASCII_LOWER_ALPHA \
  110. if (current_input_character.has_value() && current_input_character.value() >= 'a' && current_input_character.value() <= 'z')
  111. #define ON_ASCII_DIGIT \
  112. if (current_input_character.has_value() && isdigit(current_input_character.value()))
  113. #define ON_ASCII_HEX_DIGIT \
  114. if (current_input_character.has_value() && isxdigit(current_input_character.value()))
  115. #define ON_WHITESPACE \
  116. if (current_input_character.has_value() && (current_input_character.value() == '\t' || current_input_character.value() == '\n' || current_input_character.value() == '\f' || current_input_character.value() == ' '))
  117. #define ANYTHING_ELSE if (1)
  118. #define EMIT_EOF \
  119. do { \
  120. if (m_has_emitted_eof) \
  121. return {}; \
  122. m_has_emitted_eof = true; \
  123. create_new_token(HTMLToken::Type::EndOfFile); \
  124. will_emit(m_current_token); \
  125. m_queued_tokens.enqueue(m_current_token); \
  126. return m_queued_tokens.dequeue(); \
  127. } while (0)
  128. #define EMIT_CURRENT_TOKEN \
  129. do { \
  130. will_emit(m_current_token); \
  131. m_queued_tokens.enqueue(m_current_token); \
  132. return m_queued_tokens.dequeue(); \
  133. } while (0)
  134. #define EMIT_CHARACTER(codepoint) \
  135. do { \
  136. create_new_token(HTMLToken::Type::Character); \
  137. m_current_token.m_comment_or_character.data.append_codepoint(codepoint); \
  138. m_queued_tokens.enqueue(m_current_token); \
  139. return m_queued_tokens.dequeue(); \
  140. } while (0)
  141. #define EMIT_CURRENT_CHARACTER \
  142. EMIT_CHARACTER(current_input_character.value());
  143. #define SWITCH_TO_AND_EMIT_CHARACTER(codepoint, new_state) \
  144. do { \
  145. will_switch_to(State::new_state); \
  146. m_state = State::new_state; \
  147. EMIT_CHARACTER(codepoint); \
  148. } while (0)
  149. #define SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(new_state) \
  150. SWITCH_TO_AND_EMIT_CHARACTER(current_input_character.value(), new_state)
  151. #define BEGIN_STATE(state) \
  152. state: \
  153. case State::state: { \
  154. { \
  155. {
  156. #define END_STATE \
  157. ASSERT_NOT_REACHED(); \
  158. break; \
  159. } \
  160. } \
  161. }
  162. static inline bool is_surrogate(u32 codepoint)
  163. {
  164. return (codepoint & 0xfffff800) == 0xd800;
  165. }
  166. static inline bool is_noncharacter(u32 codepoint)
  167. {
  168. return codepoint >= 0xfdd0 && (codepoint <= 0xfdef || (codepoint & 0xfffe) == 0xfffe) && codepoint <= 0x10ffff;
  169. }
  170. static inline bool is_c0_control(u32 codepoint)
  171. {
  172. return codepoint <= 0x1f;
  173. }
  174. static inline bool is_control(u32 codepoint)
  175. {
  176. return is_c0_control(codepoint) || (codepoint >= 0x7f && codepoint <= 0x9f);
  177. }
  178. namespace Web {
  179. Optional<u32> HTMLTokenizer::next_codepoint()
  180. {
  181. if (m_utf8_iterator == m_utf8_view.end())
  182. return {};
  183. m_prev_utf8_iterator = m_utf8_iterator;
  184. ++m_utf8_iterator;
  185. #ifdef TOKENIZER_TRACE
  186. dbg() << "(Tokenizer) Next codepoint: " << (char)*m_prev_utf8_iterator;
  187. #endif
  188. return *m_prev_utf8_iterator;
  189. }
  190. Optional<u32> HTMLTokenizer::peek_codepoint(size_t offset) const
  191. {
  192. auto it = m_utf8_iterator;
  193. for (size_t i = 0; i < offset && it != m_utf8_view.end(); ++i)
  194. ++it;
  195. if (it == m_utf8_view.end())
  196. return {};
  197. return *it;
  198. }
  199. Optional<HTMLToken> HTMLTokenizer::next_token()
  200. {
  201. _StartOfFunction:
  202. if (!m_queued_tokens.is_empty())
  203. return m_queued_tokens.dequeue();
  204. for (;;) {
  205. auto current_input_character = next_codepoint();
  206. switch (m_state) {
  207. BEGIN_STATE(Data)
  208. {
  209. ON('&')
  210. {
  211. m_return_state = State::Data;
  212. SWITCH_TO(CharacterReference);
  213. }
  214. ON('<')
  215. {
  216. SWITCH_TO(TagOpen);
  217. }
  218. ON(0)
  219. {
  220. PARSE_ERROR();
  221. EMIT_CURRENT_CHARACTER;
  222. }
  223. ON_EOF
  224. {
  225. EMIT_EOF;
  226. }
  227. ANYTHING_ELSE
  228. {
  229. EMIT_CURRENT_CHARACTER;
  230. }
  231. }
  232. END_STATE
  233. BEGIN_STATE(TagOpen)
  234. {
  235. ON('!')
  236. {
  237. SWITCH_TO(MarkupDeclarationOpen);
  238. }
  239. ON('/')
  240. {
  241. SWITCH_TO(EndTagOpen);
  242. }
  243. ON_ASCII_ALPHA
  244. {
  245. create_new_token(HTMLToken::Type::StartTag);
  246. RECONSUME_IN(TagName);
  247. }
  248. ON('?')
  249. {
  250. PARSE_ERROR();
  251. create_new_token(HTMLToken::Type::Comment);
  252. RECONSUME_IN(BogusComment);
  253. }
  254. ON_EOF
  255. {
  256. TODO();
  257. }
  258. ANYTHING_ELSE
  259. {
  260. PARSE_ERROR();
  261. EMIT_CHARACTER_AND_RECONSUME_IN('<', Data);
  262. }
  263. }
  264. END_STATE
  265. BEGIN_STATE(TagName)
  266. {
  267. ON_WHITESPACE
  268. {
  269. SWITCH_TO(BeforeAttributeName);
  270. }
  271. ON('/')
  272. {
  273. SWITCH_TO(SelfClosingStartTag);
  274. }
  275. ON('>')
  276. {
  277. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  278. }
  279. ON_ASCII_UPPER_ALPHA
  280. {
  281. m_current_token.m_tag.tag_name.append(tolower(current_input_character.value()));
  282. continue;
  283. }
  284. ON(0)
  285. {
  286. PARSE_ERROR();
  287. m_current_token.m_tag.tag_name.append_codepoint(0xFFFD);
  288. continue;
  289. }
  290. ON_EOF
  291. {
  292. PARSE_ERROR();
  293. EMIT_EOF;
  294. }
  295. ANYTHING_ELSE
  296. {
  297. m_current_token.m_tag.tag_name.append_codepoint(current_input_character.value());
  298. continue;
  299. }
  300. }
  301. END_STATE
  302. BEGIN_STATE(EndTagOpen)
  303. {
  304. ON_ASCII_ALPHA
  305. {
  306. create_new_token(HTMLToken::Type::EndTag);
  307. RECONSUME_IN(TagName);
  308. }
  309. ON('>')
  310. {
  311. PARSE_ERROR();
  312. SWITCH_TO(Data);
  313. }
  314. ON_EOF
  315. {
  316. PARSE_ERROR();
  317. // FIXME: Emit a U+003C LESS-THAN SIGN character token, a U+002F SOLIDUS character token and an end-of-file token.
  318. continue;
  319. }
  320. ANYTHING_ELSE
  321. {
  322. PARSE_ERROR();
  323. create_new_token(HTMLToken::Type::Comment);
  324. RECONSUME_IN(BogusComment);
  325. }
  326. }
  327. END_STATE
  328. BEGIN_STATE(MarkupDeclarationOpen)
  329. {
  330. DONT_CONSUME_NEXT_INPUT_CHARACTER;
  331. if (consume_next_if_match("--")) {
  332. create_new_token(HTMLToken::Type::Comment);
  333. SWITCH_TO(CommentStart);
  334. }
  335. if (consume_next_if_match("DOCTYPE", CaseSensitivity::CaseInsensitive)) {
  336. SWITCH_TO(DOCTYPE);
  337. }
  338. if (consume_next_if_match("[CDATA[")) {
  339. TODO();
  340. }
  341. ANYTHING_ELSE
  342. {
  343. PARSE_ERROR();
  344. create_new_token(HTMLToken::Type::Comment);
  345. SWITCH_TO(BogusComment);
  346. }
  347. }
  348. END_STATE
  349. BEGIN_STATE(BogusComment)
  350. {
  351. ON('>')
  352. {
  353. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  354. }
  355. ON_EOF
  356. {
  357. m_queued_tokens.enqueue(m_current_token);
  358. EMIT_EOF;
  359. }
  360. ON(0)
  361. {
  362. PARSE_ERROR();
  363. m_current_token.m_comment_or_character.data.append_codepoint(0xFFFD);
  364. continue;
  365. }
  366. ANYTHING_ELSE
  367. {
  368. m_current_token.m_comment_or_character.data.append_codepoint(current_input_character.value());
  369. continue;
  370. }
  371. }
  372. END_STATE
  373. BEGIN_STATE(DOCTYPE)
  374. {
  375. ON_WHITESPACE
  376. {
  377. SWITCH_TO(BeforeDOCTYPEName);
  378. }
  379. ON('>')
  380. {
  381. RECONSUME_IN(BeforeDOCTYPEName);
  382. }
  383. ON_EOF
  384. {
  385. PARSE_ERROR();
  386. create_new_token(HTMLToken::Type::DOCTYPE);
  387. m_current_token.m_doctype.force_quirks = true;
  388. m_queued_tokens.enqueue(m_current_token);
  389. EMIT_EOF;
  390. }
  391. ANYTHING_ELSE
  392. {
  393. PARSE_ERROR();
  394. RECONSUME_IN(BeforeDOCTYPEName);
  395. }
  396. }
  397. END_STATE
  398. BEGIN_STATE(BeforeDOCTYPEName)
  399. {
  400. ON_WHITESPACE
  401. {
  402. continue;
  403. }
  404. ON_ASCII_UPPER_ALPHA
  405. {
  406. create_new_token(HTMLToken::Type::DOCTYPE);
  407. m_current_token.m_doctype.name.append(tolower(current_input_character.value()));
  408. SWITCH_TO(DOCTYPEName);
  409. }
  410. ON(0)
  411. {
  412. PARSE_ERROR();
  413. create_new_token(HTMLToken::Type::DOCTYPE);
  414. m_current_token.m_doctype.name.append_codepoint(0xFFFD);
  415. SWITCH_TO(DOCTYPEName);
  416. }
  417. ON('>')
  418. {
  419. PARSE_ERROR();
  420. create_new_token(HTMLToken::Type::DOCTYPE);
  421. m_current_token.m_doctype.force_quirks = true;
  422. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  423. }
  424. ON_EOF
  425. {
  426. PARSE_ERROR();
  427. create_new_token(HTMLToken::Type::DOCTYPE);
  428. m_current_token.m_doctype.force_quirks = true;
  429. m_queued_tokens.enqueue(m_current_token);
  430. EMIT_EOF;
  431. }
  432. ANYTHING_ELSE
  433. {
  434. create_new_token(HTMLToken::Type::DOCTYPE);
  435. m_current_token.m_doctype.name.append_codepoint(current_input_character.value());
  436. SWITCH_TO(DOCTYPEName);
  437. }
  438. }
  439. END_STATE
  440. BEGIN_STATE(DOCTYPEName)
  441. {
  442. ON_WHITESPACE
  443. {
  444. SWITCH_TO(AfterDOCTYPEName);
  445. }
  446. ON('>')
  447. {
  448. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  449. }
  450. ON_ASCII_UPPER_ALPHA
  451. {
  452. m_current_token.m_doctype.name.append(tolower(current_input_character.value()));
  453. continue;
  454. }
  455. ON(0)
  456. {
  457. PARSE_ERROR();
  458. m_current_token.m_doctype.name.append_codepoint(0xFFFD);
  459. continue;
  460. }
  461. ON_EOF
  462. {
  463. PARSE_ERROR();
  464. m_current_token.m_doctype.force_quirks = true;
  465. m_queued_tokens.enqueue(m_current_token);
  466. EMIT_EOF;
  467. }
  468. ANYTHING_ELSE
  469. {
  470. m_current_token.m_doctype.name.append_codepoint(current_input_character.value());
  471. continue;
  472. }
  473. }
  474. END_STATE
  475. BEGIN_STATE(AfterDOCTYPEName)
  476. {
  477. ON_WHITESPACE
  478. {
  479. continue;
  480. }
  481. ON('>')
  482. {
  483. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  484. }
  485. ON_EOF
  486. {
  487. PARSE_ERROR();
  488. m_current_token.m_doctype.force_quirks = true;
  489. m_queued_tokens.enqueue(m_current_token);
  490. EMIT_EOF;
  491. }
  492. ANYTHING_ELSE
  493. {
  494. if (toupper(current_input_character.value()) == 'P' && consume_next_if_match("UBLIC", CaseSensitivity::CaseInsensitive)) {
  495. SWITCH_TO(AfterDOCTYPEPublicKeyword);
  496. }
  497. if (toupper(current_input_character.value()) == 'S' && consume_next_if_match("YSTEM", CaseSensitivity::CaseInsensitive)) {
  498. SWITCH_TO(AfterDOCTYPESystemKeyword);
  499. }
  500. PARSE_ERROR();
  501. m_current_token.m_doctype.force_quirks = true;
  502. RECONSUME_IN(BogusDOCTYPE);
  503. }
  504. }
  505. END_STATE
  506. BEGIN_STATE(AfterDOCTYPEPublicKeyword)
  507. {
  508. ON_WHITESPACE
  509. {
  510. SWITCH_TO(BeforeDOCTYPEPublicIdentifier);
  511. }
  512. ON('"')
  513. {
  514. PARSE_ERROR();
  515. m_current_token.m_doctype.public_identifier.clear();
  516. SWITCH_TO(DOCTYPEPublicIdentifierDoubleQuoted);
  517. }
  518. ON('\'')
  519. {
  520. PARSE_ERROR();
  521. m_current_token.m_doctype.public_identifier.clear();
  522. SWITCH_TO(DOCTYPEPublicIdentifierSingleQuoted);
  523. }
  524. ON('>')
  525. {
  526. PARSE_ERROR();
  527. m_current_token.m_doctype.force_quirks = true;
  528. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  529. }
  530. ON_EOF
  531. {
  532. PARSE_ERROR();
  533. m_current_token.m_doctype.force_quirks = true;
  534. m_queued_tokens.enqueue(m_current_token);
  535. EMIT_EOF;
  536. }
  537. ANYTHING_ELSE
  538. {
  539. PARSE_ERROR();
  540. m_current_token.m_doctype.force_quirks = true;
  541. RECONSUME_IN(BogusDOCTYPE);
  542. }
  543. }
  544. END_STATE
  545. BEGIN_STATE(AfterDOCTYPESystemKeyword)
  546. {
  547. ON_WHITESPACE
  548. {
  549. SWITCH_TO(BeforeDOCTYPESystemIdentifier);
  550. }
  551. ON('"')
  552. {
  553. PARSE_ERROR();
  554. m_current_token.m_doctype.system_identifier.clear();
  555. SWITCH_TO(DOCTYPESystemIdentifierDoubleQuoted);
  556. }
  557. ON('\'')
  558. {
  559. PARSE_ERROR();
  560. m_current_token.m_doctype.system_identifier.clear();
  561. SWITCH_TO(DOCTYPESystemIdentifierSingleQuoted);
  562. }
  563. ON('>')
  564. {
  565. PARSE_ERROR();
  566. m_current_token.m_doctype.force_quirks = true;
  567. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  568. }
  569. ON_EOF
  570. {
  571. PARSE_ERROR();
  572. m_current_token.m_doctype.force_quirks = true;
  573. m_queued_tokens.enqueue(m_current_token);
  574. EMIT_EOF;
  575. }
  576. ANYTHING_ELSE
  577. {
  578. PARSE_ERROR();
  579. m_current_token.m_doctype.force_quirks = true;
  580. RECONSUME_IN(BogusDOCTYPE);
  581. }
  582. }
  583. END_STATE
  584. BEGIN_STATE(BeforeDOCTYPEPublicIdentifier)
  585. {
  586. ON_WHITESPACE
  587. {
  588. continue;
  589. }
  590. ON('"')
  591. {
  592. m_current_token.m_doctype.public_identifier.clear();
  593. SWITCH_TO(DOCTYPEPublicIdentifierDoubleQuoted);
  594. }
  595. ON('\'')
  596. {
  597. m_current_token.m_doctype.public_identifier.clear();
  598. SWITCH_TO(DOCTYPEPublicIdentifierSingleQuoted);
  599. }
  600. ON('>')
  601. {
  602. PARSE_ERROR();
  603. m_current_token.m_doctype.force_quirks = true;
  604. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  605. }
  606. ON_EOF
  607. {
  608. PARSE_ERROR();
  609. m_current_token.m_doctype.force_quirks = true;
  610. m_queued_tokens.enqueue(m_current_token);
  611. EMIT_EOF;
  612. }
  613. ANYTHING_ELSE
  614. {
  615. PARSE_ERROR();
  616. m_current_token.m_doctype.force_quirks = true;
  617. RECONSUME_IN(BogusDOCTYPE);
  618. }
  619. }
  620. END_STATE
  621. BEGIN_STATE(BeforeDOCTYPESystemIdentifier)
  622. {
  623. ON_WHITESPACE
  624. {
  625. continue;
  626. }
  627. ON('"')
  628. {
  629. m_current_token.m_doctype.system_identifier.clear();
  630. SWITCH_TO(DOCTYPESystemIdentifierDoubleQuoted);
  631. }
  632. ON('\'')
  633. {
  634. m_current_token.m_doctype.system_identifier.clear();
  635. SWITCH_TO(DOCTYPESystemIdentifierSingleQuoted);
  636. }
  637. ON('>')
  638. {
  639. PARSE_ERROR();
  640. m_current_token.m_doctype.force_quirks = true;
  641. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  642. }
  643. ON_EOF
  644. {
  645. PARSE_ERROR();
  646. m_current_token.m_doctype.force_quirks = true;
  647. m_queued_tokens.enqueue(m_current_token);
  648. EMIT_EOF;
  649. }
  650. ANYTHING_ELSE
  651. {
  652. PARSE_ERROR();
  653. m_current_token.m_doctype.force_quirks = true;
  654. RECONSUME_IN(BogusDOCTYPE);
  655. }
  656. }
  657. END_STATE
  658. BEGIN_STATE(DOCTYPEPublicIdentifierDoubleQuoted)
  659. {
  660. ON('"')
  661. {
  662. SWITCH_TO(AfterDOCTYPEPublicIdentifier);
  663. }
  664. ON(0)
  665. {
  666. PARSE_ERROR();
  667. m_current_token.m_doctype.public_identifier.append_codepoint(0xFFFD);
  668. continue;
  669. }
  670. ON('>')
  671. {
  672. PARSE_ERROR();
  673. m_current_token.m_doctype.force_quirks = true;
  674. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  675. }
  676. ON_EOF
  677. {
  678. PARSE_ERROR();
  679. m_current_token.m_doctype.force_quirks = true;
  680. m_queued_tokens.enqueue(m_current_token);
  681. EMIT_EOF;
  682. }
  683. ANYTHING_ELSE
  684. {
  685. m_current_token.m_doctype.public_identifier.append_codepoint(current_input_character.value());
  686. continue;
  687. }
  688. }
  689. END_STATE
  690. BEGIN_STATE(DOCTYPEPublicIdentifierSingleQuoted)
  691. {
  692. ON('\'')
  693. {
  694. SWITCH_TO(AfterDOCTYPEPublicIdentifier);
  695. }
  696. ON(0)
  697. {
  698. PARSE_ERROR();
  699. m_current_token.m_doctype.public_identifier.append_codepoint(0xFFFD);
  700. continue;
  701. }
  702. ON('>')
  703. {
  704. PARSE_ERROR();
  705. m_current_token.m_doctype.force_quirks = true;
  706. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  707. }
  708. ON_EOF
  709. {
  710. PARSE_ERROR();
  711. m_current_token.m_doctype.force_quirks = true;
  712. m_queued_tokens.enqueue(m_current_token);
  713. EMIT_EOF;
  714. }
  715. ANYTHING_ELSE
  716. {
  717. m_current_token.m_doctype.public_identifier.append_codepoint(current_input_character.value());
  718. continue;
  719. }
  720. }
  721. END_STATE
  722. BEGIN_STATE(DOCTYPESystemIdentifierDoubleQuoted)
  723. {
  724. ON('"')
  725. {
  726. SWITCH_TO(AfterDOCTYPESystemIdentifier);
  727. }
  728. ON(0)
  729. {
  730. PARSE_ERROR();
  731. m_current_token.m_doctype.system_identifier.append_codepoint(0xFFFD);
  732. continue;
  733. }
  734. ON('>')
  735. {
  736. PARSE_ERROR();
  737. m_current_token.m_doctype.force_quirks = true;
  738. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  739. }
  740. ON_EOF
  741. {
  742. PARSE_ERROR();
  743. m_current_token.m_doctype.force_quirks = true;
  744. m_queued_tokens.enqueue(m_current_token);
  745. EMIT_EOF;
  746. }
  747. ANYTHING_ELSE
  748. {
  749. m_current_token.m_doctype.system_identifier.append_codepoint(current_input_character.value());
  750. continue;
  751. }
  752. }
  753. END_STATE
  754. BEGIN_STATE(DOCTYPESystemIdentifierSingleQuoted)
  755. {
  756. ON('\'')
  757. {
  758. SWITCH_TO(AfterDOCTYPESystemIdentifier);
  759. }
  760. ON(0)
  761. {
  762. PARSE_ERROR();
  763. m_current_token.m_doctype.system_identifier.append_codepoint(0xFFFD);
  764. continue;
  765. }
  766. ON('>')
  767. {
  768. PARSE_ERROR();
  769. m_current_token.m_doctype.force_quirks = true;
  770. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  771. }
  772. ON_EOF
  773. {
  774. PARSE_ERROR();
  775. m_current_token.m_doctype.force_quirks = true;
  776. m_queued_tokens.enqueue(m_current_token);
  777. EMIT_EOF;
  778. }
  779. ANYTHING_ELSE
  780. {
  781. m_current_token.m_doctype.system_identifier.append_codepoint(current_input_character.value());
  782. continue;
  783. }
  784. }
  785. END_STATE
  786. BEGIN_STATE(AfterDOCTYPEPublicIdentifier)
  787. {
  788. ON_WHITESPACE
  789. {
  790. SWITCH_TO(BetweenDOCTYPEPublicAndSystemIdentifiers);
  791. }
  792. ON('>')
  793. {
  794. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  795. }
  796. ON('"')
  797. {
  798. PARSE_ERROR();
  799. m_current_token.m_doctype.system_identifier.clear();
  800. SWITCH_TO(DOCTYPESystemIdentifierDoubleQuoted);
  801. }
  802. ON('\'')
  803. {
  804. PARSE_ERROR();
  805. m_current_token.m_doctype.system_identifier.clear();
  806. SWITCH_TO(DOCTYPESystemIdentifierSingleQuoted);
  807. }
  808. ON_EOF
  809. {
  810. PARSE_ERROR();
  811. m_current_token.m_doctype.force_quirks = true;
  812. m_queued_tokens.enqueue(m_current_token);
  813. EMIT_EOF;
  814. }
  815. ANYTHING_ELSE
  816. {
  817. PARSE_ERROR();
  818. m_current_token.m_doctype.force_quirks = true;
  819. RECONSUME_IN(BogusDOCTYPE);
  820. }
  821. }
  822. END_STATE
  823. BEGIN_STATE(BetweenDOCTYPEPublicAndSystemIdentifiers)
  824. {
  825. ON_WHITESPACE
  826. {
  827. continue;
  828. }
  829. ON('>')
  830. {
  831. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  832. }
  833. ON('"')
  834. {
  835. m_current_token.m_doctype.system_identifier.clear();
  836. SWITCH_TO(DOCTYPESystemIdentifierDoubleQuoted);
  837. }
  838. ON('\'')
  839. {
  840. m_current_token.m_doctype.system_identifier.clear();
  841. SWITCH_TO(DOCTYPESystemIdentifierSingleQuoted);
  842. }
  843. ON_EOF
  844. {
  845. PARSE_ERROR();
  846. m_current_token.m_doctype.force_quirks = true;
  847. m_queued_tokens.enqueue(m_current_token);
  848. EMIT_EOF;
  849. }
  850. ANYTHING_ELSE
  851. {
  852. PARSE_ERROR();
  853. m_current_token.m_doctype.force_quirks = true;
  854. RECONSUME_IN(BogusDOCTYPE);
  855. }
  856. }
  857. END_STATE
  858. BEGIN_STATE(AfterDOCTYPESystemIdentifier)
  859. {
  860. ON_WHITESPACE
  861. {
  862. continue;
  863. }
  864. ON('>')
  865. {
  866. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  867. }
  868. ON_EOF
  869. {
  870. PARSE_ERROR();
  871. m_current_token.m_doctype.force_quirks = true;
  872. m_queued_tokens.enqueue(m_current_token);
  873. EMIT_EOF;
  874. }
  875. ANYTHING_ELSE
  876. {
  877. PARSE_ERROR();
  878. RECONSUME_IN(BogusDOCTYPE);
  879. }
  880. }
  881. END_STATE
  882. BEGIN_STATE(BogusDOCTYPE)
  883. {
  884. ON('>')
  885. {
  886. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  887. }
  888. ON(0)
  889. {
  890. PARSE_ERROR();
  891. continue;
  892. }
  893. ON_EOF
  894. {
  895. m_queued_tokens.enqueue(m_current_token);
  896. EMIT_EOF;
  897. }
  898. ANYTHING_ELSE
  899. {
  900. continue;
  901. }
  902. }
  903. END_STATE
  904. BEGIN_STATE(BeforeAttributeName)
  905. {
  906. ON_WHITESPACE
  907. {
  908. continue;
  909. }
  910. ON('/')
  911. {
  912. RECONSUME_IN(AfterAttributeName);
  913. }
  914. ON('>')
  915. {
  916. RECONSUME_IN(AfterAttributeName);
  917. }
  918. ON_EOF
  919. {
  920. RECONSUME_IN(AfterAttributeName);
  921. }
  922. ON('=')
  923. {
  924. TODO();
  925. }
  926. ANYTHING_ELSE
  927. {
  928. m_current_token.m_tag.attributes.append(HTMLToken::AttributeBuilder());
  929. RECONSUME_IN(AttributeName);
  930. }
  931. }
  932. END_STATE
  933. BEGIN_STATE(SelfClosingStartTag)
  934. {
  935. ON('>')
  936. {
  937. m_current_token.m_tag.self_closing = true;
  938. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  939. }
  940. ON_EOF
  941. {
  942. PARSE_ERROR();
  943. EMIT_EOF;
  944. }
  945. ANYTHING_ELSE
  946. {
  947. PARSE_ERROR();
  948. RECONSUME_IN(BeforeAttributeName);
  949. }
  950. }
  951. END_STATE
  952. BEGIN_STATE(AttributeName)
  953. {
  954. ON_WHITESPACE
  955. {
  956. RECONSUME_IN(AfterAttributeName);
  957. }
  958. ON('/')
  959. {
  960. RECONSUME_IN(AfterAttributeName);
  961. }
  962. ON('>')
  963. {
  964. RECONSUME_IN(AfterAttributeName);
  965. }
  966. ON_EOF
  967. {
  968. RECONSUME_IN(AfterAttributeName);
  969. }
  970. ON('=')
  971. {
  972. SWITCH_TO(BeforeAttributeValue);
  973. }
  974. ANYTHING_ELSE
  975. {
  976. m_current_token.m_tag.attributes.last().local_name_builder.append_codepoint(current_input_character.value());
  977. continue;
  978. }
  979. }
  980. END_STATE
  981. BEGIN_STATE(AfterAttributeName)
  982. {
  983. ON_WHITESPACE
  984. {
  985. continue;
  986. }
  987. ON('/')
  988. {
  989. SWITCH_TO(SelfClosingStartTag);
  990. }
  991. ON('=')
  992. {
  993. SWITCH_TO(BeforeAttributeValue);
  994. }
  995. ON('>')
  996. {
  997. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  998. }
  999. ON_EOF
  1000. {
  1001. TODO();
  1002. }
  1003. ANYTHING_ELSE
  1004. {
  1005. m_current_token.m_tag.attributes.append(HTMLToken::AttributeBuilder());
  1006. RECONSUME_IN(AttributeName);
  1007. }
  1008. }
  1009. END_STATE
  1010. BEGIN_STATE(BeforeAttributeValue)
  1011. {
  1012. ON_WHITESPACE
  1013. {
  1014. continue;
  1015. }
  1016. ON('"')
  1017. {
  1018. SWITCH_TO(AttributeValueDoubleQuoted);
  1019. }
  1020. ON('\'')
  1021. {
  1022. SWITCH_TO(AttributeValueSingleQuoted);
  1023. }
  1024. ON('>')
  1025. {
  1026. PARSE_ERROR();
  1027. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  1028. }
  1029. ANYTHING_ELSE
  1030. {
  1031. RECONSUME_IN(AttributeValueUnquoted);
  1032. }
  1033. }
  1034. END_STATE
  1035. BEGIN_STATE(AttributeValueDoubleQuoted)
  1036. {
  1037. ON('"')
  1038. {
  1039. SWITCH_TO(AfterAttributeValueQuoted);
  1040. }
  1041. ON('&')
  1042. {
  1043. m_return_state = State::AttributeValueDoubleQuoted;
  1044. SWITCH_TO(CharacterReference);
  1045. }
  1046. ON(0)
  1047. {
  1048. TODO();
  1049. }
  1050. ON_EOF
  1051. {
  1052. TODO();
  1053. }
  1054. ANYTHING_ELSE
  1055. {
  1056. m_current_token.m_tag.attributes.last().value_builder.append_codepoint(current_input_character.value());
  1057. continue;
  1058. }
  1059. }
  1060. END_STATE
  1061. BEGIN_STATE(AttributeValueSingleQuoted)
  1062. {
  1063. ON('\'')
  1064. {
  1065. SWITCH_TO(AfterAttributeValueQuoted);
  1066. }
  1067. ON('&')
  1068. {
  1069. m_return_state = State::AttributeValueSingleQuoted;
  1070. SWITCH_TO(CharacterReference);
  1071. }
  1072. ON(0)
  1073. {
  1074. TODO();
  1075. }
  1076. ON_EOF
  1077. {
  1078. TODO();
  1079. }
  1080. ANYTHING_ELSE
  1081. {
  1082. m_current_token.m_tag.attributes.last().value_builder.append_codepoint(current_input_character.value());
  1083. continue;
  1084. }
  1085. }
  1086. END_STATE
  1087. BEGIN_STATE(AttributeValueUnquoted)
  1088. {
  1089. ON_WHITESPACE
  1090. {
  1091. SWITCH_TO(BeforeAttributeName);
  1092. }
  1093. ON('&')
  1094. {
  1095. m_return_state = State::AttributeValueUnquoted;
  1096. SWITCH_TO(CharacterReference);
  1097. }
  1098. ON('>')
  1099. {
  1100. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  1101. }
  1102. ON(0)
  1103. {
  1104. TODO();
  1105. }
  1106. ON_EOF
  1107. {
  1108. TODO();
  1109. }
  1110. ANYTHING_ELSE
  1111. {
  1112. m_current_token.m_tag.attributes.last().value_builder.append_codepoint(current_input_character.value());
  1113. continue;
  1114. }
  1115. }
  1116. END_STATE
  1117. BEGIN_STATE(AfterAttributeValueQuoted)
  1118. {
  1119. ON_WHITESPACE
  1120. {
  1121. SWITCH_TO(BeforeAttributeName);
  1122. }
  1123. ON('/')
  1124. {
  1125. SWITCH_TO(SelfClosingStartTag);
  1126. }
  1127. ON('>')
  1128. {
  1129. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  1130. }
  1131. ON_EOF
  1132. {
  1133. PARSE_ERROR();
  1134. EMIT_EOF;
  1135. }
  1136. ANYTHING_ELSE
  1137. {
  1138. PARSE_ERROR();
  1139. RECONSUME_IN(BeforeAttributeName);
  1140. }
  1141. }
  1142. END_STATE
  1143. BEGIN_STATE(CommentStart)
  1144. {
  1145. ON('-')
  1146. {
  1147. SWITCH_TO(CommentStartDash);
  1148. }
  1149. ON('>')
  1150. {
  1151. PARSE_ERROR();
  1152. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  1153. }
  1154. ANYTHING_ELSE
  1155. {
  1156. RECONSUME_IN(Comment);
  1157. }
  1158. }
  1159. END_STATE
  1160. BEGIN_STATE(CommentStartDash)
  1161. {
  1162. ON('-')
  1163. {
  1164. SWITCH_TO(CommentEnd);
  1165. }
  1166. ON('>')
  1167. {
  1168. PARSE_ERROR();
  1169. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  1170. }
  1171. ON_EOF
  1172. {
  1173. PARSE_ERROR();
  1174. m_queued_tokens.enqueue(m_current_token);
  1175. EMIT_EOF;
  1176. }
  1177. ANYTHING_ELSE
  1178. {
  1179. m_current_token.m_comment_or_character.data.append('-');
  1180. RECONSUME_IN(Comment);
  1181. }
  1182. }
  1183. END_STATE
  1184. BEGIN_STATE(Comment)
  1185. {
  1186. ON('<')
  1187. {
  1188. m_current_token.m_comment_or_character.data.append_codepoint(current_input_character.value());
  1189. SWITCH_TO(CommentLessThanSign);
  1190. }
  1191. ON('-')
  1192. {
  1193. SWITCH_TO(CommentEndDash);
  1194. }
  1195. ON(0)
  1196. {
  1197. PARSE_ERROR();
  1198. m_current_token.m_comment_or_character.data.append_codepoint(0xFFFD);
  1199. continue;
  1200. }
  1201. ON_EOF
  1202. {
  1203. PARSE_ERROR();
  1204. m_queued_tokens.enqueue(m_current_token);
  1205. EMIT_EOF;
  1206. }
  1207. ANYTHING_ELSE
  1208. {
  1209. m_current_token.m_comment_or_character.data.append_codepoint(current_input_character.value());
  1210. continue;
  1211. }
  1212. }
  1213. END_STATE
  1214. BEGIN_STATE(CommentEnd)
  1215. {
  1216. ON('>')
  1217. {
  1218. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  1219. }
  1220. ON('!')
  1221. {
  1222. SWITCH_TO(CommentEndBang);
  1223. }
  1224. ON('-')
  1225. {
  1226. m_current_token.m_comment_or_character.data.append('-');
  1227. continue;
  1228. }
  1229. ON_EOF
  1230. {
  1231. PARSE_ERROR();
  1232. m_queued_tokens.enqueue(m_current_token);
  1233. EMIT_EOF;
  1234. }
  1235. ANYTHING_ELSE
  1236. {
  1237. m_current_token.m_comment_or_character.data.append('-');
  1238. RECONSUME_IN(Comment);
  1239. }
  1240. }
  1241. END_STATE
  1242. BEGIN_STATE(CommentEndBang)
  1243. {
  1244. ON('-')
  1245. {
  1246. m_current_token.m_comment_or_character.data.append("--!");
  1247. SWITCH_TO(CommentEndDash);
  1248. }
  1249. ON('>')
  1250. {
  1251. PARSE_ERROR();
  1252. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  1253. }
  1254. ON_EOF
  1255. {
  1256. PARSE_ERROR();
  1257. m_queued_tokens.enqueue(m_current_token);
  1258. EMIT_EOF;
  1259. }
  1260. ANYTHING_ELSE
  1261. {
  1262. m_current_token.m_comment_or_character.data.append("--!");
  1263. RECONSUME_IN(Comment);
  1264. }
  1265. }
  1266. END_STATE
  1267. BEGIN_STATE(CommentEndDash)
  1268. {
  1269. ON('-')
  1270. {
  1271. SWITCH_TO(CommentEnd);
  1272. }
  1273. ON_EOF
  1274. {
  1275. PARSE_ERROR();
  1276. m_queued_tokens.enqueue(m_current_token);
  1277. EMIT_EOF;
  1278. }
  1279. ANYTHING_ELSE
  1280. {
  1281. m_current_token.m_comment_or_character.data.append('-');
  1282. RECONSUME_IN(Comment);
  1283. }
  1284. }
  1285. END_STATE
  1286. BEGIN_STATE(CommentLessThanSign)
  1287. {
  1288. ON('!')
  1289. {
  1290. m_current_token.m_comment_or_character.data.append_codepoint(current_input_character.value());
  1291. SWITCH_TO(CommentLessThanSignBang);
  1292. }
  1293. ON('<')
  1294. {
  1295. m_current_token.m_comment_or_character.data.append_codepoint(current_input_character.value());
  1296. continue;
  1297. }
  1298. ANYTHING_ELSE
  1299. {
  1300. RECONSUME_IN(Comment);
  1301. }
  1302. }
  1303. END_STATE
  1304. BEGIN_STATE(CommentLessThanSignBang)
  1305. {
  1306. ON('-')
  1307. {
  1308. SWITCH_TO(CommentLessThanSignBangDash);
  1309. }
  1310. ANYTHING_ELSE
  1311. {
  1312. RECONSUME_IN(Comment);
  1313. }
  1314. }
  1315. END_STATE
  1316. BEGIN_STATE(CommentLessThanSignBangDash)
  1317. {
  1318. ON('-')
  1319. {
  1320. SWITCH_TO(CommentLessThanSignBangDashDash);
  1321. }
  1322. ANYTHING_ELSE
  1323. {
  1324. RECONSUME_IN(CommentEndDash);
  1325. }
  1326. }
  1327. END_STATE
  1328. BEGIN_STATE(CommentLessThanSignBangDashDash)
  1329. {
  1330. ON('>')
  1331. {
  1332. RECONSUME_IN(CommentEnd);
  1333. }
  1334. ON_EOF
  1335. {
  1336. RECONSUME_IN(CommentEnd);
  1337. }
  1338. ANYTHING_ELSE
  1339. {
  1340. PARSE_ERROR();
  1341. RECONSUME_IN(CommentEnd);
  1342. }
  1343. }
  1344. END_STATE
  1345. BEGIN_STATE(CharacterReference)
  1346. {
  1347. m_temporary_buffer.clear();
  1348. m_temporary_buffer.append('&');
  1349. ON_ASCII_ALPHANUMERIC
  1350. {
  1351. RECONSUME_IN(NamedCharacterReference);
  1352. }
  1353. ON('#')
  1354. {
  1355. m_temporary_buffer.append(current_input_character.value());
  1356. SWITCH_TO(NumericCharacterReference);
  1357. }
  1358. ANYTHING_ELSE
  1359. {
  1360. RECONSUME_IN_RETURN_STATE;
  1361. }
  1362. }
  1363. END_STATE
  1364. BEGIN_STATE(NamedCharacterReference)
  1365. {
  1366. size_t byte_offset = m_utf8_view.byte_offset_of(m_prev_utf8_iterator);
  1367. auto match = HTML::codepoints_from_entity(m_decoded_input.substring_view(byte_offset, m_decoded_input.length() - byte_offset - 1));
  1368. if (match.has_value()) {
  1369. for (size_t i = 0; i < match.value().entity.length() - 1; ++i) {
  1370. m_prev_utf8_iterator = m_utf8_iterator;
  1371. ++m_utf8_iterator;
  1372. }
  1373. for (auto ch : match.value().entity)
  1374. m_temporary_buffer.append(ch);
  1375. if (consumed_as_part_of_an_attribute() && match.value().entity.ends_with(';')) {
  1376. auto next_codepoint = peek_codepoint(0);
  1377. if (next_codepoint.has_value() && next_codepoint.value() == '=') {
  1378. FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE;
  1379. SWITCH_TO_RETURN_STATE;
  1380. }
  1381. }
  1382. if (!match.value().entity.ends_with(';')) {
  1383. PARSE_ERROR();
  1384. }
  1385. m_temporary_buffer.clear();
  1386. m_temporary_buffer.append(match.value().codepoints);
  1387. FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE;
  1388. SWITCH_TO_RETURN_STATE;
  1389. } else {
  1390. FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE;
  1391. SWITCH_TO(AmbiguousAmpersand);
  1392. }
  1393. }
  1394. END_STATE
  1395. BEGIN_STATE(AmbiguousAmpersand)
  1396. {
  1397. ON_ASCII_ALPHANUMERIC
  1398. {
  1399. if (consumed_as_part_of_an_attribute()) {
  1400. m_current_token.m_tag.attributes.last().value_builder.append_codepoint(current_input_character.value());
  1401. continue;
  1402. } else {
  1403. EMIT_CURRENT_CHARACTER;
  1404. }
  1405. }
  1406. ON(';')
  1407. {
  1408. TODO();
  1409. }
  1410. ANYTHING_ELSE
  1411. {
  1412. RECONSUME_IN_RETURN_STATE;
  1413. }
  1414. }
  1415. END_STATE
  1416. BEGIN_STATE(NumericCharacterReference)
  1417. {
  1418. m_character_reference_code = 0;
  1419. ON('X')
  1420. {
  1421. m_temporary_buffer.append(current_input_character.value());
  1422. SWITCH_TO(HexadecimalCharacterReferenceStart);
  1423. }
  1424. ON('x')
  1425. {
  1426. m_temporary_buffer.append(current_input_character.value());
  1427. SWITCH_TO(HexadecimalCharacterReferenceStart);
  1428. }
  1429. ANYTHING_ELSE
  1430. {
  1431. RECONSUME_IN(DecimalCharacterReferenceStart);
  1432. }
  1433. }
  1434. END_STATE
  1435. BEGIN_STATE(HexadecimalCharacterReferenceStart)
  1436. {
  1437. ON_ASCII_HEX_DIGIT
  1438. {
  1439. RECONSUME_IN(HexadecimalCharacterReference);
  1440. }
  1441. ANYTHING_ELSE
  1442. {
  1443. TODO();
  1444. }
  1445. }
  1446. END_STATE
  1447. BEGIN_STATE(DecimalCharacterReferenceStart)
  1448. {
  1449. ON_ASCII_DIGIT
  1450. {
  1451. RECONSUME_IN(DecimalCharacterReference);
  1452. }
  1453. ANYTHING_ELSE
  1454. {
  1455. TODO();
  1456. }
  1457. }
  1458. END_STATE
  1459. BEGIN_STATE(HexadecimalCharacterReference)
  1460. {
  1461. ON_ASCII_DIGIT
  1462. {
  1463. m_character_reference_code *= 16;
  1464. m_character_reference_code += current_input_character.value() - 0x30;
  1465. continue;
  1466. }
  1467. ON_ASCII_UPPER_ALPHA
  1468. {
  1469. m_character_reference_code *= 16;
  1470. m_character_reference_code += current_input_character.value() - 0x37;
  1471. continue;
  1472. }
  1473. ON_ASCII_LOWER_ALPHA
  1474. {
  1475. m_character_reference_code *= 16;
  1476. m_character_reference_code += current_input_character.value() - 0x57;
  1477. continue;
  1478. }
  1479. ON(';')
  1480. {
  1481. SWITCH_TO(NumericCharacterReferenceEnd);
  1482. }
  1483. ANYTHING_ELSE
  1484. {
  1485. TODO();
  1486. }
  1487. }
  1488. END_STATE
  1489. BEGIN_STATE(DecimalCharacterReference)
  1490. {
  1491. ON_ASCII_DIGIT
  1492. {
  1493. m_character_reference_code *= 10;
  1494. m_character_reference_code += current_input_character.value() - 0x30;
  1495. continue;
  1496. }
  1497. ON(';')
  1498. {
  1499. SWITCH_TO(NumericCharacterReferenceEnd);
  1500. }
  1501. ANYTHING_ELSE
  1502. {
  1503. TODO();
  1504. }
  1505. }
  1506. END_STATE
  1507. BEGIN_STATE(NumericCharacterReferenceEnd)
  1508. {
  1509. DONT_CONSUME_NEXT_INPUT_CHARACTER;
  1510. if (m_character_reference_code == 0) {
  1511. TODO();
  1512. }
  1513. if (m_character_reference_code > 0x10ffff) {
  1514. TODO();
  1515. }
  1516. if (is_surrogate(m_character_reference_code)) {
  1517. TODO();
  1518. }
  1519. if (is_noncharacter(m_character_reference_code)) {
  1520. TODO();
  1521. }
  1522. if (m_character_reference_code == 0xd || (is_control(m_character_reference_code) && !isspace(m_character_reference_code))) {
  1523. TODO();
  1524. }
  1525. if (is_control(m_character_reference_code)) {
  1526. constexpr struct {
  1527. u32 number;
  1528. u32 codepoint;
  1529. } conversion_table[] = {
  1530. { 0x80, 0x20AC },
  1531. { 0x82, 0x201A },
  1532. { 0x83, 0x0192 },
  1533. { 0x84, 0x201E },
  1534. { 0x85, 0x2026 },
  1535. { 0x86, 0x2020 },
  1536. { 0x87, 0x2021 },
  1537. { 0x88, 0x02C6 },
  1538. { 0x89, 0x2030 },
  1539. { 0x8A, 0x0160 },
  1540. { 0x8B, 0x2039 },
  1541. { 0x8C, 0x0152 },
  1542. { 0x8E, 0x017D },
  1543. { 0x91, 0x2018 },
  1544. { 0x92, 0x2019 },
  1545. { 0x93, 0x201C },
  1546. { 0x94, 0x201D },
  1547. { 0x95, 0x2022 },
  1548. { 0x96, 0x2013 },
  1549. { 0x97, 0x2014 },
  1550. { 0x98, 0x02DC },
  1551. { 0x99, 0x2122 },
  1552. { 0x9A, 0x0161 },
  1553. { 0x9B, 0x203A },
  1554. { 0x9C, 0x0153 },
  1555. { 0x9E, 0x017E },
  1556. { 0x9F, 0x0178 },
  1557. };
  1558. for (auto& entry : conversion_table) {
  1559. if (m_character_reference_code == entry.number) {
  1560. m_character_reference_code = entry.codepoint;
  1561. break;
  1562. }
  1563. }
  1564. }
  1565. m_temporary_buffer.clear();
  1566. m_temporary_buffer.append(m_character_reference_code);
  1567. FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE;
  1568. SWITCH_TO_RETURN_STATE;
  1569. }
  1570. END_STATE
  1571. BEGIN_STATE(RCDATA)
  1572. {
  1573. ON('&')
  1574. {
  1575. m_return_state = State::RCDATA;
  1576. SWITCH_TO(CharacterReference);
  1577. }
  1578. ON('<')
  1579. {
  1580. SWITCH_TO(RCDATALessThanSign);
  1581. }
  1582. ON(0)
  1583. {
  1584. PARSE_ERROR();
  1585. EMIT_CHARACTER(0xFFFD);
  1586. }
  1587. ON_EOF
  1588. {
  1589. EMIT_EOF;
  1590. }
  1591. ANYTHING_ELSE
  1592. {
  1593. EMIT_CURRENT_CHARACTER;
  1594. }
  1595. }
  1596. END_STATE
  1597. BEGIN_STATE(RCDATALessThanSign)
  1598. {
  1599. ON('/')
  1600. {
  1601. m_temporary_buffer.clear();
  1602. SWITCH_TO(RCDATAEndTagOpen);
  1603. }
  1604. ANYTHING_ELSE
  1605. {
  1606. EMIT_CHARACTER_AND_RECONSUME_IN('<', RCDATA);
  1607. }
  1608. }
  1609. END_STATE
  1610. BEGIN_STATE(RCDATAEndTagOpen)
  1611. {
  1612. ON_ASCII_ALPHA
  1613. {
  1614. create_new_token(HTMLToken::Type::EndTag);
  1615. RECONSUME_IN(RCDATAEndTagName);
  1616. }
  1617. ANYTHING_ELSE
  1618. {
  1619. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  1620. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  1621. RECONSUME_IN(RCDATA);
  1622. }
  1623. }
  1624. END_STATE
  1625. BEGIN_STATE(RCDATAEndTagName)
  1626. {
  1627. ON_WHITESPACE
  1628. {
  1629. if (!current_end_tag_token_is_appropriate()) {
  1630. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  1631. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  1632. for (auto codepoint : m_temporary_buffer)
  1633. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint));
  1634. RECONSUME_IN(RCDATA);
  1635. }
  1636. SWITCH_TO(BeforeAttributeName);
  1637. }
  1638. ON('/')
  1639. {
  1640. if (!current_end_tag_token_is_appropriate()) {
  1641. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  1642. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  1643. for (auto codepoint : m_temporary_buffer)
  1644. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint));
  1645. RECONSUME_IN(RCDATA);
  1646. }
  1647. SWITCH_TO(SelfClosingStartTag);
  1648. }
  1649. ON('>')
  1650. {
  1651. if (!current_end_tag_token_is_appropriate()) {
  1652. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  1653. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  1654. for (auto codepoint : m_temporary_buffer)
  1655. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint));
  1656. RECONSUME_IN(RCDATA);
  1657. }
  1658. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  1659. }
  1660. ON_ASCII_UPPER_ALPHA
  1661. {
  1662. m_current_token.m_tag.tag_name.append(tolower(current_input_character.value()));
  1663. m_temporary_buffer.append(current_input_character.value());
  1664. continue;
  1665. }
  1666. ON_ASCII_LOWER_ALPHA
  1667. {
  1668. m_current_token.m_tag.tag_name.append_codepoint(current_input_character.value());
  1669. m_temporary_buffer.append(current_input_character.value());
  1670. continue;
  1671. }
  1672. ANYTHING_ELSE
  1673. {
  1674. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  1675. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  1676. for (auto codepoint : m_temporary_buffer)
  1677. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint));
  1678. RECONSUME_IN(RCDATA);
  1679. }
  1680. }
  1681. END_STATE
  1682. BEGIN_STATE(RAWTEXT)
  1683. {
  1684. ON('<')
  1685. {
  1686. SWITCH_TO(RAWTEXTLessThanSign);
  1687. }
  1688. ON(0)
  1689. {
  1690. PARSE_ERROR();
  1691. EMIT_CHARACTER(0xFFFD);
  1692. }
  1693. ON_EOF
  1694. {
  1695. EMIT_EOF;
  1696. }
  1697. ANYTHING_ELSE
  1698. {
  1699. EMIT_CURRENT_CHARACTER;
  1700. }
  1701. }
  1702. END_STATE
  1703. BEGIN_STATE(RAWTEXTLessThanSign)
  1704. {
  1705. ON('/')
  1706. {
  1707. m_temporary_buffer.clear();
  1708. SWITCH_TO(RAWTEXTEndTagOpen);
  1709. }
  1710. ANYTHING_ELSE
  1711. {
  1712. EMIT_CHARACTER_AND_RECONSUME_IN('<', RAWTEXT);
  1713. }
  1714. }
  1715. END_STATE
  1716. BEGIN_STATE(RAWTEXTEndTagOpen)
  1717. {
  1718. ON_ASCII_ALPHA
  1719. {
  1720. create_new_token(HTMLToken::Type::EndTag);
  1721. RECONSUME_IN(RAWTEXTEndTagName);
  1722. }
  1723. ANYTHING_ELSE
  1724. {
  1725. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  1726. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  1727. RECONSUME_IN(RAWTEXT);
  1728. }
  1729. }
  1730. END_STATE
  1731. BEGIN_STATE(RAWTEXTEndTagName)
  1732. {
  1733. ON_WHITESPACE
  1734. {
  1735. if (!current_end_tag_token_is_appropriate()) {
  1736. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  1737. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  1738. for (auto codepoint : m_temporary_buffer)
  1739. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint));
  1740. RECONSUME_IN(RAWTEXT);
  1741. }
  1742. SWITCH_TO(BeforeAttributeName);
  1743. }
  1744. ON('/')
  1745. {
  1746. if (!current_end_tag_token_is_appropriate()) {
  1747. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  1748. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  1749. for (auto codepoint : m_temporary_buffer)
  1750. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint));
  1751. RECONSUME_IN(RAWTEXT);
  1752. }
  1753. SWITCH_TO(SelfClosingStartTag);
  1754. }
  1755. ON('>')
  1756. {
  1757. if (!current_end_tag_token_is_appropriate()) {
  1758. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  1759. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  1760. for (auto codepoint : m_temporary_buffer)
  1761. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint));
  1762. RECONSUME_IN(RAWTEXT);
  1763. }
  1764. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  1765. }
  1766. ON_ASCII_UPPER_ALPHA
  1767. {
  1768. m_current_token.m_tag.tag_name.append(tolower(current_input_character.value()));
  1769. m_temporary_buffer.append(current_input_character.value());
  1770. continue;
  1771. }
  1772. ON_ASCII_LOWER_ALPHA
  1773. {
  1774. m_current_token.m_tag.tag_name.append(current_input_character.value());
  1775. m_temporary_buffer.append(current_input_character.value());
  1776. continue;
  1777. }
  1778. ANYTHING_ELSE
  1779. {
  1780. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  1781. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  1782. for (auto codepoint : m_temporary_buffer)
  1783. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint));
  1784. RECONSUME_IN(RAWTEXT);
  1785. }
  1786. }
  1787. END_STATE
  1788. BEGIN_STATE(ScriptData)
  1789. {
  1790. ON('<')
  1791. {
  1792. SWITCH_TO(ScriptDataLessThanSign);
  1793. }
  1794. ON(0)
  1795. {
  1796. PARSE_ERROR();
  1797. EMIT_CHARACTER(0xFFFD);
  1798. }
  1799. ON_EOF
  1800. {
  1801. EMIT_EOF;
  1802. }
  1803. ANYTHING_ELSE
  1804. {
  1805. EMIT_CURRENT_CHARACTER;
  1806. }
  1807. }
  1808. END_STATE
  1809. BEGIN_STATE(PLAINTEXT)
  1810. {
  1811. ON(0)
  1812. {
  1813. PARSE_ERROR();
  1814. EMIT_CHARACTER(0xFFFD);
  1815. }
  1816. ON_EOF
  1817. {
  1818. EMIT_EOF;
  1819. }
  1820. ANYTHING_ELSE
  1821. {
  1822. EMIT_CURRENT_CHARACTER;
  1823. }
  1824. }
  1825. END_STATE
  1826. BEGIN_STATE(ScriptDataLessThanSign)
  1827. {
  1828. ON('/')
  1829. {
  1830. m_temporary_buffer.clear();
  1831. SWITCH_TO(ScriptDataEndTagOpen);
  1832. }
  1833. ON('!')
  1834. {
  1835. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  1836. m_queued_tokens.enqueue(HTMLToken::make_character('!'));
  1837. SWITCH_TO(ScriptDataEscapeStart);
  1838. }
  1839. ANYTHING_ELSE
  1840. {
  1841. EMIT_CHARACTER_AND_RECONSUME_IN('<', ScriptData);
  1842. }
  1843. }
  1844. END_STATE
  1845. BEGIN_STATE(ScriptDataEscapeStart)
  1846. {
  1847. ON('-')
  1848. {
  1849. SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataEscapeStartDash);
  1850. }
  1851. ANYTHING_ELSE
  1852. {
  1853. RECONSUME_IN(ScriptData);
  1854. }
  1855. }
  1856. END_STATE
  1857. BEGIN_STATE(ScriptDataEscapeStartDash)
  1858. {
  1859. ON('-')
  1860. {
  1861. SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataEscapedDashDash);
  1862. }
  1863. ANYTHING_ELSE
  1864. {
  1865. RECONSUME_IN(ScriptData);
  1866. }
  1867. }
  1868. END_STATE
  1869. BEGIN_STATE(ScriptDataEscapedDashDash)
  1870. {
  1871. ON('-')
  1872. {
  1873. EMIT_CHARACTER('-');
  1874. }
  1875. ON('<')
  1876. {
  1877. SWITCH_TO(ScriptDataEscapedLessThanSign);
  1878. }
  1879. ON('>')
  1880. {
  1881. SWITCH_TO_AND_EMIT_CHARACTER('>', ScriptData);
  1882. }
  1883. ON(0)
  1884. {
  1885. PARSE_ERROR();
  1886. SWITCH_TO_AND_EMIT_CHARACTER(0xFFFD, ScriptDataEscaped);
  1887. }
  1888. ON_EOF
  1889. {
  1890. PARSE_ERROR();
  1891. EMIT_EOF;
  1892. }
  1893. ANYTHING_ELSE
  1894. {
  1895. SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped);
  1896. }
  1897. }
  1898. END_STATE
  1899. BEGIN_STATE(ScriptDataEscapedLessThanSign)
  1900. {
  1901. ON('/')
  1902. {
  1903. m_temporary_buffer.clear();
  1904. SWITCH_TO(ScriptDataEscapedEndTagOpen);
  1905. }
  1906. ON_ASCII_ALPHA
  1907. {
  1908. m_temporary_buffer.clear();
  1909. EMIT_CHARACTER_AND_RECONSUME_IN('<', ScriptDataDoubleEscapeStart);
  1910. }
  1911. ANYTHING_ELSE
  1912. {
  1913. EMIT_CHARACTER_AND_RECONSUME_IN('<', ScriptDataEscaped);
  1914. }
  1915. }
  1916. END_STATE
  1917. BEGIN_STATE(ScriptDataEscapedEndTagOpen)
  1918. {
  1919. ON_ASCII_ALPHA
  1920. {
  1921. create_new_token(HTMLToken::Type::EndTag);
  1922. RECONSUME_IN(ScriptDataEscapedEndTagName);
  1923. }
  1924. ANYTHING_ELSE
  1925. {
  1926. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  1927. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  1928. RECONSUME_IN(ScriptDataEscaped);
  1929. }
  1930. }
  1931. END_STATE
  1932. BEGIN_STATE(ScriptDataEscapedEndTagName)
  1933. {
  1934. ON_WHITESPACE
  1935. {
  1936. if (current_end_tag_token_is_appropriate())
  1937. SWITCH_TO(BeforeAttributeName);
  1938. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  1939. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  1940. for (auto codepoint : m_temporary_buffer) {
  1941. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint));
  1942. }
  1943. RECONSUME_IN(ScriptDataEscaped);
  1944. }
  1945. ON('/')
  1946. {
  1947. if (current_end_tag_token_is_appropriate())
  1948. SWITCH_TO(SelfClosingStartTag);
  1949. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  1950. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  1951. for (auto codepoint : m_temporary_buffer) {
  1952. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint));
  1953. }
  1954. RECONSUME_IN(ScriptDataEscaped);
  1955. }
  1956. ON('>')
  1957. {
  1958. if (current_end_tag_token_is_appropriate())
  1959. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  1960. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  1961. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  1962. for (auto codepoint : m_temporary_buffer) {
  1963. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint));
  1964. }
  1965. RECONSUME_IN(ScriptDataEscaped);
  1966. }
  1967. ON_ASCII_UPPER_ALPHA
  1968. {
  1969. m_current_token.m_tag.tag_name.append(tolower(current_input_character.value()));
  1970. m_temporary_buffer.append(current_input_character.value());
  1971. continue;
  1972. }
  1973. ON_ASCII_LOWER_ALPHA
  1974. {
  1975. m_current_token.m_tag.tag_name.append(current_input_character.value());
  1976. m_temporary_buffer.append(current_input_character.value());
  1977. continue;
  1978. }
  1979. ANYTHING_ELSE
  1980. {
  1981. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  1982. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  1983. for (auto codepoint : m_temporary_buffer) {
  1984. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint));
  1985. }
  1986. RECONSUME_IN(ScriptDataEscaped);
  1987. }
  1988. }
  1989. END_STATE
  1990. BEGIN_STATE(ScriptDataDoubleEscapeStart)
  1991. {
  1992. auto temporary_buffer_equal_to_script = [this]() -> bool {
  1993. if (m_temporary_buffer.size() != 6)
  1994. return false;
  1995. // FIXME: Is there a better way of doing this?
  1996. return m_temporary_buffer[0] == 's' &&
  1997. m_temporary_buffer[1] == 'c' &&
  1998. m_temporary_buffer[2] == 'r' &&
  1999. m_temporary_buffer[3] == 'i' &&
  2000. m_temporary_buffer[4] == 'p' &&
  2001. m_temporary_buffer[5] == 't';
  2002. };
  2003. ON_WHITESPACE
  2004. {
  2005. if (temporary_buffer_equal_to_script())
  2006. SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped);
  2007. else
  2008. SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped);
  2009. }
  2010. ON('/')
  2011. {
  2012. if (temporary_buffer_equal_to_script())
  2013. SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped);
  2014. else
  2015. SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped);
  2016. }
  2017. ON('>')
  2018. {
  2019. if (temporary_buffer_equal_to_script())
  2020. SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped);
  2021. else
  2022. SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped);
  2023. }
  2024. ON_ASCII_UPPER_ALPHA
  2025. {
  2026. m_temporary_buffer.append(tolower(current_input_character.value()));
  2027. EMIT_CURRENT_CHARACTER;
  2028. }
  2029. ON_ASCII_LOWER_ALPHA
  2030. {
  2031. m_temporary_buffer.append(current_input_character.value());
  2032. EMIT_CURRENT_CHARACTER;
  2033. }
  2034. ANYTHING_ELSE
  2035. {
  2036. RECONSUME_IN(ScriptDataEscaped);
  2037. }
  2038. }
  2039. END_STATE
  2040. BEGIN_STATE(ScriptDataDoubleEscaped)
  2041. {
  2042. ON('-')
  2043. {
  2044. SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataDoubleEscapedDash);
  2045. }
  2046. ON('<')
  2047. {
  2048. SWITCH_TO_AND_EMIT_CHARACTER('<', ScriptDataDoubleEscapedLessThanSign);
  2049. }
  2050. ON(0)
  2051. {
  2052. PARSE_ERROR();
  2053. EMIT_CHARACTER(0xFFFD);
  2054. }
  2055. ON_EOF
  2056. {
  2057. PARSE_ERROR();
  2058. EMIT_EOF;
  2059. }
  2060. ANYTHING_ELSE
  2061. {
  2062. EMIT_CURRENT_CHARACTER;
  2063. }
  2064. }
  2065. END_STATE
  2066. BEGIN_STATE(ScriptDataDoubleEscapedDash)
  2067. {
  2068. ON('-')
  2069. {
  2070. SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataDoubleEscapedDashDash);
  2071. }
  2072. ON('<')
  2073. {
  2074. SWITCH_TO_AND_EMIT_CHARACTER('<', ScriptDataDoubleEscapedLessThanSign);
  2075. }
  2076. ON(0)
  2077. {
  2078. PARSE_ERROR();
  2079. SWITCH_TO_AND_EMIT_CHARACTER(0xFFFD, ScriptDataDoubleEscaped);
  2080. }
  2081. ON_EOF
  2082. {
  2083. PARSE_ERROR();
  2084. EMIT_EOF;
  2085. }
  2086. ANYTHING_ELSE
  2087. {
  2088. SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped);
  2089. }
  2090. }
  2091. END_STATE
  2092. BEGIN_STATE(ScriptDataDoubleEscapedDashDash)
  2093. {
  2094. ON('-')
  2095. {
  2096. EMIT_CHARACTER('-');
  2097. }
  2098. ON('<')
  2099. {
  2100. SWITCH_TO_AND_EMIT_CHARACTER('<', ScriptDataDoubleEscapedLessThanSign);
  2101. }
  2102. ON('>')
  2103. {
  2104. SWITCH_TO_AND_EMIT_CHARACTER('>', ScriptData);
  2105. }
  2106. ON(0)
  2107. {
  2108. PARSE_ERROR();
  2109. SWITCH_TO_AND_EMIT_CHARACTER(0xFFFD, ScriptDataDoubleEscaped);
  2110. }
  2111. ON_EOF
  2112. {
  2113. PARSE_ERROR();
  2114. EMIT_EOF;
  2115. }
  2116. ANYTHING_ELSE
  2117. {
  2118. SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped);
  2119. }
  2120. }
  2121. END_STATE
  2122. BEGIN_STATE(ScriptDataDoubleEscapedLessThanSign)
  2123. {
  2124. ON('/')
  2125. {
  2126. m_temporary_buffer.clear();
  2127. SWITCH_TO_AND_EMIT_CHARACTER('/', ScriptDataDoubleEscapeEnd);
  2128. }
  2129. ANYTHING_ELSE
  2130. {
  2131. RECONSUME_IN(ScriptDataDoubleEscaped);
  2132. }
  2133. }
  2134. END_STATE
  2135. BEGIN_STATE(ScriptDataDoubleEscapeEnd)
  2136. {
  2137. auto temporary_buffer_equal_to_script = [this]() -> bool {
  2138. if (m_temporary_buffer.size() != 6)
  2139. return false;
  2140. // FIXME: Is there a better way of doing this?
  2141. return m_temporary_buffer[0] == 's' &&
  2142. m_temporary_buffer[1] == 'c' &&
  2143. m_temporary_buffer[2] == 'r' &&
  2144. m_temporary_buffer[3] == 'i' &&
  2145. m_temporary_buffer[4] == 'p' &&
  2146. m_temporary_buffer[5] == 't';
  2147. };
  2148. ON_WHITESPACE
  2149. {
  2150. if (temporary_buffer_equal_to_script())
  2151. SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped);
  2152. else
  2153. SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped);
  2154. }
  2155. ON('/')
  2156. {
  2157. if (temporary_buffer_equal_to_script())
  2158. SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped);
  2159. else
  2160. SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped);
  2161. }
  2162. ON('>')
  2163. {
  2164. if (temporary_buffer_equal_to_script())
  2165. SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped);
  2166. else
  2167. SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataDoubleEscaped);
  2168. }
  2169. ON_ASCII_UPPER_ALPHA
  2170. {
  2171. m_temporary_buffer.append(tolower(current_input_character.value()));
  2172. EMIT_CURRENT_CHARACTER;
  2173. }
  2174. ON_ASCII_LOWER_ALPHA
  2175. {
  2176. m_temporary_buffer.append(current_input_character.value());
  2177. EMIT_CURRENT_CHARACTER;
  2178. }
  2179. ANYTHING_ELSE
  2180. {
  2181. RECONSUME_IN(ScriptDataDoubleEscaped);
  2182. }
  2183. }
  2184. END_STATE
  2185. BEGIN_STATE(ScriptDataEscapedDash)
  2186. {
  2187. ON('-')
  2188. {
  2189. SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataEscapedDashDash);
  2190. }
  2191. ON('<')
  2192. {
  2193. SWITCH_TO(ScriptDataEscapedLessThanSign);
  2194. }
  2195. ON(0)
  2196. {
  2197. PARSE_ERROR();
  2198. SWITCH_TO_AND_EMIT_CHARACTER(0xFFFD, ScriptDataEscaped);
  2199. }
  2200. ON_EOF
  2201. {
  2202. PARSE_ERROR();
  2203. EMIT_EOF;
  2204. }
  2205. ANYTHING_ELSE
  2206. {
  2207. SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(ScriptDataEscaped);
  2208. }
  2209. }
  2210. END_STATE
  2211. BEGIN_STATE(ScriptDataEscaped)
  2212. {
  2213. ON('-')
  2214. {
  2215. SWITCH_TO_AND_EMIT_CHARACTER('-', ScriptDataEscapedDash);
  2216. }
  2217. ON('<')
  2218. {
  2219. SWITCH_TO(ScriptDataEscapedLessThanSign);
  2220. }
  2221. ON(0)
  2222. {
  2223. PARSE_ERROR();
  2224. EMIT_CHARACTER(0xFFFD);
  2225. }
  2226. ON_EOF
  2227. {
  2228. PARSE_ERROR();
  2229. EMIT_EOF;
  2230. }
  2231. ANYTHING_ELSE
  2232. {
  2233. EMIT_CURRENT_CHARACTER;
  2234. }
  2235. }
  2236. END_STATE
  2237. BEGIN_STATE(ScriptDataEndTagOpen)
  2238. {
  2239. ON_ASCII_ALPHA
  2240. {
  2241. create_new_token(HTMLToken::Type::EndTag);
  2242. RECONSUME_IN(ScriptDataEndTagName);
  2243. }
  2244. ANYTHING_ELSE
  2245. {
  2246. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  2247. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  2248. RECONSUME_IN(ScriptData);
  2249. }
  2250. }
  2251. END_STATE
  2252. BEGIN_STATE(ScriptDataEndTagName)
  2253. {
  2254. ON_WHITESPACE
  2255. {
  2256. if (current_end_tag_token_is_appropriate())
  2257. SWITCH_TO(BeforeAttributeName);
  2258. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  2259. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  2260. for (auto codepoint : m_temporary_buffer)
  2261. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint));
  2262. RECONSUME_IN(ScriptData);
  2263. }
  2264. ON('/')
  2265. {
  2266. if (current_end_tag_token_is_appropriate())
  2267. SWITCH_TO(SelfClosingStartTag);
  2268. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  2269. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  2270. for (auto codepoint : m_temporary_buffer)
  2271. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint));
  2272. RECONSUME_IN(ScriptData);
  2273. }
  2274. ON('>')
  2275. {
  2276. if (current_end_tag_token_is_appropriate())
  2277. SWITCH_TO_AND_EMIT_CURRENT_TOKEN(Data);
  2278. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  2279. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  2280. for (auto codepoint : m_temporary_buffer)
  2281. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint));
  2282. RECONSUME_IN(ScriptData);
  2283. }
  2284. ON_ASCII_UPPER_ALPHA
  2285. {
  2286. m_current_token.m_tag.tag_name.append(tolower(current_input_character.value()));
  2287. m_temporary_buffer.append(current_input_character.value());
  2288. continue;
  2289. }
  2290. ON_ASCII_LOWER_ALPHA
  2291. {
  2292. m_current_token.m_tag.tag_name.append(current_input_character.value());
  2293. m_temporary_buffer.append(current_input_character.value());
  2294. continue;
  2295. }
  2296. ANYTHING_ELSE
  2297. {
  2298. m_queued_tokens.enqueue(HTMLToken::make_character('<'));
  2299. m_queued_tokens.enqueue(HTMLToken::make_character('/'));
  2300. for (auto codepoint : m_temporary_buffer)
  2301. m_queued_tokens.enqueue(HTMLToken::make_character(codepoint));
  2302. RECONSUME_IN(ScriptData);
  2303. }
  2304. }
  2305. END_STATE
  2306. BEGIN_STATE(CDATASection)
  2307. {
  2308. ON(']')
  2309. {
  2310. SWITCH_TO(CDATASectionBracket);
  2311. }
  2312. ON_EOF
  2313. {
  2314. PARSE_ERROR();
  2315. EMIT_EOF;
  2316. }
  2317. ANYTHING_ELSE
  2318. {
  2319. EMIT_CURRENT_CHARACTER;
  2320. }
  2321. }
  2322. END_STATE
  2323. BEGIN_STATE(CDATASectionBracket)
  2324. {
  2325. ON(']')
  2326. {
  2327. SWITCH_TO(CDATASectionEnd);
  2328. }
  2329. ANYTHING_ELSE
  2330. {
  2331. EMIT_CHARACTER_AND_RECONSUME_IN(']', CDATASection);
  2332. }
  2333. }
  2334. END_STATE
  2335. BEGIN_STATE(CDATASectionEnd)
  2336. {
  2337. ON(']')
  2338. {
  2339. EMIT_CHARACTER(']');
  2340. }
  2341. ON('>')
  2342. {
  2343. SWITCH_TO(Data);
  2344. }
  2345. ANYTHING_ELSE
  2346. {
  2347. m_queued_tokens.enqueue(HTMLToken::make_character(']'));
  2348. m_queued_tokens.enqueue(HTMLToken::make_character(']'));
  2349. RECONSUME_IN(CDATASection);
  2350. }
  2351. }
  2352. END_STATE
  2353. default:
  2354. TODO();
  2355. }
  2356. }
  2357. }
  2358. bool HTMLTokenizer::consume_next_if_match(const StringView& string, CaseSensitivity case_sensitivity)
  2359. {
  2360. for (size_t i = 0; i < string.length(); ++i) {
  2361. auto codepoint = peek_codepoint(i);
  2362. if (!codepoint.has_value())
  2363. return false;
  2364. // FIXME: This should be more Unicode-aware.
  2365. if (case_sensitivity == CaseSensitivity::CaseInsensitive) {
  2366. if (codepoint.value() < 0x80) {
  2367. if (tolower(codepoint.value()) != tolower(string[i]))
  2368. return false;
  2369. continue;
  2370. }
  2371. }
  2372. if (codepoint.value() != (u32)string[i])
  2373. return false;
  2374. }
  2375. for (size_t i = 0; i < string.length(); ++i) {
  2376. m_prev_utf8_iterator = m_utf8_iterator;
  2377. ++m_utf8_iterator;
  2378. }
  2379. return true;
  2380. }
  2381. void HTMLTokenizer::create_new_token(HTMLToken::Type type)
  2382. {
  2383. m_current_token = {};
  2384. m_current_token.m_type = type;
  2385. }
  2386. HTMLTokenizer::HTMLTokenizer(const StringView& input, const String& encoding)
  2387. {
  2388. auto* decoder = TextCodec::decoder_for(encoding);
  2389. ASSERT(decoder);
  2390. m_decoded_input = decoder->to_utf8(input);
  2391. m_utf8_view = Utf8View(m_decoded_input);
  2392. m_utf8_iterator = m_utf8_view.begin();
  2393. }
  2394. void HTMLTokenizer::will_switch_to([[maybe_unused]] State new_state)
  2395. {
  2396. #ifdef TOKENIZER_TRACE
  2397. dbg() << "[" << state_name(m_state) << "] Switch to " << state_name(new_state);
  2398. #endif
  2399. }
  2400. void HTMLTokenizer::will_reconsume_in([[maybe_unused]] State new_state)
  2401. {
  2402. #ifdef TOKENIZER_TRACE
  2403. dbg() << "[" << state_name(m_state) << "] Reconsume in " << state_name(new_state);
  2404. #endif
  2405. }
  2406. void HTMLTokenizer::switch_to(Badge<HTMLDocumentParser>, State new_state)
  2407. {
  2408. #ifdef TOKENIZER_TRACE
  2409. dbg() << "[" << state_name(m_state) << "] Parser switches tokenizer state to " << state_name(new_state);
  2410. #endif
  2411. m_state = new_state;
  2412. }
  2413. void HTMLTokenizer::will_emit(HTMLToken& token)
  2414. {
  2415. if (token.is_start_tag())
  2416. m_last_emitted_start_tag = token;
  2417. }
  2418. bool HTMLTokenizer::current_end_tag_token_is_appropriate() const
  2419. {
  2420. ASSERT(m_current_token.is_end_tag());
  2421. if (!m_last_emitted_start_tag.is_start_tag())
  2422. return false;
  2423. return m_current_token.tag_name() == m_last_emitted_start_tag.tag_name();
  2424. }
  2425. bool HTMLTokenizer::consumed_as_part_of_an_attribute() const
  2426. {
  2427. return m_return_state == State::AttributeValueUnquoted || m_return_state == State::AttributeValueSingleQuoted || m_return_state == State::AttributeValueDoubleQuoted;
  2428. }
  2429. }