Decoder.cpp 87 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780
  1. /*
  2. * Copyright (c) 2020, Andreas Kling <kling@serenityos.org>
  3. * Copyright (c) 2022, Jelle Raaijmakers <jelle@gmta.nl>
  4. * Copyright (c) 2023, Sam Atkins <atkinssj@serenityos.org>
  5. * Copyright (c) 2024, Simon Wanner <simon@skyrising.xyz>
  6. *
  7. * SPDX-License-Identifier: BSD-2-Clause
  8. */
  9. #include <AK/BinarySearch.h>
  10. #include <AK/StringBuilder.h>
  11. #include <AK/Utf16View.h>
  12. #include <AK/Utf8View.h>
  13. #include <LibTextCodec/Decoder.h>
  14. #include <LibTextCodec/LookupTables.h>
  15. namespace TextCodec {
  16. static constexpr u32 replacement_code_point = 0xfffd;
  17. namespace {
  18. Latin1Decoder s_latin1_decoder;
  19. UTF8Decoder s_utf8_decoder;
  20. UTF16BEDecoder s_utf16be_decoder;
  21. UTF16LEDecoder s_utf16le_decoder;
  22. Latin2Decoder s_latin2_decoder;
  23. Latin9Decoder s_latin9_decoder;
  24. PDFDocEncodingDecoder s_pdf_doc_encoding_decoder;
  25. TurkishDecoder s_turkish_decoder;
  26. XUserDefinedDecoder s_x_user_defined_decoder;
  27. GB18030Decoder s_gb18030_decoder;
  28. Big5Decoder s_big5_decoder;
  29. EUCJPDecoder s_euc_jp_decoder;
  30. ISO2022JPDecoder s_iso_2022_jp_decoder;
  31. ShiftJISDecoder s_shift_jis_decoder;
  32. EUCKRDecoder s_euc_kr_decoder;
  33. // clang-format off
  34. // https://encoding.spec.whatwg.org/index-ibm866.txt
  35. SingleByteDecoder s_ibm866_decoder {{
  36. 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F,
  37. 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F,
  38. 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F,
  39. 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, 0x255D, 0x255C, 0x255B, 0x2510,
  40. 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x255E, 0x255F, 0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x2567,
  41. 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256B, 0x256A, 0x2518, 0x250C, 0x2588, 0x2584, 0x258C, 0x2590, 0x2580,
  42. 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F,
  43. 0x0401, 0x0451, 0x0404, 0x0454, 0x0407, 0x0457, 0x040E, 0x045E, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x2116, 0x00A4, 0x25A0, 0x00A0,
  44. }};
  45. // https://encoding.spec.whatwg.org/index-iso-8859-3.txt
  46. SingleByteDecoder s_latin3_decoder {{
  47. 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
  48. 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
  49. 0x00A0, 0x0126, 0x02D8, 0x00A3, 0x00A4, 0xFFFD, 0x0124, 0x00A7, 0x00A8, 0x0130, 0x015E, 0x011E, 0x0134, 0x00AD, 0xFFFD, 0x017B,
  50. 0x00B0, 0x0127, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x0125, 0x00B7, 0x00B8, 0x0131, 0x015F, 0x011F, 0x0135, 0x00BD, 0xFFFD, 0x017C,
  51. 0x00C0, 0x00C1, 0x00C2, 0xFFFD, 0x00C4, 0x010A, 0x0108, 0x00C7, 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
  52. 0xFFFD, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x0120, 0x00D6, 0x00D7, 0x011C, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x016C, 0x015C, 0x00DF,
  53. 0x00E0, 0x00E1, 0x00E2, 0xFFFD, 0x00E4, 0x010B, 0x0109, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
  54. 0xFFFD, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x0121, 0x00F6, 0x00F7, 0x011D, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x016D, 0x015D, 0x02D9,
  55. }};
  56. // https://encoding.spec.whatwg.org/index-iso-8859-4.txt
  57. SingleByteDecoder s_latin4_decoder {{
  58. 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
  59. 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
  60. 0x00A0, 0x0104, 0x0138, 0x0156, 0x00A4, 0x0128, 0x013B, 0x00A7, 0x00A8, 0x0160, 0x0112, 0x0122, 0x0166, 0x00AD, 0x017D, 0x00AF,
  61. 0x00B0, 0x0105, 0x02DB, 0x0157, 0x00B4, 0x0129, 0x013C, 0x02C7, 0x00B8, 0x0161, 0x0113, 0x0123, 0x0167, 0x014A, 0x017E, 0x014B,
  62. 0x0100, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x012E, 0x010C, 0x00C9, 0x0118, 0x00CB, 0x0116, 0x00CD, 0x00CE, 0x012A,
  63. 0x0110, 0x0145, 0x014C, 0x0136, 0x00D4, 0x00D5, 0x00D6, 0x00D7, 0x00D8, 0x0172, 0x00DA, 0x00DB, 0x00DC, 0x0168, 0x016A, 0x00DF,
  64. 0x0101, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x012F, 0x010D, 0x00E9, 0x0119, 0x00EB, 0x0117, 0x00ED, 0x00EE, 0x012B,
  65. 0x0111, 0x0146, 0x014D, 0x0137, 0x00F4, 0x00F5, 0x00F6, 0x00F7, 0x00F8, 0x0173, 0x00FA, 0x00FB, 0x00FC, 0x0169, 0x016B, 0x02D9,
  66. }};
  67. // https://encoding.spec.whatwg.org/index-iso-8859-5.txt
  68. SingleByteDecoder s_latin_cyrillic_decoder {{
  69. 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
  70. 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
  71. 0x00A0, 0x0401, 0x0402, 0x0403, 0x0404, 0x0405, 0x0406, 0x0407, 0x0408, 0x0409, 0x040A, 0x040B, 0x040C, 0x00AD, 0x040E, 0x040F,
  72. 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F,
  73. 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F,
  74. 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F,
  75. 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F,
  76. 0x2116, 0x0451, 0x0452, 0x0453, 0x0454, 0x0455, 0x0456, 0x0457, 0x0458, 0x0459, 0x045A, 0x045B, 0x045C, 0x00A7, 0x045E, 0x045F,
  77. }};
  78. // https://encoding.spec.whatwg.org/index-iso-8859-6.txt
  79. SingleByteDecoder s_latin_arabic_decoder {{
  80. 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
  81. 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
  82. 0x00A0, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A4, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x060C, 0x00AD, 0xFFFD, 0xFFFD,
  83. 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x061B, 0xFFFD, 0xFFFD, 0xFFFD, 0x061F,
  84. 0xFFFD, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, 0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F,
  85. 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x0637, 0x0638, 0x0639, 0x063A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
  86. 0x0640, 0x0641, 0x0642, 0x0643, 0x0644, 0x0645, 0x0646, 0x0647, 0x0648, 0x0649, 0x064A, 0x064B, 0x064C, 0x064D, 0x064E, 0x064F,
  87. 0x0650, 0x0651, 0x0652, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
  88. }};
  89. // https://encoding.spec.whatwg.org/index-iso-8859-7.txt
  90. SingleByteDecoder s_latin_greek_decoder {{
  91. 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
  92. 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
  93. 0x00A0, 0x2018, 0x2019, 0x00A3, 0x20AC, 0x20AF, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x037A, 0x00AB, 0x00AC, 0x00AD, 0xFFFD, 0x2015,
  94. 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x0384, 0x0385, 0x0386, 0x00B7, 0x0388, 0x0389, 0x038A, 0x00BB, 0x038C, 0x00BD, 0x038E, 0x038F,
  95. 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, 0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F,
  96. 0x03A0, 0x03A1, 0xFFFD, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7, 0x03A8, 0x03A9, 0x03AA, 0x03AB, 0x03AC, 0x03AD, 0x03AE, 0x03AF,
  97. 0x03B0, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, 0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF,
  98. 0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7, 0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CC, 0x03CD, 0x03CE, 0xFFFD,
  99. }};
  100. // https://encoding.spec.whatwg.org/index-iso-8859-8.txt
  101. SingleByteDecoder s_latin_hebrew_decoder {{
  102. 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
  103. 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
  104. 0x00A0, 0xFFFD, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00D7, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF,
  105. 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00F7, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0xFFFD,
  106. 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
  107. 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2017,
  108. 0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7, 0x05D8, 0x05D9, 0x05DA, 0x05DB, 0x05DC, 0x05DD, 0x05DE, 0x05DF,
  109. 0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7, 0x05E8, 0x05E9, 0x05EA, 0xFFFD, 0xFFFD, 0x200E, 0x200F, 0xFFFD,
  110. }};
  111. // https://encoding.spec.whatwg.org/index-iso-8859-10.txt
  112. SingleByteDecoder s_latin6_decoder {{
  113. 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
  114. 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
  115. 0x00A0, 0x0104, 0x0112, 0x0122, 0x012A, 0x0128, 0x0136, 0x00A7, 0x013B, 0x0110, 0x0160, 0x0166, 0x017D, 0x00AD, 0x016A, 0x014A,
  116. 0x00B0, 0x0105, 0x0113, 0x0123, 0x012B, 0x0129, 0x0137, 0x00B7, 0x013C, 0x0111, 0x0161, 0x0167, 0x017E, 0x2015, 0x016B, 0x014B,
  117. 0x0100, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x012E, 0x010C, 0x00C9, 0x0118, 0x00CB, 0x0116, 0x00CD, 0x00CE, 0x00CF,
  118. 0x00D0, 0x0145, 0x014C, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x0168, 0x00D8, 0x0172, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF,
  119. 0x0101, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x012F, 0x010D, 0x00E9, 0x0119, 0x00EB, 0x0117, 0x00ED, 0x00EE, 0x00EF,
  120. 0x00F0, 0x0146, 0x014D, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x0169, 0x00F8, 0x0173, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x0138,
  121. }};
  122. // https://encoding.spec.whatwg.org/index-iso-8859-13.txt
  123. SingleByteDecoder s_latin7_decoder {{
  124. 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
  125. 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
  126. 0x00A0, 0x201D, 0x00A2, 0x00A3, 0x00A4, 0x201E, 0x00A6, 0x00A7, 0x00D8, 0x00A9, 0x0156, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00C6,
  127. 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x201C, 0x00B5, 0x00B6, 0x00B7, 0x00F8, 0x00B9, 0x0157, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00E6,
  128. 0x0104, 0x012E, 0x0100, 0x0106, 0x00C4, 0x00C5, 0x0118, 0x0112, 0x010C, 0x00C9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012A, 0x013B,
  129. 0x0160, 0x0143, 0x0145, 0x00D3, 0x014C, 0x00D5, 0x00D6, 0x00D7, 0x0172, 0x0141, 0x015A, 0x016A, 0x00DC, 0x017B, 0x017D, 0x00DF,
  130. 0x0105, 0x012F, 0x0101, 0x0107, 0x00E4, 0x00E5, 0x0119, 0x0113, 0x010D, 0x00E9, 0x017A, 0x0117, 0x0123, 0x0137, 0x012B, 0x013C,
  131. 0x0161, 0x0144, 0x0146, 0x00F3, 0x014D, 0x00F5, 0x00F6, 0x00F7, 0x0173, 0x0142, 0x015B, 0x016B, 0x00FC, 0x017C, 0x017E, 0x2019,
  132. }};
  133. // https://encoding.spec.whatwg.org/index-iso-8859-14.txt
  134. SingleByteDecoder s_latin8_decoder {{
  135. 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
  136. 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
  137. 0x00A0, 0x1E02, 0x1E03, 0x00A3, 0x010A, 0x010B, 0x1E0A, 0x00A7, 0x1E80, 0x00A9, 0x1E82, 0x1E0B, 0x1EF2, 0x00AD, 0x00AE, 0x0178,
  138. 0x1E1E, 0x1E1F, 0x0120, 0x0121, 0x1E40, 0x1E41, 0x00B6, 0x1E56, 0x1E81, 0x1E57, 0x1E83, 0x1E60, 0x1EF3, 0x1E84, 0x1E85, 0x1E61,
  139. 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7, 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
  140. 0x0174, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x1E6A, 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x0176, 0x00DF,
  141. 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
  142. 0x0175, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x1E6B, 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x0177, 0x00FF,
  143. }};
  144. // https://encoding.spec.whatwg.org/index-iso-8859-16.txt
  145. SingleByteDecoder s_latin10_decoder {{
  146. 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
  147. 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
  148. 0x00A0, 0x0104, 0x0105, 0x0141, 0x20AC, 0x201E, 0x0160, 0x00A7, 0x0161, 0x00A9, 0x0218, 0x00AB, 0x0179, 0x00AD, 0x017A, 0x017B,
  149. 0x00B0, 0x00B1, 0x010C, 0x0142, 0x017D, 0x201D, 0x00B6, 0x00B7, 0x017E, 0x010D, 0x0219, 0x00BB, 0x0152, 0x0153, 0x0178, 0x017C,
  150. 0x00C0, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0106, 0x00C6, 0x00C7, 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
  151. 0x0110, 0x0143, 0x00D2, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x015A, 0x0170, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x0118, 0x021A, 0x00DF,
  152. 0x00E0, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x0107, 0x00E6, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
  153. 0x0111, 0x0144, 0x00F2, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x015B, 0x0171, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x0119, 0x021B, 0x00FF,
  154. }};
  155. // https://encoding.spec.whatwg.org/index-windows-1250.txt
  156. SingleByteDecoder s_centraleurope_decoder {{
  157. 0x20AC, 0x0081, 0x201A, 0x0083, 0x201E, 0x2026, 0x2020, 0x2021, 0x0088, 0x2030, 0x0160, 0x2039, 0x015A, 0x0164, 0x017D, 0x0179,
  158. 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x0098, 0x2122, 0x0161, 0x203A, 0x015B, 0x0165, 0x017E, 0x017A,
  159. 0x00A0, 0x02C7, 0x02D8, 0x0141, 0x00A4, 0x0104, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x015E, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x017B,
  160. 0x00B0, 0x00B1, 0x02DB, 0x0142, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00B8, 0x0105, 0x015F, 0x00BB, 0x013D, 0x02DD, 0x013E, 0x017C,
  161. 0x0154, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0139, 0x0106, 0x00C7, 0x010C, 0x00C9, 0x0118, 0x00CB, 0x011A, 0x00CD, 0x00CE, 0x010E,
  162. 0x0110, 0x0143, 0x0147, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x00D7, 0x0158, 0x016E, 0x00DA, 0x0170, 0x00DC, 0x00DD, 0x0162, 0x00DF,
  163. 0x0155, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x013A, 0x0107, 0x00E7, 0x010D, 0x00E9, 0x0119, 0x00EB, 0x011B, 0x00ED, 0x00EE, 0x010F,
  164. 0x0111, 0x0144, 0x0148, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x00F7, 0x0159, 0x016F, 0x00FA, 0x0171, 0x00FC, 0x00FD, 0x0163, 0x02D9,
  165. }};
  166. // https://encoding.spec.whatwg.org/index-windows-1251.txt
  167. SingleByteDecoder s_cyrillic_decoder {{
  168. 0x0402, 0x0403, 0x201A, 0x0453, 0x201E, 0x2026, 0x2020, 0x2021, 0x20AC, 0x2030, 0x0409, 0x2039, 0x040A, 0x040C, 0x040B, 0x040F,
  169. 0x0452, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x0098, 0x2122, 0x0459, 0x203A, 0x045A, 0x045C, 0x045B, 0x045F,
  170. 0x00A0, 0x040E, 0x045E, 0x0408, 0x00A4, 0x0490, 0x00A6, 0x00A7, 0x0401, 0x00A9, 0x0404, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x0407,
  171. 0x00B0, 0x00B1, 0x0406, 0x0456, 0x0491, 0x00B5, 0x00B6, 0x00B7, 0x0451, 0x2116, 0x0454, 0x00BB, 0x0458, 0x0405, 0x0455, 0x0457,
  172. 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F,
  173. 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F,
  174. 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F,
  175. 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F,
  176. }};
  177. // https://encoding.spec.whatwg.org/index-windows-1255.txt
  178. SingleByteDecoder s_hebrew_decoder {{
  179. 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x008A, 0x2039, 0x008C, 0x008D, 0x008E, 0x008F,
  180. 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x009A, 0x203A, 0x009C, 0x009D, 0x009E, 0x009F,
  181. 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x20AA, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00D7, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF,
  182. 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00F7, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF,
  183. 0x05B0, 0x05B1, 0x05B2, 0x05B3, 0x05B4, 0x05B5, 0x05B6, 0x05B7, 0x05B8, 0x05B9, 0x05BA, 0x05BB, 0x05BC, 0x05BD, 0x05BE, 0x05BF,
  184. 0x05C0, 0x05C1, 0x05C2, 0x05C3, 0x05F0, 0x05F1, 0x05F2, 0x05F3, 0x05F4, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
  185. 0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7, 0x05D8, 0x05D9, 0x05DA, 0x05DB, 0x05DC, 0x05DD, 0x05DE, 0x05DF,
  186. 0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7, 0x05E8, 0x05E9, 0x05EA, 0xFFFD, 0xFFFD, 0x200E, 0x200F, 0xFFFD,
  187. }};
  188. // https://encoding.spec.whatwg.org/index-koi8-r.txt
  189. SingleByteDecoder s_koi8r_decoder {{
  190. 0x2500, 0x2502, 0x250C, 0x2510, 0x2514, 0x2518, 0x251C, 0x2524, 0x252C, 0x2534, 0x253C, 0x2580, 0x2584, 0x2588, 0x258C, 0x2590,
  191. 0x2591, 0x2592, 0x2593, 0x2320, 0x25A0, 0x2219, 0x221A, 0x2248, 0x2264, 0x2265, 0x00A0, 0x2321, 0x00B0, 0x00B2, 0x00B7, 0x00F7,
  192. 0x2550, 0x2551, 0x2552, 0xD191, 0x2553, 0x2554, 0x2555, 0x2556, 0x2557, 0x2558, 0x2559, 0x255A, 0x255B, 0x255C, 0x255D, 0x255E,
  193. 0x255F, 0x2560, 0x2561, 0xD081, 0x2562, 0x2563, 0x2564, 0x2565, 0x2566, 0x2567, 0x2568, 0x2569, 0x256A, 0x256B, 0x256C, 0x00A9,
  194. 0x044E, 0x0430, 0x0431, 0x0446, 0x0434, 0x0435, 0x0444, 0x0433, 0x0445, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E,
  195. 0x043F, 0x044F, 0x0440, 0x0441, 0x0442, 0x0443, 0x0436, 0x0432, 0x044C, 0x044B, 0x0437, 0x0448, 0x044D, 0x0449, 0x0447, 0x044A,
  196. 0x042E, 0x0410, 0x0441, 0x0426, 0x0414, 0x0415, 0x0424, 0x0413, 0x0425, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E,
  197. 0x041F, 0x042F, 0x0420, 0x0421, 0x0422, 0x0423, 0x0416, 0x0412, 0x042C, 0x042B, 0x0417, 0x0428, 0x042D, 0x0429, 0x0427, 0x042A,
  198. }};
  199. // https://encoding.spec.whatwg.org/index-koi8-u.txt
  200. SingleByteDecoder s_koi8u_decoder {{
  201. 0x2500, 0x2502, 0x250C, 0x2510, 0x2514, 0x2518, 0x251C, 0x2524, 0x252C, 0x2534, 0x253C, 0x2580, 0x2584, 0x2588, 0x258C, 0x2590,
  202. 0x2591, 0x2592, 0x2593, 0x2320, 0x25A0, 0x2219, 0x221A, 0x2248, 0x2264, 0x2265, 0x00A0, 0x2321, 0x00B0, 0x00B2, 0x00B7, 0x00F7,
  203. 0x2550, 0x2551, 0x2552, 0x0451, 0x0454, 0x2554, 0x0456, 0x0457, 0x2557, 0x2558, 0x2559, 0x255A, 0x255B, 0x0491, 0x045E, 0x255E,
  204. 0x255F, 0x2560, 0x2561, 0x0401, 0x0404, 0x2563, 0x0406, 0x0407, 0x2566, 0x2567, 0x2568, 0x2569, 0x256A, 0x0490, 0x040E, 0x00A9,
  205. 0x044E, 0x0430, 0x0431, 0x0446, 0x0434, 0x0435, 0x0444, 0x0433, 0x0445, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E,
  206. 0x043F, 0x044F, 0x0440, 0x0441, 0x0442, 0x0443, 0x0436, 0x0432, 0x044C, 0x044B, 0x0437, 0x0448, 0x044D, 0x0449, 0x0447, 0x044A,
  207. 0x042E, 0x0410, 0x0411, 0x0426, 0x0414, 0x0415, 0x0424, 0x0413, 0x0425, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E,
  208. 0x041F, 0x042F, 0x0420, 0x0421, 0x0422, 0x0423, 0x0416, 0x0412, 0x042C, 0x042B, 0x0417, 0x0428, 0x042D, 0x0429, 0x0427, 0x042A,
  209. }};
  210. // https://encoding.spec.whatwg.org/index-macintosh.txt
  211. SingleByteDecoder s_mac_roman_decoder {{
  212. 0x00C4, 0x00C5, 0x00C7, 0x00C9, 0x00D1, 0x00D6, 0x00DC, 0x00E1, 0x00E0, 0x00E2, 0x00E4, 0x00E3, 0x00E5, 0x00E7, 0x00E9, 0x00E8,
  213. 0x00EA, 0x00EB, 0x00ED, 0x00EC, 0x00EE, 0x00EF, 0x00F1, 0x00F3, 0x00F2, 0x00F4, 0x00F6, 0x00F5, 0x00FA, 0x00F9, 0x00FB, 0x00FC,
  214. 0x2020, 0x00B0, 0x00A2, 0x00A3, 0x00A7, 0x2022, 0x00B6, 0x00DF, 0x00AE, 0x00A9, 0x2122, 0x00B4, 0x00A8, 0x2260, 0x00C6, 0x00D8,
  215. 0x221E, 0x00B1, 0x2264, 0x2265, 0x00A5, 0x00B5, 0x2202, 0x2211, 0x220F, 0x03C0, 0x222B, 0x00AA, 0x00BA, 0x03A9, 0x00E6, 0x00F8,
  216. 0x00BF, 0x00A1, 0x00AC, 0x221A, 0x0192, 0x2248, 0x2206, 0x00AB, 0x00BB, 0x2026, 0x00A0, 0x00C0, 0x00C3, 0x00D5, 0x0152, 0x0153,
  217. 0x2013, 0x2014, 0x201C, 0x201D, 0x2018, 0x2019, 0x00F7, 0x25CA, 0x00FF, 0x0178, 0x2044, 0x20AC, 0x2039, 0x203A, 0xFB01, 0xFB02,
  218. 0x2021, 0x00B7, 0x201A, 0x201E, 0x2030, 0x00C2, 0x00CA, 0x00C1, 0x00CB, 0x00C8, 0x00CD, 0x00CE, 0x00CF, 0x00CC, 0x00D3, 0x00D4,
  219. 0xF8FF, 0x00D2, 0x00DA, 0x00DB, 0x00D9, 0x0131, 0x02C6, 0x02DC, 0x00AF, 0x02D8, 0x02D9, 0x02DA, 0x00B8, 0x02DD, 0x02DB, 0x02C7,
  220. }};
  221. // https://encoding.spec.whatwg.org/index-windows-874.txt
  222. SingleByteDecoder s_windows874_decoder {{
  223. 0x20AC, 0x0081, 0x0082, 0x0083, 0x0084, 0x2026, 0x0086, 0x0087, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
  224. 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
  225. 0x00A0, 0x0E01, 0x0E02, 0x0E03, 0x0E04, 0x0E05, 0x0E06, 0x0E07, 0x0E08, 0x0E09, 0x0E0A, 0x0E0B, 0x0E0C, 0x0E0D, 0x0E0E, 0x0E0F,
  226. 0x0E10, 0x0E11, 0x0E12, 0x0E13, 0x0E14, 0x0E15, 0x0E16, 0x0E17, 0x0E18, 0x0E19, 0x0E1A, 0x0E1B, 0x0E1C, 0x0E1D, 0x0E1E, 0x0E1F,
  227. 0x0E20, 0x0E21, 0x0E22, 0x0E23, 0x0E24, 0x0E25, 0x0E26, 0x0E27, 0x0E28, 0x0E29, 0x0E2A, 0x0E2B, 0x0E2C, 0x0E2D, 0x0E2E, 0x0E2F,
  228. 0x0E30, 0x0E31, 0x0E32, 0x0E33, 0x0E34, 0x0E35, 0x0E36, 0x0E37, 0x0E38, 0x0E39, 0x0E3A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x0E3F,
  229. 0x0E40, 0x0E41, 0x0E42, 0x0E43, 0x0E44, 0x0E45, 0x0E46, 0x0E47, 0x0E48, 0x0E49, 0x0E4A, 0x0E4B, 0x0E4C, 0x0E4D, 0x0E4E, 0x0E4F,
  230. 0x0E50, 0x0E51, 0x0E52, 0x0E53, 0x0E54, 0x0E55, 0x0E56, 0x0E57, 0x0E58, 0x0E59, 0x0E5A, 0x0E5B, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
  231. }};
  232. // https://encoding.spec.whatwg.org/index-windows-1253.txt
  233. SingleByteDecoder s_windows1253_decoder {{
  234. 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x0088, 0x2030, 0x008A, 0x2039, 0x008C, 0x008D, 0x008E, 0x008F,
  235. 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x0098, 0x2122, 0x009A, 0x203A, 0x009C, 0x009D, 0x009E, 0x009F,
  236. 0x00A0, 0x0385, 0x0386, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0xFFFD, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x2015,
  237. 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x0384, 0x00B5, 0x00B6, 0x00B7, 0x0388, 0x0389, 0x038A, 0x00BB, 0x038C, 0x00BD, 0x038E, 0x038F,
  238. 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, 0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F,
  239. 0x03A0, 0x03A1, 0xFFFD, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7, 0x03A8, 0x03A9, 0x03AA, 0x03AB, 0x03AC, 0x03AD, 0x03AE, 0x03AF,
  240. 0x03B0, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, 0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF,
  241. 0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7, 0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CC, 0x03CD, 0x03CE, 0xFFFD,
  242. }};
  243. // https://encoding.spec.whatwg.org/index-windows-1256.txt
  244. SingleByteDecoder s_windows1256_decoder {{
  245. 0x20AC, 0x067E, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688,
  246. 0x06AF, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x06A9, 0x2122, 0x0691, 0x203A, 0x0153, 0x200C, 0x200D, 0x06BA,
  247. 0x00A0, 0x060C, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x06BE, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF,
  248. 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x061B, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x061F,
  249. 0x06C1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, 0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F,
  250. 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00D7, 0x0637, 0x0638, 0x0639, 0x063A, 0x0640, 0x0641, 0x0642, 0x0643,
  251. 0x00E0, 0x0644, 0x00E2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x0649, 0x064A, 0x00EE, 0x00EF,
  252. 0x064B, 0x064C, 0x064D, 0x064E, 0x00F4, 0x064F, 0x0650, 0x00F7, 0x0651, 0x00F9, 0x0652, 0x00FB, 0x00FC, 0x200E, 0x200F, 0x06D2,
  253. }};
  254. // https://encoding.spec.whatwg.org/index-windows-1257.txt
  255. SingleByteDecoder s_windows1257_decoder {{
  256. 0x20AC, 0x0081, 0x201A, 0x0083, 0x201E, 0x2026, 0x2020, 0x2021, 0x0088, 0x2030, 0x008A, 0x2039, 0x008C, 0x00A8, 0x02C7, 0x00B8,
  257. 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x0098, 0x2122, 0x009A, 0x203A, 0x009C, 0x00AF, 0x02DB, 0x009F,
  258. 0x00A0, 0xFFFD, 0x00A2, 0x00A3, 0x00A4, 0xFFFD, 0x00A6, 0x00A7, 0x00D8, 0x00A9, 0x0156, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00C6,
  259. 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00F8, 0x00B9, 0x0157, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00E6,
  260. 0x0104, 0x012E, 0x0100, 0x0106, 0x00C4, 0x00C5, 0x0118, 0x0112, 0x010C, 0x00C9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012A, 0x013B,
  261. 0x0160, 0x0143, 0x0145, 0x00D3, 0x014C, 0x00D5, 0x00D6, 0x00D7, 0x0172, 0x0141, 0x015A, 0x016A, 0x00DC, 0x017B, 0x017D, 0x00DF,
  262. 0x0105, 0x012F, 0x0101, 0x0107, 0x00E4, 0x00E5, 0x0119, 0x0113, 0x010D, 0x00E9, 0x017A, 0x0117, 0x0123, 0x0137, 0x012B, 0x013C,
  263. 0x0161, 0x0144, 0x0146, 0x00F3, 0x014D, 0x00F5, 0x00F6, 0x00F7, 0x0173, 0x0142, 0x015B, 0x016B, 0x00FC, 0x017C, 0x017E, 0x02D9,
  264. }};
  265. // https://encoding.spec.whatwg.org/index-windows-1258.txt
  266. SingleByteDecoder s_windows1258_decoder {{
  267. 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x008A, 0x2039, 0x0152, 0x008D, 0x008E, 0x008F,
  268. 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x009A, 0x203A, 0x0153, 0x009D, 0x009E, 0x0178,
  269. 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF,
  270. 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF,
  271. 0x00C0, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x00C5, 0x00C6, 0x00C7, 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x0300, 0x00CD, 0x00CE, 0x00CF,
  272. 0x0110, 0x00D1, 0x0309, 0x00D3, 0x00D4, 0x01A0, 0x00D6, 0x00D7, 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x01AF, 0x0303, 0x00DF,
  273. 0x00E0, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x00E5, 0x00E6, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x0301, 0x00ED, 0x00EE, 0x00EF,
  274. 0x0111, 0x00F1, 0x0323, 0x00F3, 0x00F4, 0x01A1, 0x00F6, 0x00F7, 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x01B0, 0x20AB, 0x00FF,
  275. }};
  276. // https://encoding.spec.whatwg.org/index-x-mac-cyrillic.txt
  277. SingleByteDecoder s_mac_cyrillic_decoder {{
  278. 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F,
  279. 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F,
  280. 0x2020, 0x00B0, 0x0490, 0x00A3, 0x00A7, 0x2022, 0x00B6, 0x0406, 0x00AE, 0x00A9, 0x2122, 0x0402, 0x0452, 0x2260, 0x0403, 0x0453,
  281. 0x221E, 0x00B1, 0x2264, 0x2265, 0x0456, 0x00B5, 0x0491, 0x0408, 0x0404, 0x0454, 0x0407, 0x0457, 0x0409, 0x0459, 0x040A, 0x045A,
  282. 0x0458, 0x0405, 0x00AC, 0x221A, 0x0192, 0x2248, 0x2206, 0x00AB, 0x00BB, 0x2026, 0x00A0, 0x040B, 0x045B, 0x040C, 0x045C, 0x0455,
  283. 0x2013, 0x2014, 0x201C, 0x201D, 0x2018, 0x2019, 0x00F7, 0x201E, 0x040E, 0x045E, 0x040F, 0x045F, 0x2116, 0x0401, 0x0451, 0x044F,
  284. 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F,
  285. 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x20AC,
  286. }};
  287. // clang-format on
  288. }
  289. Optional<Decoder&> decoder_for(StringView a_encoding)
  290. {
  291. auto encoding = get_standardized_encoding(a_encoding);
  292. if (encoding.has_value()) {
  293. if (encoding.value().equals_ignoring_ascii_case("windows-1252"sv))
  294. return s_latin1_decoder;
  295. if (encoding.value().equals_ignoring_ascii_case("utf-8"sv))
  296. return s_utf8_decoder;
  297. if (encoding.value().equals_ignoring_ascii_case("utf-16be"sv))
  298. return s_utf16be_decoder;
  299. if (encoding.value().equals_ignoring_ascii_case("utf-16le"sv))
  300. return s_utf16le_decoder;
  301. if (encoding.value().equals_ignoring_ascii_case("big5"sv))
  302. return s_big5_decoder;
  303. if (encoding.value().equals_ignoring_ascii_case("euc-jp"sv))
  304. return s_euc_jp_decoder;
  305. if (encoding.value().equals_ignoring_ascii_case("euc-kr"sv))
  306. return s_euc_kr_decoder;
  307. if (encoding.value().equals_ignoring_ascii_case("gbk"sv))
  308. return s_gb18030_decoder;
  309. if (encoding.value().equals_ignoring_ascii_case("gb18030"sv))
  310. return s_gb18030_decoder;
  311. if (encoding.value().equals_ignoring_ascii_case("ibm866"sv))
  312. return s_ibm866_decoder;
  313. if (encoding.value().equals_ignoring_ascii_case("iso-2022-jp"sv))
  314. return s_iso_2022_jp_decoder;
  315. if (encoding.value().equals_ignoring_ascii_case("iso-8859-2"sv))
  316. return s_latin2_decoder;
  317. if (encoding.value().equals_ignoring_ascii_case("iso-8859-3"sv))
  318. return s_latin3_decoder;
  319. if (encoding.value().equals_ignoring_ascii_case("iso-8859-4"sv))
  320. return s_latin4_decoder;
  321. if (encoding.value().equals_ignoring_ascii_case("iso-8859-5"sv))
  322. return s_latin_cyrillic_decoder;
  323. if (encoding.value().equals_ignoring_ascii_case("iso-8859-6"sv))
  324. return s_latin_arabic_decoder;
  325. if (encoding.value().equals_ignoring_ascii_case("iso-8859-7"sv))
  326. return s_latin_greek_decoder;
  327. if (encoding.value().equals_ignoring_ascii_case("iso-8859-8"sv))
  328. return s_latin_hebrew_decoder;
  329. if (encoding.value().equals_ignoring_ascii_case("iso-8859-8-i"sv))
  330. return s_latin_hebrew_decoder;
  331. if (encoding.value().equals_ignoring_ascii_case("iso-8859-10"sv))
  332. return s_latin6_decoder;
  333. if (encoding.value().equals_ignoring_ascii_case("iso-8859-13"sv))
  334. return s_latin7_decoder;
  335. if (encoding.value().equals_ignoring_ascii_case("iso-8859-14"sv))
  336. return s_latin8_decoder;
  337. if (encoding.value().equals_ignoring_ascii_case("iso-8859-15"sv))
  338. return s_latin9_decoder;
  339. if (encoding.value().equals_ignoring_ascii_case("iso-8859-16"sv))
  340. return s_latin10_decoder;
  341. if (encoding.value().equals_ignoring_ascii_case("koi8-r"sv))
  342. return s_koi8r_decoder;
  343. if (encoding.value().equals_ignoring_ascii_case("koi8-u"sv))
  344. return s_koi8u_decoder;
  345. if (encoding.value().equals_ignoring_ascii_case("macintosh"sv))
  346. return s_mac_roman_decoder;
  347. if (encoding.value().equals_ignoring_ascii_case("PDFDocEncoding"sv))
  348. return s_pdf_doc_encoding_decoder;
  349. if (encoding.value().equals_ignoring_ascii_case("shift_jis"sv))
  350. return s_shift_jis_decoder;
  351. if (encoding.value().equals_ignoring_ascii_case("windows-874"sv))
  352. return s_windows874_decoder;
  353. if (encoding.value().equals_ignoring_ascii_case("windows-1250"sv))
  354. return s_centraleurope_decoder;
  355. if (encoding.value().equals_ignoring_ascii_case("windows-1251"sv))
  356. return s_cyrillic_decoder;
  357. if (encoding.value().equals_ignoring_ascii_case("windows-1253"sv))
  358. return s_windows1253_decoder;
  359. if (encoding.value().equals_ignoring_ascii_case("windows-1254"sv))
  360. return s_turkish_decoder;
  361. if (encoding.value().equals_ignoring_ascii_case("windows-1255"sv))
  362. return s_hebrew_decoder;
  363. if (encoding.value().equals_ignoring_ascii_case("windows-1256"sv))
  364. return s_windows1256_decoder;
  365. if (encoding.value().equals_ignoring_ascii_case("windows-1257"sv))
  366. return s_windows1257_decoder;
  367. if (encoding.value().equals_ignoring_ascii_case("windows-1258"sv))
  368. return s_windows1258_decoder;
  369. if (encoding.value().equals_ignoring_ascii_case("x-mac-cyrillic"sv))
  370. return s_mac_cyrillic_decoder;
  371. if (encoding.value().equals_ignoring_ascii_case("x-user-defined"sv))
  372. return s_x_user_defined_decoder;
  373. }
  374. dbgln("TextCodec: No decoder implemented for encoding '{}'", a_encoding);
  375. return {};
  376. }
  377. // https://encoding.spec.whatwg.org/#concept-encoding-get
  378. Optional<StringView> get_standardized_encoding(StringView encoding)
  379. {
  380. encoding = encoding.trim_whitespace();
  381. if (encoding.is_one_of_ignoring_ascii_case("unicode-1-1-utf-8"sv, "unicode11utf8"sv, "unicode20utf8"sv, "utf-8"sv, "utf8"sv, "x-unicode20utf8"sv))
  382. return "UTF-8"sv;
  383. if (encoding.is_one_of_ignoring_ascii_case("866"sv, "cp866"sv, "csibm866"sv, "ibm866"sv))
  384. return "IBM866"sv;
  385. if (encoding.is_one_of_ignoring_ascii_case("csisolatin2"sv, "iso-8859-2"sv, "iso-ir-101"sv, "iso8859-2"sv, "iso88592"sv, "iso_8859-2"sv, "iso_8859-2:1987"sv, "l2"sv, "latin2"sv))
  386. return "ISO-8859-2"sv;
  387. if (encoding.is_one_of_ignoring_ascii_case("csisolatin3"sv, "iso-8859-3"sv, "iso-ir-109"sv, "iso8859-3"sv, "iso88593"sv, "iso_8859-3"sv, "iso_8859-3:1988"sv, "l3"sv, "latin3"sv))
  388. return "ISO-8859-3"sv;
  389. if (encoding.is_one_of_ignoring_ascii_case("csisolatin4"sv, "iso-8859-4"sv, "iso-ir-110"sv, "iso8859-4"sv, "iso88594"sv, "iso_8859-4"sv, "iso_8859-4:1989"sv, "l4"sv, "latin4"sv))
  390. return "ISO-8859-4"sv;
  391. if (encoding.is_one_of_ignoring_ascii_case("csisolatincyrillic"sv, "cyrillic"sv, "iso-8859-5"sv, "iso-ir-144"sv, "iso8859-5"sv, "iso88595"sv, "iso_8859-5"sv, "iso_8859-5:1988"sv))
  392. return "ISO-8859-5"sv;
  393. if (encoding.is_one_of_ignoring_ascii_case("arabic"sv, "asmo-708"sv, "csiso88596e"sv, "csiso88596i"sv, "csisolatinarabic"sv, "ecma-114"sv, "iso-8859-6"sv, "iso-8859-6-e"sv, "iso-8859-6-i"sv, "iso-ir-127"sv, "iso8859-6"sv, "iso88596"sv, "iso_8859-6"sv, "iso_8859-6:1987"sv))
  394. return "ISO-8859-6"sv;
  395. if (encoding.is_one_of_ignoring_ascii_case("csisolatingreek"sv, "ecma-118"sv, "elot_928"sv, "greek"sv, "greek8"sv, "iso-8859-7"sv, "iso-ir-126"sv, "iso8859-7"sv, "iso88597"sv, "iso_8859-7"sv, "iso_8859-7:1987"sv, "sun_eu_greek"sv))
  396. return "ISO-8859-7"sv;
  397. if (encoding.is_one_of_ignoring_ascii_case("csiso88598e"sv, "csisolatinhebrew"sv, "hebrew"sv, "iso-8859-8"sv, "iso-8859-8-e"sv, "iso-ir-138"sv, "iso8859-8"sv, "iso88598"sv, "iso_8859-8"sv, "iso_8859-8:1988"sv, "visual"sv))
  398. return "ISO-8859-8"sv;
  399. if (encoding.is_one_of_ignoring_ascii_case("csiso88598i"sv, "iso-8859-8-i"sv, "logical"sv))
  400. return "ISO-8859-8-I"sv;
  401. if (encoding.is_one_of_ignoring_ascii_case("csisolatin6"sv, "iso-8859-10"sv, "iso-ir-157"sv, "iso8859-10"sv, "iso885910"sv, "l6"sv, "latin6"sv))
  402. return "ISO-8859-10"sv;
  403. if (encoding.is_one_of_ignoring_ascii_case("iso-8859-13"sv, "iso8859-13"sv, "iso885913"sv))
  404. return "ISO-8859-13"sv;
  405. if (encoding.is_one_of_ignoring_ascii_case("iso-8859-14"sv, "iso8859-14"sv, "iso885914"sv))
  406. return "ISO-8859-14"sv;
  407. if (encoding.is_one_of_ignoring_ascii_case("csisolatin9"sv, "iso-8859-15"sv, "iso8859-15"sv, "iso885915"sv, "iso_8859-15"sv, "l9"sv))
  408. return "ISO-8859-15"sv;
  409. if (encoding.is_one_of_ignoring_ascii_case("iso-8859-16"sv))
  410. return "ISO-8859-16"sv;
  411. if (encoding.is_one_of_ignoring_ascii_case("cskoi8r"sv, "koi"sv, "koi8"sv, "koi8-r"sv, "koi8_r"sv))
  412. return "KOI8-R"sv;
  413. if (encoding.is_one_of_ignoring_ascii_case("koi8-ru"sv, "koi8-u"sv))
  414. return "KOI8-U"sv;
  415. if (encoding.is_one_of_ignoring_ascii_case("csmacintosh"sv, "mac"sv, "macintosh"sv, "x-mac-roman"sv))
  416. return "macintosh"sv;
  417. if (encoding.is_one_of_ignoring_ascii_case("pdfdocencoding"sv))
  418. return "PDFDocEncoding"sv;
  419. if (encoding.is_one_of_ignoring_ascii_case("dos-874"sv, "iso-8859-11"sv, "iso8859-11"sv, "iso885911"sv, "tis-620"sv, "windows-874"sv))
  420. return "windows-874"sv;
  421. if (encoding.is_one_of_ignoring_ascii_case("cp1250"sv, "windows-1250"sv, "x-cp1250"sv))
  422. return "windows-1250"sv;
  423. if (encoding.is_one_of_ignoring_ascii_case("cp1251"sv, "windows-1251"sv, "x-cp1251"sv))
  424. return "windows-1251"sv;
  425. if (encoding.is_one_of_ignoring_ascii_case("ansi_x3.4-1968"sv, "ascii"sv, "cp1252"sv, "cp819"sv, "csisolatin1"sv, "ibm819"sv, "iso-8859-1"sv, "iso-ir-100"sv, "iso8859-1"sv, "iso88591"sv, "iso_8859-1"sv, "iso_8859-1:1987"sv, "l1"sv, "latin1"sv, "us-ascii"sv, "windows-1252"sv, "x-cp1252"sv))
  426. return "windows-1252"sv;
  427. if (encoding.is_one_of_ignoring_ascii_case("cp1253"sv, "windows-1253"sv, "x-cp1253"sv))
  428. return "windows-1253"sv;
  429. if (encoding.is_one_of_ignoring_ascii_case("cp1254"sv, "csisolatin5"sv, "iso-8859-9"sv, "iso-ir-148"sv, "iso-8859-9"sv, "iso-88599"sv, "iso_8859-9"sv, "iso_8859-9:1989"sv, "l5"sv, "latin5"sv, "windows-1254"sv, "x-cp1254"sv))
  430. return "windows-1254"sv;
  431. if (encoding.is_one_of_ignoring_ascii_case("cp1255"sv, "windows-1255"sv, "x-cp1255"sv))
  432. return "windows-1255"sv;
  433. if (encoding.is_one_of_ignoring_ascii_case("cp1256"sv, "windows-1256"sv, "x-cp1256"sv))
  434. return "windows-1256"sv;
  435. if (encoding.is_one_of_ignoring_ascii_case("cp1257"sv, "windows-1257"sv, "x-cp1257"sv))
  436. return "windows-1257"sv;
  437. if (encoding.is_one_of_ignoring_ascii_case("cp1258"sv, "windows-1258"sv, "x-cp1258"sv))
  438. return "windows-1258"sv;
  439. if (encoding.is_one_of_ignoring_ascii_case("x-mac-cyrillic"sv, "x-mac-ukrainian"sv))
  440. return "x-mac-cyrillic"sv;
  441. if (encoding.is_one_of_ignoring_ascii_case("koi8-r"sv, "koi8r"sv))
  442. return "koi8-r"sv;
  443. if (encoding.is_one_of_ignoring_ascii_case("chinese"sv, "csgb2312"sv, "csiso58gb231280"sv, "gb2312"sv, "gb_2312"sv, "gb_2312-80"sv, "gbk"sv, "iso-ir-58"sv, "x-gbk"sv))
  444. return "GBK"sv;
  445. if (encoding.is_one_of_ignoring_ascii_case("gb18030"sv))
  446. return "gb18030"sv;
  447. if (encoding.is_one_of_ignoring_ascii_case("big5"sv, "big5-hkscs"sv, "cn-big5"sv, "csbig5"sv, "x-x-big5"sv))
  448. return "Big5"sv;
  449. if (encoding.is_one_of_ignoring_ascii_case("cseucpkdfmtjapanese"sv, "euc-jp"sv, "x-euc-jp"sv))
  450. return "EUC-JP"sv;
  451. if (encoding.is_one_of_ignoring_ascii_case("csiso2022jp"sv, "iso-2022-jp"sv))
  452. return "ISO-2022-JP"sv;
  453. if (encoding.is_one_of_ignoring_ascii_case("csshiftjis"sv, "ms932"sv, "ms_kanji"sv, "shift-jis"sv, "shift_jis"sv, "sjis"sv, "windows-31j"sv, "x-sjis"sv))
  454. return "Shift_JIS"sv;
  455. if (encoding.is_one_of_ignoring_ascii_case("cseuckr"sv, "csksc56011987"sv, "euc-kr"sv, "iso-ir-149"sv, "korean"sv, "ks_c_5601-1987"sv, "ks_c_5601-1989"sv, "ksc5601"sv, "ksc_5601"sv, "windows-949"sv))
  456. return "EUC-KR"sv;
  457. if (encoding.is_one_of_ignoring_ascii_case("csiso2022kr"sv, "hz-gb-2312"sv, "iso-2022-cn"sv, "iso-2022-cn-ext"sv, "iso-2022-kr"sv, "replacement"sv))
  458. return "replacement"sv;
  459. if (encoding.is_one_of_ignoring_ascii_case("unicodefffe"sv, "utf-16be"sv))
  460. return "UTF-16BE"sv;
  461. if (encoding.is_one_of_ignoring_ascii_case("csunicode"sv, "iso-10646-ucs-2"sv, "ucs-2"sv, "unicode"sv, "unicodefeff"sv, "utf-16"sv, "utf-16le"sv))
  462. return "UTF-16LE"sv;
  463. if (encoding.is_one_of_ignoring_ascii_case("x-user-defined"sv))
  464. return "x-user-defined"sv;
  465. dbgln("TextCodec: Unrecognized encoding: {}", encoding);
  466. return {};
  467. }
  468. // https://encoding.spec.whatwg.org/#bom-sniff
  469. Optional<Decoder&> bom_sniff_to_decoder(StringView input)
  470. {
  471. // 1. Let BOM be the result of peeking 3 bytes from ioQueue, converted to a byte sequence.
  472. // 2. For each of the rows in the table below, starting with the first one and going down,
  473. // if BOM starts with the bytes given in the first column, then return the encoding given
  474. // in the cell in the second column of that row. Otherwise, return null.
  475. // Byte Order Mark | Encoding
  476. // --------------------------
  477. // 0xEF 0xBB 0xBF | UTF-8
  478. // 0xFE 0xFF | UTF-16BE
  479. // 0xFF 0xFE | UTF-16LE
  480. auto bytes = input.bytes();
  481. if (bytes.size() < 2)
  482. return {};
  483. auto first_byte = bytes[0];
  484. switch (first_byte) {
  485. case 0xEF: // UTF-8
  486. if (bytes.size() < 3)
  487. return {};
  488. if (bytes[1] == 0xBB && bytes[2] == 0xBF)
  489. return s_utf8_decoder;
  490. return {};
  491. case 0xFE: // UTF-16BE
  492. if (bytes[1] == 0xFF)
  493. return s_utf16be_decoder;
  494. return {};
  495. case 0xFF: // UTF-16LE
  496. if (bytes[1] == 0xFE)
  497. return s_utf16le_decoder;
  498. return {};
  499. }
  500. return {};
  501. }
  502. // https://encoding.spec.whatwg.org/#decode
  503. ErrorOr<String> convert_input_to_utf8_using_given_decoder_unless_there_is_a_byte_order_mark(Decoder& fallback_decoder, StringView input)
  504. {
  505. Decoder* actual_decoder = &fallback_decoder;
  506. // 1. Let BOMEncoding be the result of BOM sniffing ioQueue.
  507. // 2. If BOMEncoding is non-null:
  508. if (auto unicode_decoder = bom_sniff_to_decoder(input); unicode_decoder.has_value()) {
  509. // 1. Set encoding to BOMEncoding.
  510. actual_decoder = &unicode_decoder.value();
  511. // 2. Read three bytes from ioQueue, if BOMEncoding is UTF-8; otherwise read two bytes. (Do nothing with those bytes.)
  512. // FIXME: I imagine this will be pretty slow for large inputs, as it's regenerating the input without the first 2/3 bytes.
  513. input = input.substring_view(&unicode_decoder.value() == &s_utf8_decoder ? 3 : 2);
  514. }
  515. VERIFY(actual_decoder);
  516. // 3. Process a queue with an instance of encoding’s decoder, ioQueue, output, and "replacement".
  517. // FIXME: This isn't the exact same as the spec, which is written in terms of I/O queues.
  518. auto output = TRY(actual_decoder->to_utf8(input));
  519. // 4. Return output.
  520. return output;
  521. }
  522. // https://encoding.spec.whatwg.org/#get-an-output-encoding
  523. StringView get_output_encoding(StringView encoding)
  524. {
  525. // 1. If encoding is replacement or UTF-16BE/LE, then return UTF-8.
  526. if (encoding.is_one_of_ignoring_ascii_case("replacement"sv, "utf-16le"sv, "utf-16be"sv))
  527. return "UTF-8"sv;
  528. // 2. Return encoding.
  529. return encoding;
  530. }
  531. bool Decoder::validate(StringView input)
  532. {
  533. auto result = this->process(input, [](auto code_point) -> ErrorOr<void> {
  534. if (code_point == replacement_code_point)
  535. return Error::from_errno(EINVAL);
  536. return {};
  537. });
  538. return !result.is_error();
  539. }
  540. ErrorOr<String> Decoder::to_utf8(StringView input)
  541. {
  542. StringBuilder builder(input.length());
  543. TRY(process(input, [&builder](u32 c) { return builder.try_append_code_point(c); }));
  544. return builder.to_string_without_validation();
  545. }
  546. ErrorOr<void> UTF8Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
  547. {
  548. for (auto c : Utf8View(input)) {
  549. TRY(on_code_point(c));
  550. }
  551. return {};
  552. }
  553. bool UTF8Decoder::validate(StringView input)
  554. {
  555. return Utf8View(input).validate();
  556. }
  557. ErrorOr<String> UTF8Decoder::to_utf8(StringView input)
  558. {
  559. // Discard the BOM
  560. auto bomless_input = input;
  561. if (auto bytes = input.bytes(); bytes.size() >= 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF) {
  562. bomless_input = input.substring_view(3);
  563. }
  564. return Decoder::to_utf8(bomless_input);
  565. }
  566. ErrorOr<void> UTF16BEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
  567. {
  568. // rfc2781, 2.2 Decoding UTF-16
  569. size_t utf16_length = input.length() - (input.length() % 2);
  570. for (size_t i = 0; i < utf16_length; i += 2) {
  571. // 1) If W1 < 0xD800 or W1 > 0xDFFF, the character value U is the value
  572. // of W1. Terminate.
  573. u16 w1 = (static_cast<u8>(input[i]) << 8) | static_cast<u8>(input[i + 1]);
  574. if (!is_unicode_surrogate(w1)) {
  575. TRY(on_code_point(w1));
  576. continue;
  577. }
  578. // 2) Determine if W1 is between 0xD800 and 0xDBFF. If not, the sequence
  579. // is in error and no valid character can be obtained using W1.
  580. // Terminate.
  581. // 3) If there is no W2 (that is, the sequence ends with W1), or if W2
  582. // is not between 0xDC00 and 0xDFFF, the sequence is in error.
  583. // Terminate.
  584. if (!Utf16View::is_high_surrogate(w1) || i + 2 == utf16_length) {
  585. TRY(on_code_point(replacement_code_point));
  586. continue;
  587. }
  588. u16 w2 = (static_cast<u8>(input[i + 2]) << 8) | static_cast<u8>(input[i + 3]);
  589. if (!Utf16View::is_low_surrogate(w2)) {
  590. TRY(on_code_point(replacement_code_point));
  591. continue;
  592. }
  593. // 4) Construct a 20-bit unsigned integer U', taking the 10 low-order
  594. // bits of W1 as its 10 high-order bits and the 10 low-order bits of
  595. // W2 as its 10 low-order bits.
  596. // 5) Add 0x10000 to U' to obtain the character value U. Terminate.
  597. TRY(on_code_point(Utf16View::decode_surrogate_pair(w1, w2)));
  598. i += 2;
  599. }
  600. return {};
  601. }
  602. bool UTF16BEDecoder::validate(StringView input)
  603. {
  604. size_t utf16_length = input.length() - (input.length() % 2);
  605. for (size_t i = 0; i < utf16_length; i += 2) {
  606. u16 w1 = (static_cast<u8>(input[i]) << 8) | static_cast<u8>(input[i + 1]);
  607. if (!is_unicode_surrogate(w1))
  608. continue;
  609. if (!Utf16View::is_high_surrogate(w1) || i + 2 == utf16_length)
  610. return false;
  611. u16 w2 = (static_cast<u8>(input[i + 2]) << 8) | static_cast<u8>(input[i + 3]);
  612. if (!Utf16View::is_low_surrogate(w2))
  613. return false;
  614. i += 2;
  615. }
  616. return true;
  617. }
  618. ErrorOr<String> UTF16BEDecoder::to_utf8(StringView input)
  619. {
  620. // Discard the BOM
  621. auto bomless_input = input;
  622. if (auto bytes = input.bytes(); bytes.size() >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF)
  623. bomless_input = input.substring_view(2);
  624. StringBuilder builder(bomless_input.length() / 2);
  625. TRY(process(bomless_input, [&builder](u32 c) { return builder.try_append_code_point(c); }));
  626. return builder.to_string();
  627. }
  628. ErrorOr<void> UTF16LEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
  629. {
  630. // rfc2781, 2.2 Decoding UTF-16
  631. size_t utf16_length = input.length() - (input.length() % 2);
  632. for (size_t i = 0; i < utf16_length; i += 2) {
  633. // 1) If W1 < 0xD800 or W1 > 0xDFFF, the character value U is the value
  634. // of W1. Terminate.
  635. u16 w1 = static_cast<u8>(input[i]) | (static_cast<u8>(input[i + 1]) << 8);
  636. if (!is_unicode_surrogate(w1)) {
  637. TRY(on_code_point(w1));
  638. continue;
  639. }
  640. // 2) Determine if W1 is between 0xD800 and 0xDBFF. If not, the sequence
  641. // is in error and no valid character can be obtained using W1.
  642. // Terminate.
  643. // 3) If there is no W2 (that is, the sequence ends with W1), or if W2
  644. // is not between 0xDC00 and 0xDFFF, the sequence is in error.
  645. // Terminate.
  646. if (!Utf16View::is_high_surrogate(w1) || i + 2 == utf16_length) {
  647. TRY(on_code_point(replacement_code_point));
  648. continue;
  649. }
  650. u16 w2 = static_cast<u8>(input[i + 2]) | (static_cast<u8>(input[i + 3]) << 8);
  651. if (!Utf16View::is_low_surrogate(w2)) {
  652. TRY(on_code_point(replacement_code_point));
  653. continue;
  654. }
  655. // 4) Construct a 20-bit unsigned integer U', taking the 10 low-order
  656. // bits of W1 as its 10 high-order bits and the 10 low-order bits of
  657. // W2 as its 10 low-order bits.
  658. // 5) Add 0x10000 to U' to obtain the character value U. Terminate.
  659. TRY(on_code_point(Utf16View::decode_surrogate_pair(w1, w2)));
  660. i += 2;
  661. }
  662. return {};
  663. }
  664. bool UTF16LEDecoder::validate(StringView input)
  665. {
  666. size_t utf16_length = input.length() - (input.length() % 2);
  667. for (size_t i = 0; i < utf16_length; i += 2) {
  668. u16 w1 = static_cast<u8>(input[i]) | (static_cast<u8>(input[i + 1]) << 8);
  669. if (!is_unicode_surrogate(w1))
  670. continue;
  671. if (!Utf16View::is_high_surrogate(w1) || i + 2 == utf16_length)
  672. return false;
  673. u16 w2 = static_cast<u8>(input[i + 2]) | (static_cast<u8>(input[i + 3]) << 8);
  674. if (!Utf16View::is_low_surrogate(w2))
  675. return false;
  676. i += 2;
  677. }
  678. return true;
  679. }
  680. ErrorOr<String> UTF16LEDecoder::to_utf8(StringView input)
  681. {
  682. // Discard the BOM
  683. auto bomless_input = input;
  684. if (auto bytes = input.bytes(); bytes.size() >= 2 && bytes[0] == 0xFF && bytes[1] == 0xFE)
  685. bomless_input = input.substring_view(2);
  686. StringBuilder builder(bomless_input.length() / 2);
  687. TRY(process(bomless_input, [&builder](u32 c) { return builder.try_append_code_point(c); }));
  688. return builder.to_string();
  689. }
  690. ErrorOr<void> Latin1Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
  691. {
  692. for (u8 ch : input) {
  693. // Latin1 is the same as the first 256 Unicode code_points, so no mapping is needed, just utf-8 encoding.
  694. TRY(on_code_point(ch));
  695. }
  696. return {};
  697. }
  698. namespace {
  699. u32 convert_latin2_to_utf8(u8 in)
  700. {
  701. switch (in) {
  702. #define MAP(X, Y) \
  703. case X: \
  704. return Y
  705. MAP(0xA1, 0x104);
  706. MAP(0xA2, 0x2D8);
  707. MAP(0xA3, 0x141);
  708. MAP(0xA5, 0x13D);
  709. MAP(0xA6, 0x15A);
  710. MAP(0xA9, 0x160);
  711. MAP(0xAA, 0x15E);
  712. MAP(0xAB, 0x164);
  713. MAP(0xAC, 0x179);
  714. MAP(0xAE, 0x17D);
  715. MAP(0xAF, 0x17B);
  716. MAP(0xB1, 0x105);
  717. MAP(0xB2, 0x2DB);
  718. MAP(0xB3, 0x142);
  719. MAP(0xB5, 0x13E);
  720. MAP(0xB6, 0x15B);
  721. MAP(0xB7, 0x2C7);
  722. MAP(0xB9, 0x161);
  723. MAP(0xBA, 0x15F);
  724. MAP(0xBB, 0x165);
  725. MAP(0xBC, 0x17A);
  726. MAP(0xBD, 0x2DD);
  727. MAP(0xBE, 0x17E);
  728. MAP(0xBF, 0x17C);
  729. MAP(0xC0, 0x154);
  730. MAP(0xC3, 0x102);
  731. MAP(0xC5, 0x139);
  732. MAP(0xC6, 0x106);
  733. MAP(0xC8, 0x10C);
  734. MAP(0xCA, 0x118);
  735. MAP(0xCC, 0x11A);
  736. MAP(0xCF, 0x10E);
  737. MAP(0xD0, 0x110);
  738. MAP(0xD1, 0x143);
  739. MAP(0xD2, 0x147);
  740. MAP(0xD5, 0x150);
  741. MAP(0xD8, 0x158);
  742. MAP(0xD9, 0x16E);
  743. MAP(0xDB, 0x170);
  744. MAP(0xDE, 0x162);
  745. MAP(0xE0, 0x155);
  746. MAP(0xE3, 0x103);
  747. MAP(0xE5, 0x13A);
  748. MAP(0xE6, 0x107);
  749. MAP(0xE8, 0x10D);
  750. MAP(0xEA, 0x119);
  751. MAP(0xEC, 0x11B);
  752. MAP(0xEF, 0x10F);
  753. MAP(0xF0, 0x111);
  754. MAP(0xF1, 0x144);
  755. MAP(0xF2, 0x148);
  756. MAP(0xF5, 0x151);
  757. MAP(0xF8, 0x159);
  758. MAP(0xF9, 0x16F);
  759. MAP(0xFB, 0x171);
  760. MAP(0xFE, 0x163);
  761. MAP(0xFF, 0x2D9);
  762. #undef MAP
  763. default:
  764. return in;
  765. }
  766. }
  767. }
  768. ErrorOr<void> Latin2Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
  769. {
  770. for (auto c : input) {
  771. TRY(on_code_point(convert_latin2_to_utf8(c)));
  772. }
  773. return {};
  774. }
  775. ErrorOr<void> Latin9Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
  776. {
  777. auto convert_latin9_to_utf8 = [](u8 ch) -> u32 {
  778. // Latin9 is the same as the first 256 Unicode code points, except for 8 characters.
  779. switch (ch) {
  780. case 0xA4:
  781. return 0x20AC;
  782. case 0xA6:
  783. return 0x160;
  784. case 0xA8:
  785. return 0x161;
  786. case 0xB4:
  787. return 0x17D;
  788. case 0xB8:
  789. return 0x17E;
  790. case 0xBC:
  791. return 0x152;
  792. case 0xBD:
  793. return 0x153;
  794. case 0xBE:
  795. return 0x178;
  796. default:
  797. return ch;
  798. }
  799. };
  800. for (auto ch : input) {
  801. TRY(on_code_point(convert_latin9_to_utf8(ch)));
  802. }
  803. return {};
  804. }
  805. ErrorOr<void> PDFDocEncodingDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
  806. {
  807. // PDF 1.7 spec, Appendix D.2 "PDFDocEncoding Character Set"
  808. // Character codes 0-8, 11-12, 14-23, 127, 159, 173 are not defined per spec.
  809. // clang-format off
  810. static constexpr Array<u32, 256> translation_table = {
  811. 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007,
  812. 0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F,
  813. 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0017, 0x0017,
  814. 0x02D8, 0x02C7, 0x02C6, 0x02D9, 0x02DD, 0x02DB, 0x02DA, 0x02DC,
  815. 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027,
  816. 0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F,
  817. 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037,
  818. 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F,
  819. 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
  820. 0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F,
  821. 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057,
  822. 0x0058, 0x0059, 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F,
  823. 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067,
  824. 0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F,
  825. 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077,
  826. 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, 0x007E, 0xFFFC,
  827. 0x2022, 0x2020, 0x2021, 0x2026, 0x2014, 0x2013, 0x0192, 0x2044,
  828. 0x2039, 0x203A, 0x2212, 0x2030, 0x201E, 0x201C, 0x201D, 0x2018,
  829. 0x2019, 0x201A, 0x2122, 0xFB01, 0xFB02, 0x0141, 0x0152, 0x0160,
  830. 0x0178, 0x017D, 0x0131, 0x0142, 0x0153, 0x0161, 0x017E, 0xFFFC,
  831. 0x20AC, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7,
  832. 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0xFFFC, 0x00AE, 0x00AF,
  833. 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7,
  834. 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF,
  835. 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7,
  836. 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
  837. 0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7,
  838. 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF,
  839. 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7,
  840. 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
  841. 0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7,
  842. 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF,
  843. };
  844. // clang-format on
  845. for (u8 ch : input)
  846. TRY(on_code_point(translation_table[ch]));
  847. return {};
  848. }
  849. ErrorOr<void> TurkishDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
  850. {
  851. auto convert_turkish_to_utf8 = [](u8 ch) -> u32 {
  852. // Turkish (aka ISO-8859-9, Windows-1254) is the same as the first 256 Unicode code points, except for 6 characters.
  853. switch (ch) {
  854. case 0xD0:
  855. return 0x11E;
  856. case 0xDD:
  857. return 0x130;
  858. case 0xDE:
  859. return 0x15E;
  860. case 0xF0:
  861. return 0x11F;
  862. case 0xFD:
  863. return 0x131;
  864. case 0xFE:
  865. return 0x15F;
  866. default:
  867. return ch;
  868. }
  869. };
  870. for (auto ch : input) {
  871. TRY(on_code_point(convert_turkish_to_utf8(ch)));
  872. }
  873. return {};
  874. }
  875. // https://encoding.spec.whatwg.org/#x-user-defined-decoder
  876. ErrorOr<void> XUserDefinedDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
  877. {
  878. auto convert_x_user_defined_to_utf8 = [](u8 ch) -> u32 {
  879. // 2. If byte is an ASCII byte, return a code point whose value is byte.
  880. // https://infra.spec.whatwg.org/#ascii-byte
  881. // An ASCII byte is a byte in the range 0x00 (NUL) to 0x7F (DEL), inclusive.
  882. // NOTE: This doesn't check for ch >= 0x00, as that would always be true due to being unsigned.
  883. if (ch <= 0x7f)
  884. return ch;
  885. // 3. Return a code point whose value is 0xF780 + byte − 0x80.
  886. return 0xF780 + ch - 0x80;
  887. };
  888. for (auto ch : input) {
  889. TRY(on_code_point(convert_x_user_defined_to_utf8(ch)));
  890. }
  891. // 1. If byte is end-of-queue, return finished.
  892. return {};
  893. }
  894. // https://encoding.spec.whatwg.org/#single-byte-decoder
  895. ErrorOr<void> SingleByteDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
  896. {
  897. for (u8 const byte : input) {
  898. if (byte < 0x80) {
  899. // 2. If byte is an ASCII byte, return a code point whose value is byte.
  900. TRY(on_code_point(byte));
  901. } else {
  902. // 3. Let code point be the index code point for byte − 0x80 in index single-byte.
  903. auto code_point = m_translation_table[byte - 0x80];
  904. // 4. If code point is null, return error.
  905. // NOTE: Error is communicated with 0xFFFD
  906. // 5. Return a code point whose value is code point.
  907. TRY(on_code_point(code_point));
  908. }
  909. }
  910. // 1. If byte is end-of-queue, return finished.
  911. return {};
  912. }
  913. // https://encoding.spec.whatwg.org/#index-gb18030-ranges-code-point
  914. static Optional<u32> index_gb18030_ranges_code_point(u32 pointer)
  915. {
  916. // 1. If pointer is greater than 39419 and less than 189000, or pointer is greater than 1237575, return null.
  917. if ((pointer > 39419 && pointer < 189000) || pointer > 1237575)
  918. return {};
  919. // 2. If pointer is 7457, return code point U+E7C7.
  920. if (pointer == 7457)
  921. return 0xE7C7;
  922. // FIXME: Encoding specification is not updated to GB-18030-2022 yet (https://github.com/whatwg/encoding/issues/312)
  923. // NOTE: This matches https://commits.webkit.org/266173@main
  924. switch (pointer) {
  925. case 19057:
  926. return 0xE81E; // 82 35 90 37
  927. case 19058:
  928. return 0xE826; // 82 35 90 38
  929. case 19059:
  930. return 0xE82B; // 82 35 90 39
  931. case 19060:
  932. return 0xE82C; // 82 35 91 30
  933. case 19061:
  934. return 0xE832; // 82 35 91 31
  935. case 19062:
  936. return 0xE843; // 82 35 91 32
  937. case 19063:
  938. return 0xE854; // 82 35 91 33
  939. case 19064:
  940. return 0xE864; // 82 35 91 34
  941. case 39076:
  942. return 0xE78D; // 84 31 82 36
  943. case 39077:
  944. return 0xE78F; // 84 31 82 37
  945. case 39078:
  946. return 0xE78E; // 84 31 82 38
  947. case 39079:
  948. return 0xE790; // 84 31 82 39
  949. case 39080:
  950. return 0xE791; // 84 31 83 30
  951. case 39081:
  952. return 0xE792; // 84 31 83 31
  953. case 39082:
  954. return 0xE793; // 84 31 83 32
  955. case 39083:
  956. return 0xE794; // 84 31 83 33
  957. case 39084:
  958. return 0xE795; // 84 31 83 34
  959. case 39085:
  960. return 0xE796; // 84 31 83 35
  961. default:
  962. break;
  963. }
  964. // 3. Let offset be the last pointer in index gb18030 ranges that is less than or equal to pointer and let code point offset be its corresponding code point.
  965. size_t last_index;
  966. binary_search(s_gb18030_ranges, pointer, &last_index, [](auto const pointer, auto const& entry) {
  967. return pointer - entry.pointer;
  968. });
  969. auto offset = s_gb18030_ranges[last_index].pointer;
  970. auto code_point_offset = s_gb18030_ranges[last_index].code_point;
  971. // 4. Return a code point whose value is code point offset + pointer − offset.
  972. return code_point_offset + pointer - offset;
  973. }
  974. // https://encoding.spec.whatwg.org/#gb18030-decoder
  975. ErrorOr<void> GB18030Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
  976. {
  977. // gb18030’s decoder has an associated gb18030 first, gb18030 second, and gb18030 third (all initially 0x00).
  978. u8 first = 0x00;
  979. u8 second = 0x00;
  980. u8 third = 0x00;
  981. // gb18030’s decoder’s handler, given ioQueue and byte, runs these steps:
  982. size_t index = 0;
  983. while (true) {
  984. // 1. If byte is end-of-queue and gb18030 first, gb18030 second, and gb18030 third are 0x00, return finished.
  985. if (index >= input.length() && first == 0x00 && second == 0x00 && third == 0x00)
  986. return {};
  987. // 2. If byte is end-of-queue, and gb18030 first, gb18030 second, or gb18030 third is not 0x00, set gb18030 first, gb18030 second, and gb18030 third to 0x00, and return error.
  988. if (index >= input.length() && (first != 0x00 || second != 0x00 || third != 0x00)) {
  989. first = 0x00;
  990. second = 0x00;
  991. third = 0x00;
  992. TRY(on_code_point(replacement_code_point));
  993. continue;
  994. }
  995. u8 const byte = input[index++];
  996. // 3. If gb18030 third is not 0x00, then:
  997. if (third != 0x00) {
  998. // 1. If byte is not in the range 0x30 to 0x39, inclusive, then:
  999. if (byte < 0x30 || byte > 0x39) {
  1000. // 1. Restore « gb18030 second, gb18030 third, byte » to ioQueue.
  1001. index -= 3;
  1002. // 2. Set gb18030 first, gb18030 second, and gb18030 third to 0x00.
  1003. first = 0x00;
  1004. second = 0x00;
  1005. third = 0x00;
  1006. // 3. Return error.
  1007. TRY(on_code_point(replacement_code_point));
  1008. continue;
  1009. }
  1010. // 2. Let code point be the index gb18030 ranges code point for ((gb18030 first − 0x81) × (10 × 126 × 10)) + ((gb18030 second − 0x30) × (10 × 126)) + ((gb18030 third − 0x81) × 10) + byte − 0x30.
  1011. auto code_point = index_gb18030_ranges_code_point(((first - 0x81) * (10 * 126 * 10)) + ((second - 0x30) * (10 * 126)) + ((third - 0x81) * 10) + byte - 0x30);
  1012. // 3. Set gb18030 first, gb18030 second, and gb18030 third to 0x00.
  1013. first = 0x00;
  1014. second = 0x00;
  1015. third = 0x00;
  1016. // 4. If code point is null, return error.
  1017. if (!code_point.has_value()) {
  1018. TRY(on_code_point(replacement_code_point));
  1019. continue;
  1020. }
  1021. // 5. Return a code point whose value is code point.
  1022. TRY(on_code_point(code_point.value()));
  1023. continue;
  1024. }
  1025. // 4. If gb18030 second is not 0x00, then:
  1026. if (second != 0x00) {
  1027. // 1. If byte is in the range 0x81 to 0xFE, inclusive, set gb18030 third to byte and return continue.
  1028. if (byte >= 0x81 && byte <= 0xFE) {
  1029. third = byte;
  1030. continue;
  1031. }
  1032. // 2. Restore « gb18030 second, byte » to ioQueue, set gb18030 first and gb18030 second to 0x00, and return error.
  1033. index -= 2;
  1034. first = 0x00;
  1035. second = 0x00;
  1036. TRY(on_code_point(replacement_code_point));
  1037. continue;
  1038. }
  1039. // 5. If gb18030 first is not 0x00, then:
  1040. if (first != 0x00) {
  1041. // 1. If byte is in the range 0x30 to 0x39, inclusive, set gb18030 second to byte and return continue.
  1042. if (byte >= 0x30 && byte <= 0x39) {
  1043. second = byte;
  1044. continue;
  1045. }
  1046. // 2. Let lead be gb18030 first, let pointer be null, and set gb18030 first to 0x00.
  1047. auto lead = first;
  1048. Optional<u32> pointer;
  1049. first = 0x00;
  1050. // 3. Let offset be 0x40 if byte is less than 0x7F, otherwise 0x41.
  1051. u8 const offset = byte < 0x7F ? 0x40 : 0x41;
  1052. // 4. If byte is in the range 0x40 to 0x7E, inclusive, or 0x80 to 0xFE, inclusive, set pointer to (lead − 0x81) × 190 + (byte − offset).
  1053. if ((byte >= 0x40 && byte <= 0x7E) || (byte >= 0x80 && byte <= 0xFE))
  1054. pointer = (lead - 0x81) * 190 + (byte - offset);
  1055. // 5. Let code point be null if pointer is null, otherwise the index code point for pointer in index gb18030.
  1056. auto code_point = pointer.has_value() ? index_gb18030_code_point(pointer.value()) : Optional<u32> {};
  1057. // 6. If code point is non-null, return a code point whose value is code point.
  1058. if (code_point.has_value()) {
  1059. TRY(on_code_point(code_point.value()));
  1060. continue;
  1061. }
  1062. // 7. If byte is an ASCII byte, restore byte to ioQueue.
  1063. if (byte <= 0x7F)
  1064. index--;
  1065. // 8. Return error.
  1066. TRY(on_code_point(replacement_code_point));
  1067. continue;
  1068. }
  1069. // 6. If byte is an ASCII byte, return a code point whose value is byte.
  1070. if (byte <= 0x7F) {
  1071. TRY(on_code_point(byte));
  1072. continue;
  1073. }
  1074. // 7. If byte is 0x80, return code point U+20AC.
  1075. if (byte == 0x80) {
  1076. TRY(on_code_point(0x20AC));
  1077. continue;
  1078. }
  1079. // 8. If byte is in the range 0x81 to 0xFE, inclusive, set gb18030 first to byte and return continue.
  1080. if (byte >= 0x81 && byte <= 0xFE) {
  1081. first = byte;
  1082. continue;
  1083. }
  1084. // 9. Return error.
  1085. TRY(on_code_point(replacement_code_point));
  1086. }
  1087. }
  1088. // https://encoding.spec.whatwg.org/#big5-decoder
  1089. ErrorOr<void> Big5Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
  1090. {
  1091. // Big5’s decoder has an associated Big5 lead (initially 0x00).
  1092. u8 big5_lead = 0x00;
  1093. // Big5’s decoder’s handler, given ioQueue and byte, runs these steps:
  1094. size_t index = 0;
  1095. while (true) {
  1096. // 1. If byte is end-of-queue and Big5 lead is not 0x00, set Big5 lead to 0x00 and return error.
  1097. if (index >= input.length() && big5_lead != 0x00) {
  1098. big5_lead = 0x00;
  1099. TRY(on_code_point(replacement_code_point));
  1100. continue;
  1101. }
  1102. // 2. If byte is end-of-queue and Big5 lead is 0x00, return finished.
  1103. if (index >= input.length() && big5_lead == 0x00)
  1104. return {};
  1105. u8 const byte = input[index++];
  1106. // 3. If Big5 lead is not 0x00, let lead be Big5 lead, let pointer be null, set Big5 lead to 0x00, and then:
  1107. if (big5_lead != 0x00) {
  1108. auto lead = big5_lead;
  1109. Optional<u32> pointer;
  1110. big5_lead = 0x00;
  1111. // 1. Let offset be 0x40 if byte is less than 0x7F, otherwise 0x62.
  1112. u8 const offset = byte < 0x7F ? 0x40 : 0x62;
  1113. // 2. If byte is in the range 0x40 to 0x7E, inclusive, or 0xA1 to 0xFE, inclusive, set pointer to (lead − 0x81) × 157 + (byte − offset).
  1114. if ((byte >= 0x40 && byte <= 0x7E) || (byte >= 0xA1 && byte <= 0xFE))
  1115. pointer = (lead - 0x81) * 157 + (byte - offset);
  1116. // 3. If there is a row in the table below whose first column is pointer, return the two code points listed in its second column (the third column is irrelevant):
  1117. if (pointer.has_value() && pointer.value() == 1133) {
  1118. TRY(on_code_point(0x00CA));
  1119. TRY(on_code_point(0x0304));
  1120. continue;
  1121. }
  1122. if (pointer.has_value() && pointer.value() == 1135) {
  1123. TRY(on_code_point(0x00CA));
  1124. TRY(on_code_point(0x030C));
  1125. continue;
  1126. }
  1127. if (pointer.has_value() && pointer.value() == 1164) {
  1128. TRY(on_code_point(0x00EA));
  1129. TRY(on_code_point(0x0304));
  1130. continue;
  1131. }
  1132. if (pointer.has_value() && pointer.value() == 1166) {
  1133. TRY(on_code_point(0x00EA));
  1134. TRY(on_code_point(0x030C));
  1135. continue;
  1136. }
  1137. // 4. Let code point be null if pointer is null, otherwise the index code point for pointer in index Big5.
  1138. auto code_pointer = pointer.has_value() ? index_big5_code_point(pointer.value()) : Optional<u32> {};
  1139. // 5. If code point is non-null, return a code point whose value is code point.
  1140. if (code_pointer.has_value()) {
  1141. TRY(on_code_point(code_pointer.value()));
  1142. continue;
  1143. }
  1144. // 6. If byte is an ASCII byte, restore byte to ioQueue.
  1145. if (byte <= 0x7F)
  1146. index--;
  1147. // 7. Return error.
  1148. TRY(on_code_point(replacement_code_point));
  1149. continue;
  1150. }
  1151. // 4. If byte is an ASCII byte, return a code point whose value is byte.
  1152. if (byte <= 0x7F) {
  1153. TRY(on_code_point(byte));
  1154. continue;
  1155. }
  1156. // 5. If byte is in the range 0x81 to 0xFE, inclusive, set Big5 lead to byte and return continue.
  1157. if (byte >= 0x81 && byte <= 0xFE) {
  1158. big5_lead = byte;
  1159. continue;
  1160. }
  1161. // 6. Return error
  1162. TRY(on_code_point(replacement_code_point));
  1163. }
  1164. }
  1165. // https://encoding.spec.whatwg.org/#euc-jp-decoder
  1166. ErrorOr<void> EUCJPDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
  1167. {
  1168. // EUC-JP’s decoder has an associated EUC-JP jis0212 (initially false) and EUC-JP lead (initially 0x00).
  1169. bool jis0212 = false;
  1170. u8 euc_jp_lead = 0x00;
  1171. // EUC-JP’s decoder’s handler, given ioQueue and byte, runs these steps:
  1172. size_t index = 0;
  1173. while (true) {
  1174. // 1. If byte is end-of-queue and EUC-JP lead is not 0x00, set EUC-JP lead to 0x00, and return error.
  1175. if (index >= input.length() && euc_jp_lead != 0x00) {
  1176. euc_jp_lead = 0x00;
  1177. TRY(on_code_point(replacement_code_point));
  1178. continue;
  1179. }
  1180. // 2. If byte is end-of-queue and EUC-JP lead is 0x00, return finished.
  1181. if (index >= input.length() && euc_jp_lead == 0x00)
  1182. return {};
  1183. u8 const byte = input[index++];
  1184. // 3. If EUC-JP lead is 0x8E and byte is in the range 0xA1 to 0xDF, inclusive, set EUC-JP lead to 0x00 and return a code point whose value is 0xFF61 − 0xA1 + byte.
  1185. if (euc_jp_lead == 0x8E && byte >= 0xA1 && byte <= 0xDF) {
  1186. euc_jp_lead = 0x00;
  1187. TRY(on_code_point(0xFF61 - 0xA1 + byte));
  1188. continue;
  1189. }
  1190. // 4. If EUC-JP lead is 0x8F and byte is in the range 0xA1 to 0xFE, inclusive, set EUC-JP jis0212 to true, set EUC-JP lead to byte, and return continue.
  1191. if (euc_jp_lead == 0x8F && byte >= 0xA1 && byte <= 0xFE) {
  1192. jis0212 = true;
  1193. euc_jp_lead = byte;
  1194. continue;
  1195. }
  1196. // 5. If EUC-JP lead is not 0x00, let lead be EUC-JP lead, set EUC-JP lead to 0x00, and then:
  1197. if (euc_jp_lead != 0x00) {
  1198. auto lead = euc_jp_lead;
  1199. euc_jp_lead = 0x00;
  1200. // 1. Let code point be null.
  1201. Optional<u32> code_point;
  1202. // 2. If lead and byte are both in the range 0xA1 to 0xFE, inclusive, then set code point to the index code point for (lead − 0xA1) × 94 + byte − 0xA1 in index jis0208 if EUC-JP jis0212 is false and in index jis0212 otherwise.
  1203. if (lead >= 0xA1 && lead <= 0xFE && byte >= 0xA1 && byte <= 0xFE) {
  1204. auto pointer = (lead - 0xA1) * 94 + byte - 0xA1;
  1205. code_point = jis0212 ? index_jis0212_code_point(pointer) : index_jis0208_code_point(pointer);
  1206. }
  1207. // 3. Set EUC-JP jis0212 to false.
  1208. jis0212 = false;
  1209. // 4. If code point is non-null, return a code point whose value is code point.
  1210. if (code_point.has_value()) {
  1211. TRY(on_code_point(code_point.value()));
  1212. continue;
  1213. }
  1214. // 5. If byte is an ASCII byte, restore byte to ioQueue.
  1215. if (byte <= 0x7F)
  1216. index--;
  1217. // 6. Return error.
  1218. TRY(on_code_point(replacement_code_point));
  1219. continue;
  1220. }
  1221. // 6. If byte is an ASCII byte, return a code point whose value is byte.
  1222. if (byte <= 0x7F) {
  1223. TRY(on_code_point(byte));
  1224. continue;
  1225. }
  1226. // 7. If byte is 0x8E, 0x8F, or in the range 0xA1 to 0xFE, inclusive, set EUC-JP lead to byte and return continue.
  1227. if (byte == 0x8E || byte == 0x8F || (byte >= 0xA1 && byte <= 0xFE)) {
  1228. euc_jp_lead = byte;
  1229. continue;
  1230. }
  1231. // 8. Return error.
  1232. TRY(on_code_point(replacement_code_point));
  1233. }
  1234. }
  1235. enum class ISO2022JPState {
  1236. ASCII,
  1237. Roman,
  1238. Katakana,
  1239. LeadByte,
  1240. TrailByte,
  1241. EscapeStart,
  1242. Escape,
  1243. };
  1244. // https://encoding.spec.whatwg.org/#iso-2022-jp-decoder
  1245. ErrorOr<void> ISO2022JPDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
  1246. {
  1247. // ISO-2022-JP’s decoder has an associated ISO-2022-JP decoder state (initially ASCII), ISO-2022-JP decoder output state (initially ASCII), ISO-2022-JP lead (initially 0x00), and ISO-2022-JP output (initially false).
  1248. auto decoder_state = ISO2022JPState::ASCII;
  1249. auto output_state = ISO2022JPState::ASCII;
  1250. u8 iso2022_jp_lead = 0x00;
  1251. bool iso2022_jp_output = false;
  1252. size_t index = 0;
  1253. while (true) {
  1254. Optional<u8> byte;
  1255. if (index < input.length())
  1256. byte = input[index++];
  1257. // ISO-2022-JP’s decoder’s handler, given ioQueue and byte, runs these steps, switching on ISO-2022-JP decoder state:
  1258. switch (decoder_state) {
  1259. case ISO2022JPState::ASCII:
  1260. // Based on byte:
  1261. // 0x1B: Set ISO-2022-JP decoder state to escape start and return continue.
  1262. if (byte == 0x1B) {
  1263. decoder_state = ISO2022JPState::EscapeStart;
  1264. continue;
  1265. }
  1266. // 0x00 to 0x7F, excluding 0x0E, 0x0F, and 0x1B: Set ISO-2022-JP output to false and return a code point whose value is byte.
  1267. if (byte.has_value() && byte.value() <= 0x7F && byte != 0x0E && byte != 0x0F) {
  1268. iso2022_jp_output = false;
  1269. TRY(on_code_point(byte.value()));
  1270. continue;
  1271. }
  1272. // end-of-queue: Return finished.
  1273. if (!byte.has_value())
  1274. return {};
  1275. // Otherwise: Set ISO-2022-JP output to false and return error.
  1276. iso2022_jp_output = false;
  1277. TRY(on_code_point(replacement_code_point));
  1278. break;
  1279. case ISO2022JPState::Roman:
  1280. // Based on byte:
  1281. // 0x1B: Set ISO-2022-JP decoder state to escape start and return continue.
  1282. if (byte == 0x1B) {
  1283. decoder_state = ISO2022JPState::EscapeStart;
  1284. continue;
  1285. }
  1286. // 0x5C: Set ISO-2022-JP output to false and return code point U+00A5.
  1287. if (byte == 0x5C) {
  1288. iso2022_jp_output = false;
  1289. TRY(on_code_point(0x00A5));
  1290. continue;
  1291. }
  1292. // 0x7E: Set ISO-2022-JP output to false and return code point U+203E.
  1293. if (byte == 0x7E) {
  1294. iso2022_jp_output = false;
  1295. TRY(on_code_point(0x203E));
  1296. continue;
  1297. }
  1298. // 0x00 to 0x7F, excluding 0x0E, 0x0F, 0x1B, 0x5C, and 0x7E: Set ISO-2022-JP output to false and return a code point whose value is byte.
  1299. if (byte.has_value() && byte.value() <= 0x7F && byte != 0x0E && byte != 0x0F) {
  1300. iso2022_jp_output = false;
  1301. TRY(on_code_point(byte.value()));
  1302. continue;
  1303. }
  1304. // end-of-queue: Return finished.
  1305. if (!byte.has_value())
  1306. return {};
  1307. // Otherwise: Set ISO-2022-JP output to false and return error.
  1308. iso2022_jp_output = false;
  1309. TRY(on_code_point(replacement_code_point));
  1310. break;
  1311. case ISO2022JPState::Katakana:
  1312. // Based on byte:
  1313. // 0x1B: Set ISO-2022-JP decoder state to escape start and return continue.
  1314. if (byte == 0x1B) {
  1315. decoder_state = ISO2022JPState::EscapeStart;
  1316. continue;
  1317. }
  1318. // 0x21 to 0x5F: Set ISO-2022-JP output to false and return a code point whose value is 0xFF61 − 0x21 + byte.
  1319. if (byte.has_value() && byte.value() >= 0x21 && byte.value() <= 0x5F) {
  1320. iso2022_jp_output = false;
  1321. TRY(on_code_point(0xFF61 - 0x21 + byte.value()));
  1322. continue;
  1323. }
  1324. // end-of-queue: Return finished.
  1325. if (!byte.has_value())
  1326. return {};
  1327. // Otherwise: Set ISO-2022-JP output to false and return error.
  1328. iso2022_jp_output = false;
  1329. TRY(on_code_point(replacement_code_point));
  1330. break;
  1331. case ISO2022JPState::LeadByte:
  1332. // Based on byte:
  1333. // 0x1B: Set ISO-2022-JP decoder state to escape start and return continue.
  1334. if (byte == 0x1B) {
  1335. decoder_state = ISO2022JPState::EscapeStart;
  1336. continue;
  1337. }
  1338. // 0x21 to 0x7E: Set ISO-2022-JP output to false, ISO-2022-JP lead to byte, ISO-2022-JP decoder state to trail byte, and return continue.
  1339. if (byte.has_value() && byte.value() >= 0x21 && byte.value() <= 0x7E) {
  1340. iso2022_jp_output = false;
  1341. iso2022_jp_lead = byte.value();
  1342. decoder_state = ISO2022JPState::TrailByte;
  1343. continue;
  1344. }
  1345. // end-of-queue: Return finished.
  1346. if (!byte.has_value())
  1347. return {};
  1348. // Otherwise: Set ISO-2022-JP output to false and return error.
  1349. iso2022_jp_output = false;
  1350. TRY(on_code_point(replacement_code_point));
  1351. break;
  1352. case ISO2022JPState::TrailByte:
  1353. // Based on byte:
  1354. // 0x1B: Set ISO-2022-JP decoder state to escape start and return error.
  1355. if (byte == 0x1B) {
  1356. decoder_state = ISO2022JPState::EscapeStart;
  1357. TRY(on_code_point(replacement_code_point));
  1358. continue;
  1359. }
  1360. // 0x21 to 0x7E:
  1361. if (byte.has_value() && byte.value() >= 0x21 && byte.value() <= 0x7E) {
  1362. // 1. Set the ISO-2022-JP decoder state to lead byte.
  1363. decoder_state = ISO2022JPState::LeadByte;
  1364. // 2. Let pointer be (ISO-2022-JP lead − 0x21) × 94 + byte − 0x21.
  1365. u32 pointer = (iso2022_jp_lead - 0x21) * 94 + byte.value() - 0x21;
  1366. // 3. Let code point be the index code point for pointer in index jis0208.
  1367. auto code_point = index_jis0208_code_point(pointer);
  1368. // 4. If code point is null, return error.
  1369. if (!code_point.has_value()) {
  1370. TRY(on_code_point(replacement_code_point));
  1371. continue;
  1372. }
  1373. // 5. Return a code point whose value is code point.
  1374. TRY(on_code_point(code_point.value()));
  1375. continue;
  1376. }
  1377. // end-of-queue: Set the ISO-2022-JP decoder state to lead byte and return error.
  1378. if (!byte.has_value()) {
  1379. decoder_state = ISO2022JPState::LeadByte;
  1380. TRY(on_code_point(replacement_code_point));
  1381. continue;
  1382. }
  1383. // Otherwise: Set ISO-2022-JP decoder state to lead byte and return error.
  1384. decoder_state = ISO2022JPState::LeadByte;
  1385. TRY(on_code_point(replacement_code_point));
  1386. break;
  1387. case ISO2022JPState::EscapeStart:
  1388. // 1. If byte is either 0x24 or 0x28, set ISO-2022-JP lead to byte, ISO-2022-JP decoder state to escape, and return continue.
  1389. if (byte == 0x24 || byte == 0x28) {
  1390. iso2022_jp_lead = byte.value();
  1391. decoder_state = ISO2022JPState::Escape;
  1392. continue;
  1393. }
  1394. // 2. If byte is not end-of-queue, then restore byte to ioQueue.
  1395. if (byte.has_value())
  1396. index--;
  1397. // 3. Set ISO-2022-JP output to false, ISO-2022-JP decoder state to ISO-2022-JP decoder output state, and return error.
  1398. iso2022_jp_output = false;
  1399. decoder_state = output_state;
  1400. TRY(on_code_point(replacement_code_point));
  1401. break;
  1402. case ISO2022JPState::Escape: {
  1403. // 1. Let lead be ISO-2022-JP lead and set ISO-2022-JP lead to 0x00.
  1404. auto lead = iso2022_jp_lead;
  1405. iso2022_jp_lead = 0x00;
  1406. // 2. Let state be null.
  1407. Optional<ISO2022JPState> state;
  1408. // 3. If lead is 0x28 and byte is 0x42, set state to ASCII.
  1409. if (lead == 0x28 && byte == 0x42)
  1410. state = ISO2022JPState::ASCII;
  1411. // 4. If lead is 0x28 and byte is 0x4A, set state to Roman.
  1412. if (lead == 0x28 && byte == 0x4A)
  1413. state = ISO2022JPState::Roman;
  1414. // 5. If lead is 0x28 and byte is 0x49, set state to katakana.
  1415. if (lead == 0x28 && byte == 0x49)
  1416. state = ISO2022JPState::Katakana;
  1417. // 6. If lead is 0x24 and byte is either 0x40 or 0x42, set state to lead byte.
  1418. if (lead == 0x24 && (byte == 0x40 || byte == 0x42))
  1419. state = ISO2022JPState::LeadByte;
  1420. // 7. If state is non-null, then:
  1421. if (state.has_value()) {
  1422. // 1. Set ISO-2022-JP decoder state and ISO-2022-JP decoder output state to state.
  1423. decoder_state = state.value();
  1424. output_state = state.value();
  1425. // 2. Let output be the value of ISO-2022-JP output.
  1426. auto output = iso2022_jp_output;
  1427. // 3. Set ISO-2022-JP output to true.
  1428. iso2022_jp_output = true;
  1429. // 4. Return continue, if output is false, and error otherwise.
  1430. if (output)
  1431. TRY(on_code_point(replacement_code_point));
  1432. continue;
  1433. }
  1434. // 8. If byte is end-of-queue, then restore lead to ioQueue; otherwise, restore « lead, byte » to ioQueue.
  1435. index -= byte.has_value() ? 2 : 1;
  1436. // 9. Set ISO-2022-JP output to false, ISO-2022-JP decoder state to ISO-2022-JP decoder output state and return error.
  1437. iso2022_jp_output = false;
  1438. decoder_state = output_state;
  1439. TRY(on_code_point(replacement_code_point));
  1440. break;
  1441. }
  1442. }
  1443. }
  1444. }
  1445. // https://encoding.spec.whatwg.org/#shift_jis-decoder
  1446. ErrorOr<void> ShiftJISDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
  1447. {
  1448. // Shift_JIS’s decoder has an associated Shift_JIS lead (initially 0x00).
  1449. u8 shift_jis_lead = 0x00;
  1450. // Shift_JIS’s decoder’s handler, given ioQueue and byte, runs these steps:
  1451. size_t index = 0;
  1452. while (true) {
  1453. // 1. If byte is end-of-queue and Shift_JIS lead is not 0x00, set Shift_JIS lead to 0x00 and return error.
  1454. if (index >= input.length() && shift_jis_lead != 0x00) {
  1455. shift_jis_lead = 0x00;
  1456. TRY(on_code_point(replacement_code_point));
  1457. continue;
  1458. }
  1459. // 2. If byte is end-of-queue and Shift_JIS lead is 0x00, return finished.
  1460. if (index >= input.length() && shift_jis_lead == 0x00)
  1461. return {};
  1462. u8 const byte = input[index++];
  1463. // 3. If Shift_JIS lead is not 0x00, let lead be Shift_JIS lead, let pointer be null, set Shift_JIS lead to 0x00, and then:
  1464. if (shift_jis_lead != 0x00) {
  1465. auto lead = shift_jis_lead;
  1466. Optional<u32> pointer;
  1467. shift_jis_lead = 0x00;
  1468. // 1. Let offset be 0x40 if byte is less than 0x7F, otherwise 0x41.
  1469. u8 const offset = byte < 0x7F ? 0x40 : 0x41;
  1470. // 2. Let lead offset be 0x81 if lead is less than 0xA0, otherwise 0xC1.
  1471. u8 const lead_offset = lead < 0xA0 ? 0x81 : 0xC1;
  1472. // 3. If byte is in the range 0x40 to 0x7E, inclusive, or 0x80 to 0xFC, inclusive, set pointer to (lead − lead offset) × 188 + byte − offset.
  1473. if ((byte >= 0x40 && byte <= 0x7E) || (byte >= 0x80 && byte <= 0xFC))
  1474. pointer = (lead - lead_offset) * 188 + byte - offset;
  1475. // 4. If pointer is in the range 8836 to 10715, inclusive, return a code point whose value is 0xE000 − 8836 + pointer.
  1476. if (pointer.has_value() && pointer.value() >= 8836 && pointer.value() <= 10715) {
  1477. TRY(on_code_point(0xE000 - 8836 + pointer.value()));
  1478. continue;
  1479. }
  1480. // 5. Let code point be null if pointer is null, otherwise the index code point for pointer in index jis0208.
  1481. auto code_point = pointer.has_value() ? index_jis0208_code_point(pointer.value()) : Optional<u32> {};
  1482. // 6. If code point is non-null, return a code point whose value is code point.
  1483. if (code_point.has_value()) {
  1484. TRY(on_code_point(code_point.value()));
  1485. continue;
  1486. }
  1487. // 7. If byte is an ASCII byte, restore byte to ioQueue.
  1488. if (byte <= 0x7F)
  1489. index--;
  1490. // 8. Return error.
  1491. TRY(on_code_point(replacement_code_point));
  1492. continue;
  1493. }
  1494. // 4. If byte is an ASCII byte or 0x80, return a code point whose value is byte.
  1495. if (byte <= 0x80) {
  1496. TRY(on_code_point(byte));
  1497. continue;
  1498. }
  1499. // 5. If byte is in the range 0xA1 to 0xDF, inclusive, return a code point whose value is 0xFF61 − 0xA1 + byte.
  1500. if (byte >= 0xA1 && byte <= 0xDF) {
  1501. TRY(on_code_point(0xFF61 - 0xA1 + byte));
  1502. continue;
  1503. }
  1504. // 6. If byte is in the range 0x81 to 0x9F, inclusive, or 0xE0 to 0xFC, inclusive, set Shift_JIS lead to byte and return continue.
  1505. if ((byte >= 0x81 && byte <= 0x9F) || (byte >= 0xE0 && byte <= 0xFC)) {
  1506. shift_jis_lead = byte;
  1507. continue;
  1508. }
  1509. // 7. Return error.
  1510. TRY(on_code_point(replacement_code_point));
  1511. }
  1512. }
  1513. // https://encoding.spec.whatwg.org/#euc-kr-decoder
  1514. ErrorOr<void> EUCKRDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
  1515. {
  1516. // EUC-KR’s decoder has an associated EUC-KR lead (initially 0x00).
  1517. u8 euc_kr_lead = 0x00;
  1518. // EUC-KR’s decoder’s handler, given ioQueue and byte, runs these steps:
  1519. size_t index = 0;
  1520. while (true) {
  1521. // 1. If byte is end-of-queue and EUC-KR lead is not 0x00, set EUC-KR lead to 0x00 and return error.
  1522. if (index >= input.length() && euc_kr_lead != 0x00) {
  1523. euc_kr_lead = 0x00;
  1524. TRY(on_code_point(replacement_code_point));
  1525. continue;
  1526. }
  1527. // 2. If byte is end-of-queue and EUC-KR lead is 0x00, return finished.
  1528. if (index >= input.length() && euc_kr_lead == 0x00)
  1529. return {};
  1530. u8 const byte = input[index++];
  1531. // 3. If EUC-KR lead is not 0x00, let lead be EUC-KR lead, let pointer be null, set EUC-KR lead to 0x00, and then:
  1532. if (euc_kr_lead != 0x00) {
  1533. auto lead = euc_kr_lead;
  1534. Optional<u32> pointer;
  1535. euc_kr_lead = 0x00;
  1536. // 1. If byte is in the range 0x41 to 0xFE, inclusive, set pointer to (lead − 0x81) × 190 + (byte − 0x41).
  1537. if (byte >= 0x41 && byte <= 0xFE)
  1538. pointer = (lead - 0x81) * 190 + (byte - 0x41);
  1539. // 2. Let code point be null if pointer is null, otherwise the index code point for pointer in index EUC-KR.
  1540. auto code_point = pointer.has_value() ? index_euc_kr_code_point(pointer.value()) : Optional<u32> {};
  1541. // 3. If code point is non-null, return a code point whose value is code point.
  1542. if (code_point.has_value()) {
  1543. TRY(on_code_point(code_point.value()));
  1544. continue;
  1545. }
  1546. // 4. If byte is an ASCII byte, restore byte to ioQueue.
  1547. if (byte <= 0x7F)
  1548. index--;
  1549. // 5. Return error.
  1550. TRY(on_code_point(replacement_code_point));
  1551. continue;
  1552. }
  1553. // 4. If byte is an ASCII byte, return a code point whose value is byte.
  1554. if (byte <= 0x7F) {
  1555. TRY(on_code_point(byte));
  1556. continue;
  1557. }
  1558. // 5. If byte is in the range 0x81 to 0xFE, inclusive, set EUC-KR lead to byte and return continue.
  1559. if (byte >= 0x81 && byte <= 0xFE) {
  1560. euc_kr_lead = byte;
  1561. continue;
  1562. }
  1563. // 6. Return error.
  1564. TRY(on_code_point(replacement_code_point));
  1565. }
  1566. }
  1567. }