beenull/ladybird
1
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2024-11-23 08:00:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6
ladybird/Userland/Libraries/LibTextCodec/Decoder.cpp
Andreas Kling 1a46d8df5f LibTextCodec: Use String::from_utf8() when decoding UTF-8 to UTF-8 
This way, we still perform UTF-8 validation, but don't go through the
slow generic code path that rebuilds the decoded string one code point
at a time.

This was a bottleneck when loading a canned copy of reddit.com, which
ended up being ~120 MiB large.

- Time spent decoding UTF-8 before this change: 1192 ms
- Time spent decoding UTF-8 after this change:  154 ms

That's still a long time, but 7.7x faster is nothing to sneeze at! :^)

Note that if the input fails UTF-8 validation, we still fall back to
the slow path and insert replacement characters per the WHATWG Encoding
spec: https://encoding.spec.whatwg.org/#utf-8-decode
2024-07-20 14:29:37 +02:00

1334 lines
56 KiB
C++
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/*
* Copyright (c) 2020, Andreas Kling <kling@serenityos.org>
* Copyright (c) 2022, Jelle Raaijmakers <jelle@gmta.nl>
* Copyright (c) 2023, Sam Atkins <atkinssj@serenityos.org>
* Copyright (c) 2024, Simon Wanner <simon@skyrising.xyz>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/BinarySearch.h>
#include <AK/StringBuilder.h>
#include <AK/Utf16View.h>
#include <AK/Utf8View.h>
#include <LibTextCodec/Decoder.h>
#include <LibTextCodec/LookupTables.h>
namespace TextCodec {
static constexpr u32 replacement_code_point = 0xfffd;
namespace {
Latin1Decoder s_latin1_decoder;
UTF8Decoder s_utf8_decoder;
UTF16BEDecoder s_utf16be_decoder;
UTF16LEDecoder s_utf16le_decoder;
PDFDocEncodingDecoder s_pdf_doc_encoding_decoder;
XUserDefinedDecoder s_x_user_defined_decoder;
GB18030Decoder s_gb18030_decoder;
Big5Decoder s_big5_decoder;
EUCJPDecoder s_euc_jp_decoder;
ISO2022JPDecoder s_iso_2022_jp_decoder;
ShiftJISDecoder s_shift_jis_decoder;
EUCKRDecoder s_euc_kr_decoder;
ReplacementDecoder s_replacement_decoder;
// s_{encoding}_index is generated from https://encoding.spec.whatwg.org/indexes.json
// Found separately in https://encoding.spec.whatwg.org/index-{encoding}.txt
SingleByteDecoder s_ibm866_decoder { s_ibm866_index };
SingleByteDecoder s_latin2_decoder { s_iso_8859_2_index };
SingleByteDecoder s_latin3_decoder { s_iso_8859_3_index };
SingleByteDecoder s_latin4_decoder { s_iso_8859_4_index };
SingleByteDecoder s_latin_cyrillic_decoder { s_iso_8859_5_index };
SingleByteDecoder s_latin_arabic_decoder { s_iso_8859_6_index };
SingleByteDecoder s_latin_greek_decoder { s_iso_8859_7_index };
SingleByteDecoder s_latin_hebrew_decoder { s_iso_8859_8_index };
SingleByteDecoder s_latin6_decoder { s_iso_8859_10_index };
SingleByteDecoder s_latin7_decoder { s_iso_8859_13_index };
SingleByteDecoder s_latin8_decoder { s_iso_8859_14_index };
SingleByteDecoder s_latin9_decoder { s_iso_8859_15_index };
SingleByteDecoder s_latin10_decoder { s_iso_8859_16_index };
SingleByteDecoder s_centraleurope_decoder { s_windows_1250_index };
SingleByteDecoder s_cyrillic_decoder { s_windows_1251_index };
SingleByteDecoder s_hebrew_decoder { s_windows_1255_index };
SingleByteDecoder s_koi8r_decoder { s_koi8_r_index };
SingleByteDecoder s_koi8u_decoder { s_koi8_u_index };
SingleByteDecoder s_mac_roman_decoder { s_macintosh_index };
SingleByteDecoder s_windows874_decoder { s_windows_874_index };
SingleByteDecoder s_windows1252_decoder { s_windows_1252_index };
SingleByteDecoder s_windows1253_decoder { s_windows_1253_index };
SingleByteDecoder s_turkish_decoder { s_windows_1254_index };
SingleByteDecoder s_windows1256_decoder { s_windows_1256_index };
SingleByteDecoder s_windows1257_decoder { s_windows_1257_index };
SingleByteDecoder s_windows1258_decoder { s_windows_1258_index };
SingleByteDecoder s_mac_cyrillic_decoder { s_x_mac_cyrillic_index };
}
Optional<Decoder&> decoder_for(StringView label)
{
auto encoding = get_standardized_encoding(label);
return encoding.has_value() ? decoder_for_exact_name(encoding.value()) : Optional<Decoder&> {};
}
Optional<Decoder&> decoder_for_exact_name(StringView encoding)
{
if (encoding.equals_ignoring_ascii_case("iso-8859-1"sv))
return s_latin1_decoder;
if (encoding.equals_ignoring_ascii_case("windows-1252"sv))
return s_windows1252_decoder;
if (encoding.equals_ignoring_ascii_case("utf-8"sv))
return s_utf8_decoder;
if (encoding.equals_ignoring_ascii_case("utf-16be"sv))
return s_utf16be_decoder;
if (encoding.equals_ignoring_ascii_case("utf-16le"sv))
return s_utf16le_decoder;
if (encoding.equals_ignoring_ascii_case("big5"sv))
return s_big5_decoder;
if (encoding.equals_ignoring_ascii_case("euc-jp"sv))
return s_euc_jp_decoder;
if (encoding.equals_ignoring_ascii_case("euc-kr"sv))
return s_euc_kr_decoder;
if (encoding.equals_ignoring_ascii_case("gbk"sv))
return s_gb18030_decoder;
if (encoding.equals_ignoring_ascii_case("gb18030"sv))
return s_gb18030_decoder;
if (encoding.equals_ignoring_ascii_case("ibm866"sv))
return s_ibm866_decoder;
if (encoding.equals_ignoring_ascii_case("iso-2022-jp"sv))
return s_iso_2022_jp_decoder;
if (encoding.equals_ignoring_ascii_case("iso-8859-2"sv))
return s_latin2_decoder;
if (encoding.equals_ignoring_ascii_case("iso-8859-3"sv))
return s_latin3_decoder;
if (encoding.equals_ignoring_ascii_case("iso-8859-4"sv))
return s_latin4_decoder;
if (encoding.equals_ignoring_ascii_case("iso-8859-5"sv))
return s_latin_cyrillic_decoder;
if (encoding.equals_ignoring_ascii_case("iso-8859-6"sv))
return s_latin_arabic_decoder;
if (encoding.equals_ignoring_ascii_case("iso-8859-7"sv))
return s_latin_greek_decoder;
if (encoding.is_one_of_ignoring_ascii_case("iso-8859-8"sv, "iso-8859-8-i"sv))
return s_latin_hebrew_decoder;
if (encoding.equals_ignoring_ascii_case("iso-8859-10"sv))
return s_latin6_decoder;
if (encoding.equals_ignoring_ascii_case("iso-8859-13"sv))
return s_latin7_decoder;
if (encoding.equals_ignoring_ascii_case("iso-8859-14"sv))
return s_latin8_decoder;
if (encoding.equals_ignoring_ascii_case("iso-8859-15"sv))
return s_latin9_decoder;
if (encoding.equals_ignoring_ascii_case("iso-8859-16"sv))
return s_latin10_decoder;
if (encoding.equals_ignoring_ascii_case("koi8-r"sv))
return s_koi8r_decoder;
if (encoding.equals_ignoring_ascii_case("koi8-u"sv))
return s_koi8u_decoder;
if (encoding.equals_ignoring_ascii_case("macintosh"sv))
return s_mac_roman_decoder;
if (encoding.equals_ignoring_ascii_case("PDFDocEncoding"sv))
return s_pdf_doc_encoding_decoder;
if (encoding.equals_ignoring_ascii_case("replacement"sv))
return s_replacement_decoder;
if (encoding.equals_ignoring_ascii_case("shift_jis"sv))
return s_shift_jis_decoder;
if (encoding.equals_ignoring_ascii_case("windows-874"sv))
return s_windows874_decoder;
if (encoding.equals_ignoring_ascii_case("windows-1250"sv))
return s_centraleurope_decoder;
if (encoding.equals_ignoring_ascii_case("windows-1251"sv))
return s_cyrillic_decoder;
if (encoding.equals_ignoring_ascii_case("windows-1253"sv))
return s_windows1253_decoder;
if (encoding.equals_ignoring_ascii_case("windows-1254"sv))
return s_turkish_decoder;
if (encoding.equals_ignoring_ascii_case("windows-1255"sv))
return s_hebrew_decoder;
if (encoding.equals_ignoring_ascii_case("windows-1256"sv))
return s_windows1256_decoder;
if (encoding.equals_ignoring_ascii_case("windows-1257"sv))
return s_windows1257_decoder;
if (encoding.equals_ignoring_ascii_case("windows-1258"sv))
return s_windows1258_decoder;
if (encoding.equals_ignoring_ascii_case("x-mac-cyrillic"sv))
return s_mac_cyrillic_decoder;
if (encoding.equals_ignoring_ascii_case("x-user-defined"sv))
return s_x_user_defined_decoder;
dbgln("TextCodec: No decoder implemented for encoding '{}'", encoding);
return {};
}
// https://encoding.spec.whatwg.org/#concept-encoding-get
Optional<StringView> get_standardized_encoding(StringView encoding)
{
// 1. Remove any leading and trailing ASCII whitespace from label.
// https://infra.spec.whatwg.org/#ascii-whitespace: ASCII whitespace is U+0009 TAB, U+000A LF, U+000C FF, U+000D CR, or U+0020 SPACE.
encoding = encoding.trim("\t\n\f\r "sv);
// 2. If label is an ASCII case-insensitive match for any of the labels listed in the table below, then return the corresponding encoding; otherwise return failure.
if (encoding.is_one_of_ignoring_ascii_case("unicode-1-1-utf-8"sv, "unicode11utf8"sv, "unicode20utf8"sv, "utf-8"sv, "utf8"sv, "x-unicode20utf8"sv))
return "UTF-8"sv;
if (encoding.is_one_of_ignoring_ascii_case("866"sv, "cp866"sv, "csibm866"sv, "ibm866"sv))
return "IBM866"sv;
if (encoding.is_one_of_ignoring_ascii_case("csisolatin2"sv, "iso-8859-2"sv, "iso-ir-101"sv, "iso8859-2"sv, "iso88592"sv, "iso_8859-2"sv, "iso_8859-2:1987"sv, "l2"sv, "latin2"sv))
return "ISO-8859-2"sv;
if (encoding.is_one_of_ignoring_ascii_case("csisolatin3"sv, "iso-8859-3"sv, "iso-ir-109"sv, "iso8859-3"sv, "iso88593"sv, "iso_8859-3"sv, "iso_8859-3:1988"sv, "l3"sv, "latin3"sv))
return "ISO-8859-3"sv;
if (encoding.is_one_of_ignoring_ascii_case("csisolatin4"sv, "iso-8859-4"sv, "iso-ir-110"sv, "iso8859-4"sv, "iso88594"sv, "iso_8859-4"sv, "iso_8859-4:1988"sv, "l4"sv, "latin4"sv))
return "ISO-8859-4"sv;
if (encoding.is_one_of_ignoring_ascii_case("csisolatincyrillic"sv, "cyrillic"sv, "iso-8859-5"sv, "iso-ir-144"sv, "iso8859-5"sv, "iso88595"sv, "iso_8859-5"sv, "iso_8859-5:1988"sv))
return "ISO-8859-5"sv;
if (encoding.is_one_of_ignoring_ascii_case("arabic"sv, "asmo-708"sv, "csiso88596e"sv, "csiso88596i"sv, "csisolatinarabic"sv, "ecma-114"sv, "iso-8859-6"sv, "iso-8859-6-e"sv, "iso-8859-6-i"sv, "iso-ir-127"sv, "iso8859-6"sv, "iso88596"sv, "iso_8859-6"sv, "iso_8859-6:1987"sv))
return "ISO-8859-6"sv;
if (encoding.is_one_of_ignoring_ascii_case("csisolatingreek"sv, "ecma-118"sv, "elot_928"sv, "greek"sv, "greek8"sv, "iso-8859-7"sv, "iso-ir-126"sv, "iso8859-7"sv, "iso88597"sv, "iso_8859-7"sv, "iso_8859-7:1987"sv, "sun_eu_greek"sv))
return "ISO-8859-7"sv;
if (encoding.is_one_of_ignoring_ascii_case("csiso88598e"sv, "csisolatinhebrew"sv, "hebrew"sv, "iso-8859-8"sv, "iso-8859-8-e"sv, "iso-ir-138"sv, "iso8859-8"sv, "iso88598"sv, "iso_8859-8"sv, "iso_8859-8:1988"sv, "visual"sv))
return "ISO-8859-8"sv;
if (encoding.is_one_of_ignoring_ascii_case("csiso88598i"sv, "iso-8859-8-i"sv, "logical"sv))
return "ISO-8859-8-I"sv;
if (encoding.is_one_of_ignoring_ascii_case("csisolatin6"sv, "iso-8859-10"sv, "iso-ir-157"sv, "iso8859-10"sv, "iso885910"sv, "l6"sv, "latin6"sv))
return "ISO-8859-10"sv;
if (encoding.is_one_of_ignoring_ascii_case("iso-8859-13"sv, "iso8859-13"sv, "iso885913"sv))
return "ISO-8859-13"sv;
if (encoding.is_one_of_ignoring_ascii_case("iso-8859-14"sv, "iso8859-14"sv, "iso885914"sv))
return "ISO-8859-14"sv;
if (encoding.is_one_of_ignoring_ascii_case("csisolatin9"sv, "iso-8859-15"sv, "iso8859-15"sv, "iso885915"sv, "iso_8859-15"sv, "l9"sv))
return "ISO-8859-15"sv;
if (encoding.is_one_of_ignoring_ascii_case("iso-8859-16"sv))
return "ISO-8859-16"sv;
if (encoding.is_one_of_ignoring_ascii_case("cskoi8r"sv, "koi"sv, "koi8"sv, "koi8-r"sv, "koi8_r"sv))
return "KOI8-R"sv;
if (encoding.is_one_of_ignoring_ascii_case("koi8-ru"sv, "koi8-u"sv))
return "KOI8-U"sv;
if (encoding.is_one_of_ignoring_ascii_case("csmacintosh"sv, "mac"sv, "macintosh"sv, "x-mac-roman"sv))
return "macintosh"sv;
if (encoding.is_one_of_ignoring_ascii_case("pdfdocencoding"sv))
return "PDFDocEncoding"sv;
if (encoding.is_one_of_ignoring_ascii_case("dos-874"sv, "iso-8859-11"sv, "iso8859-11"sv, "iso885911"sv, "tis-620"sv, "windows-874"sv))
return "windows-874"sv;
if (encoding.is_one_of_ignoring_ascii_case("cp1250"sv, "windows-1250"sv, "x-cp1250"sv))
return "windows-1250"sv;
if (encoding.is_one_of_ignoring_ascii_case("cp1251"sv, "windows-1251"sv, "x-cp1251"sv))
return "windows-1251"sv;
if (encoding.is_one_of_ignoring_ascii_case("ansi_x3.4-1968"sv, "ascii"sv, "cp1252"sv, "cp819"sv, "csisolatin1"sv, "ibm819"sv, "iso-8859-1"sv, "iso-ir-100"sv, "iso8859-1"sv, "iso88591"sv, "iso_8859-1"sv, "iso_8859-1:1987"sv, "l1"sv, "latin1"sv, "us-ascii"sv, "windows-1252"sv, "x-cp1252"sv))
return "windows-1252"sv;
if (encoding.is_one_of_ignoring_ascii_case("cp1253"sv, "windows-1253"sv, "x-cp1253"sv))
return "windows-1253"sv;
if (encoding.is_one_of_ignoring_ascii_case("cp1254"sv, "csisolatin5"sv, "iso-8859-9"sv, "iso-ir-148"sv, "iso8859-9"sv, "iso88599"sv, "iso_8859-9"sv, "iso_8859-9:1989"sv, "l5"sv, "latin5"sv, "windows-1254"sv, "x-cp1254"sv))
return "windows-1254"sv;
if (encoding.is_one_of_ignoring_ascii_case("cp1255"sv, "windows-1255"sv, "x-cp1255"sv))
return "windows-1255"sv;
if (encoding.is_one_of_ignoring_ascii_case("cp1256"sv, "windows-1256"sv, "x-cp1256"sv))
return "windows-1256"sv;
if (encoding.is_one_of_ignoring_ascii_case("cp1257"sv, "windows-1257"sv, "x-cp1257"sv))
return "windows-1257"sv;
if (encoding.is_one_of_ignoring_ascii_case("cp1258"sv, "windows-1258"sv, "x-cp1258"sv))
return "windows-1258"sv;
if (encoding.is_one_of_ignoring_ascii_case("x-mac-cyrillic"sv, "x-mac-ukrainian"sv))
return "x-mac-cyrillic"sv;
if (encoding.is_one_of_ignoring_ascii_case("koi8-r"sv, "koi8r"sv))
return "koi8-r"sv;
if (encoding.is_one_of_ignoring_ascii_case("chinese"sv, "csgb2312"sv, "csiso58gb231280"sv, "gb2312"sv, "gb_2312"sv, "gb_2312-80"sv, "gbk"sv, "iso-ir-58"sv, "x-gbk"sv))
return "GBK"sv;
if (encoding.is_one_of_ignoring_ascii_case("gb18030"sv))
return "gb18030"sv;
if (encoding.is_one_of_ignoring_ascii_case("big5"sv, "big5-hkscs"sv, "cn-big5"sv, "csbig5"sv, "x-x-big5"sv))
return "Big5"sv;
if (encoding.is_one_of_ignoring_ascii_case("cseucpkdfmtjapanese"sv, "euc-jp"sv, "x-euc-jp"sv))
return "EUC-JP"sv;
if (encoding.is_one_of_ignoring_ascii_case("csiso2022jp"sv, "iso-2022-jp"sv))
return "ISO-2022-JP"sv;
if (encoding.is_one_of_ignoring_ascii_case("csshiftjis"sv, "ms932"sv, "ms_kanji"sv, "shift-jis"sv, "shift_jis"sv, "sjis"sv, "windows-31j"sv, "x-sjis"sv))
return "Shift_JIS"sv;
if (encoding.is_one_of_ignoring_ascii_case("cseuckr"sv, "csksc56011987"sv, "euc-kr"sv, "iso-ir-149"sv, "korean"sv, "ks_c_5601-1987"sv, "ks_c_5601-1989"sv, "ksc5601"sv, "ksc_5601"sv, "windows-949"sv))
return "EUC-KR"sv;
if (encoding.is_one_of_ignoring_ascii_case("csiso2022kr"sv, "hz-gb-2312"sv, "iso-2022-cn"sv, "iso-2022-cn-ext"sv, "iso-2022-kr"sv, "replacement"sv))
return "replacement"sv;
if (encoding.is_one_of_ignoring_ascii_case("unicodefffe"sv, "utf-16be"sv))
return "UTF-16BE"sv;
if (encoding.is_one_of_ignoring_ascii_case("csunicode"sv, "iso-10646-ucs-2"sv, "ucs-2"sv, "unicode"sv, "unicodefeff"sv, "utf-16"sv, "utf-16le"sv))
return "UTF-16LE"sv;
if (encoding.is_one_of_ignoring_ascii_case("x-user-defined"sv))
return "x-user-defined"sv;
dbgln("TextCodec: Unrecognized encoding: {}", encoding);
return {};
}
// https://encoding.spec.whatwg.org/#bom-sniff
Optional<Decoder&> bom_sniff_to_decoder(StringView input)
{
// 1. Let BOM be the result of peeking 3 bytes from ioQueue, converted to a byte sequence.
// 2. For each of the rows in the table below, starting with the first one and going down,
// if BOM starts with the bytes given in the first column, then return the encoding given
// in the cell in the second column of that row. Otherwise, return null.
// Byte Order Mark | Encoding
// --------------------------
// 0xEF 0xBB 0xBF | UTF-8
// 0xFE 0xFF | UTF-16BE
// 0xFF 0xFE | UTF-16LE
auto bytes = input.bytes();
if (bytes.size() < 2)
return {};
auto first_byte = bytes[0];
switch (first_byte) {
case 0xEF: // UTF-8
if (bytes.size() < 3)
return {};
if (bytes[1] == 0xBB && bytes[2] == 0xBF)
return s_utf8_decoder;
return {};
case 0xFE: // UTF-16BE
if (bytes[1] == 0xFF)
return s_utf16be_decoder;
return {};
case 0xFF: // UTF-16LE
if (bytes[1] == 0xFE)
return s_utf16le_decoder;
return {};
}
return {};
}
// https://encoding.spec.whatwg.org/#decode
ErrorOr<String> convert_input_to_utf8_using_given_decoder_unless_there_is_a_byte_order_mark(Decoder& fallback_decoder, StringView input)
{
Decoder* actual_decoder = &fallback_decoder;
// 1. Let BOMEncoding be the result of BOM sniffing ioQueue.
// 2. If BOMEncoding is non-null:
if (auto unicode_decoder = bom_sniff_to_decoder(input); unicode_decoder.has_value()) {
// 1. Set encoding to BOMEncoding.
actual_decoder = &unicode_decoder.value();
// 2. Read three bytes from ioQueue, if BOMEncoding is UTF-8; otherwise read two bytes. (Do nothing with those bytes.)
// FIXME: I imagine this will be pretty slow for large inputs, as it's regenerating the input without the first 2/3 bytes.
input = input.substring_view(&unicode_decoder.value() == &s_utf8_decoder ? 3 : 2);
}
VERIFY(actual_decoder);
// 3. Process a queue with an instance of encoding’s decoder, ioQueue, output, and "replacement".
// FIXME: This isn't the exact same as the spec, which is written in terms of I/O queues.
auto output = TRY(actual_decoder->to_utf8(input));
// 4. Return output.
return output;
}
// https://encoding.spec.whatwg.org/#get-an-output-encoding
StringView get_output_encoding(StringView encoding)
{
// 1. If encoding is replacement or UTF-16BE/LE, then return UTF-8.
if (encoding.is_one_of_ignoring_ascii_case("replacement"sv, "utf-16le"sv, "utf-16be"sv))
return "UTF-8"sv;
// 2. Return encoding.
return encoding;
}
bool Decoder::validate(StringView input)
{
auto result = this->process(input, [](auto code_point) -> ErrorOr<void> {
if (code_point == replacement_code_point)
return Error::from_errno(EINVAL);
return {};
});
return !result.is_error();
}
ErrorOr<String> Decoder::to_utf8(StringView input)
{
StringBuilder builder(input.length());
TRY(process(input, [&builder](u32 c) { return builder.try_append_code_point(c); }));
return builder.to_string_without_validation();
}
ErrorOr<void> UTF8Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
for (auto c : Utf8View(input)) {
TRY(on_code_point(c));
}
return {};
}
bool UTF8Decoder::validate(StringView input)
{
return Utf8View(input).validate();
}
ErrorOr<String> UTF8Decoder::to_utf8(StringView input)
{
// Discard the BOM
auto bomless_input = input;
if (auto bytes = input.bytes(); bytes.size() >= 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF) {
bomless_input = input.substring_view(3);
}
if (Utf8View(bomless_input).validate())
return String::from_utf8_without_validation(bomless_input.bytes());
return Decoder::to_utf8(bomless_input);
}
static Utf16View as_utf16(StringView view, AK::Endianness endianness)
{
return Utf16View {
{ reinterpret_cast<u16 const*>(view.bytes().data()), view.length() / 2 },
endianness
};
}
ErrorOr<void> UTF16BEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
for (auto code_point : as_utf16(input, AK::Endianness::Big))
TRY(on_code_point(code_point));
return {};
}
bool UTF16BEDecoder::validate(StringView input)
{
return as_utf16(input, AK::Endianness::Big).validate();
}
ErrorOr<String> UTF16BEDecoder::to_utf8(StringView input)
{
// Discard the BOM
if (auto bytes = input.bytes(); bytes.size() >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF)
input = input.substring_view(2);
return String::from_utf16(as_utf16(input, AK::Endianness::Big));
}
ErrorOr<void> UTF16LEDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
for (auto code_point : as_utf16(input, AK::Endianness::Little))
TRY(on_code_point(code_point));
return {};
}
bool UTF16LEDecoder::validate(StringView input)
{
return as_utf16(input, AK::Endianness::Little).validate();
}
ErrorOr<String> UTF16LEDecoder::to_utf8(StringView input)
{
// Discard the BOM
if (auto bytes = input.bytes(); bytes.size() >= 2 && bytes[0] == 0xFF && bytes[1] == 0xFE)
input = input.substring_view(2);
return String::from_utf16(as_utf16(input, AK::Endianness::Little));
}
ErrorOr<void> Latin1Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
for (u8 ch : input) {
// Latin1 is the same as the first 256 Unicode code_points, so no mapping is needed, just utf-8 encoding.
TRY(on_code_point(ch));
}
return {};
}
ErrorOr<void> PDFDocEncodingDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
// PDF 1.7 spec, Appendix D.2 "PDFDocEncoding Character Set"
// Character codes 0-8, 11-12, 14-23, 127, 159, 173 are not defined per spec.
// clang-format off
static constexpr Array<u32, 256> translation_table = {
0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007,
0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F,
0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0017, 0x0017,
0x02D8, 0x02C7, 0x02C6, 0x02D9, 0x02DD, 0x02DB, 0x02DA, 0x02DC,
0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027,
0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F,
0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037,
0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F,
0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F,
0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057,
0x0058, 0x0059, 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F,
0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067,
0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F,
0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077,
0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, 0x007E, 0xFFFC,
0x2022, 0x2020, 0x2021, 0x2026, 0x2014, 0x2013, 0x0192, 0x2044,
0x2039, 0x203A, 0x2212, 0x2030, 0x201E, 0x201C, 0x201D, 0x2018,
0x2019, 0x201A, 0x2122, 0xFB01, 0xFB02, 0x0141, 0x0152, 0x0160,
0x0178, 0x017D, 0x0131, 0x0142, 0x0153, 0x0161, 0x017E, 0xFFFC,
0x20AC, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7,
0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0xFFFC, 0x00AE, 0x00AF,
0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7,
0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF,
0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7,
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7,
0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF,
0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7,
0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7,
0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF,
};
// clang-format on
for (u8 ch : input)
TRY(on_code_point(translation_table[ch]));
return {};
}
// https://encoding.spec.whatwg.org/#x-user-defined-decoder
ErrorOr<void> XUserDefinedDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
auto convert_x_user_defined_to_utf8 = [](u8 ch) -> u32 {
// 2. If byte is an ASCII byte, return a code point whose value is byte.
// https://infra.spec.whatwg.org/#ascii-byte
// An ASCII byte is a byte in the range 0x00 (NUL) to 0x7F (DEL), inclusive.
// NOTE: This doesn't check for ch >= 0x00, as that would always be true due to being unsigned.
if (ch <= 0x7f)
return ch;
// 3. Return a code point whose value is 0xF780 + byte − 0x80.
return 0xF780 + ch - 0x80;
};
for (auto ch : input) {
TRY(on_code_point(convert_x_user_defined_to_utf8(ch)));
}
// 1. If byte is end-of-queue, return finished.
return {};
}
// https://encoding.spec.whatwg.org/#single-byte-decoder
template<Integral ArrayType>
ErrorOr<void> SingleByteDecoder<ArrayType>::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
for (u8 const byte : input) {
if (byte < 0x80) {
// 2. If byte is an ASCII byte, return a code point whose value is byte.
TRY(on_code_point(byte));
} else {
// 3. Let code point be the index code point for byte − 0x80 in index single-byte.
auto code_point = m_translation_table[byte - 0x80];
// 4. If code point is null, return error.
// NOTE: Error is communicated with 0xFFFD
// 5. Return a code point whose value is code point.
TRY(on_code_point(code_point));
}
}
// 1. If byte is end-of-queue, return finished.
return {};
}
// https://encoding.spec.whatwg.org/#index-gb18030-ranges-code-point
static Optional<u32> index_gb18030_ranges_code_point(u32 pointer)
{
// 1. If pointer is greater than 39419 and less than 189000, or pointer is greater than 1237575, return null.
if ((pointer > 39419 && pointer < 189000) || pointer > 1237575)
return {};
// 2. If pointer is 7457, return code point U+E7C7.
if (pointer == 7457)
return 0xE7C7;
// FIXME: Encoding specification is not updated to GB-18030-2022 yet (https://github.com/whatwg/encoding/issues/312)
// NOTE: This matches https://commits.webkit.org/266173@main
switch (pointer) {
case 19057:
return 0xE81E; // 82 35 90 37
case 19058:
return 0xE826; // 82 35 90 38
case 19059:
return 0xE82B; // 82 35 90 39
case 19060:
return 0xE82C; // 82 35 91 30
case 19061:
return 0xE832; // 82 35 91 31
case 19062:
return 0xE843; // 82 35 91 32
case 19063:
return 0xE854; // 82 35 91 33
case 19064:
return 0xE864; // 82 35 91 34
case 39076:
return 0xE78D; // 84 31 82 36
case 39077:
return 0xE78F; // 84 31 82 37
case 39078:
return 0xE78E; // 84 31 82 38
case 39079:
return 0xE790; // 84 31 82 39
case 39080:
return 0xE791; // 84 31 83 30
case 39081:
return 0xE792; // 84 31 83 31
case 39082:
return 0xE793; // 84 31 83 32
case 39083:
return 0xE794; // 84 31 83 33
case 39084:
return 0xE795; // 84 31 83 34
case 39085:
return 0xE796; // 84 31 83 35
default:
break;
}
// 3. Let offset be the last pointer in index gb18030 ranges that is less than or equal to pointer and let code point offset be its corresponding code point.
size_t last_index;
binary_search(s_gb18030_ranges, pointer, &last_index, [](auto const pointer, auto const& entry) {
return pointer - entry.pointer;
});
auto offset = s_gb18030_ranges[last_index].pointer;
auto code_point_offset = s_gb18030_ranges[last_index].code_point;
// 4. Return a code point whose value is code point offset + pointer − offset.
return code_point_offset + pointer - offset;
}
// https://encoding.spec.whatwg.org/#gb18030-decoder
ErrorOr<void> GB18030Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
// gb18030’s decoder has an associated gb18030 first, gb18030 second, and gb18030 third (all initially 0x00).
u8 first = 0x00;
u8 second = 0x00;
u8 third = 0x00;
// gb18030’s decoder’s handler, given ioQueue and byte, runs these steps:
size_t index = 0;
while (true) {
// 1. If byte is end-of-queue and gb18030 first, gb18030 second, and gb18030 third are 0x00, return finished.
if (index >= input.length() && first == 0x00 && second == 0x00 && third == 0x00)
return {};
// 2. If byte is end-of-queue, and gb18030 first, gb18030 second, or gb18030 third is not 0x00, set gb18030 first, gb18030 second, and gb18030 third to 0x00, and return error.
if (index >= input.length() && (first != 0x00 || second != 0x00 || third != 0x00)) {
first = 0x00;
second = 0x00;
third = 0x00;
TRY(on_code_point(replacement_code_point));
continue;
}
u8 const byte = input[index++];
// 3. If gb18030 third is not 0x00, then:
if (third != 0x00) {
// 1. If byte is not in the range 0x30 to 0x39, inclusive, then:
if (byte < 0x30 || byte > 0x39) {
// 1. Restore « gb18030 second, gb18030 third, byte » to ioQueue.
index -= 3;
// 2. Set gb18030 first, gb18030 second, and gb18030 third to 0x00.
first = 0x00;
second = 0x00;
third = 0x00;
// 3. Return error.
TRY(on_code_point(replacement_code_point));
continue;
}
// 2. Let code point be the index gb18030 ranges code point for ((gb18030 first − 0x81) × (10 × 126 × 10)) + ((gb18030 second − 0x30) × (10 × 126)) + ((gb18030 third − 0x81) × 10) + byte − 0x30.
auto code_point = index_gb18030_ranges_code_point(((first - 0x81) * (10 * 126 * 10)) + ((second - 0x30) * (10 * 126)) + ((third - 0x81) * 10) + byte - 0x30);
// 3. Set gb18030 first, gb18030 second, and gb18030 third to 0x00.
first = 0x00;
second = 0x00;
third = 0x00;
// 4. If code point is null, return error.
if (!code_point.has_value()) {
TRY(on_code_point(replacement_code_point));
continue;
}
// 5. Return a code point whose value is code point.
TRY(on_code_point(code_point.value()));
continue;
}
// 4. If gb18030 second is not 0x00, then:
if (second != 0x00) {
// 1. If byte is in the range 0x81 to 0xFE, inclusive, set gb18030 third to byte and return continue.
if (byte >= 0x81 && byte <= 0xFE) {
third = byte;
continue;
}
// 2. Restore « gb18030 second, byte » to ioQueue, set gb18030 first and gb18030 second to 0x00, and return error.
index -= 2;
first = 0x00;
second = 0x00;
TRY(on_code_point(replacement_code_point));
continue;
}
// 5. If gb18030 first is not 0x00, then:
if (first != 0x00) {
// 1. If byte is in the range 0x30 to 0x39, inclusive, set gb18030 second to byte and return continue.
if (byte >= 0x30 && byte <= 0x39) {
second = byte;
continue;
}
// 2. Let lead be gb18030 first, let pointer be null, and set gb18030 first to 0x00.
auto lead = first;
Optional<u32> pointer;
first = 0x00;
// 3. Let offset be 0x40 if byte is less than 0x7F, otherwise 0x41.
u8 const offset = byte < 0x7F ? 0x40 : 0x41;
// 4. If byte is in the range 0x40 to 0x7E, inclusive, or 0x80 to 0xFE, inclusive, set pointer to (lead − 0x81) × 190 + (byte − offset).
if ((byte >= 0x40 && byte <= 0x7E) || (byte >= 0x80 && byte <= 0xFE))
pointer = (lead - 0x81) * 190 + (byte - offset);
// 5. Let code point be null if pointer is null, otherwise the index code point for pointer in index gb18030.
auto code_point = pointer.has_value() ? index_gb18030_code_point(pointer.value()) : Optional<u32> {};
// 6. If code point is non-null, return a code point whose value is code point.
if (code_point.has_value()) {
TRY(on_code_point(code_point.value()));
continue;
}
// 7. If byte is an ASCII byte, restore byte to ioQueue.
if (byte <= 0x7F)
index--;
// 8. Return error.
TRY(on_code_point(replacement_code_point));
continue;
}
// 6. If byte is an ASCII byte, return a code point whose value is byte.
if (byte <= 0x7F) {
TRY(on_code_point(byte));
continue;
}
// 7. If byte is 0x80, return code point U+20AC.
if (byte == 0x80) {
TRY(on_code_point(0x20AC));
continue;
}
// 8. If byte is in the range 0x81 to 0xFE, inclusive, set gb18030 first to byte and return continue.
if (byte >= 0x81 && byte <= 0xFE) {
first = byte;
continue;
}
// 9. Return error.
TRY(on_code_point(replacement_code_point));
}
}
// https://encoding.spec.whatwg.org/#big5-decoder
ErrorOr<void> Big5Decoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
// Big5’s decoder has an associated Big5 lead (initially 0x00).
u8 big5_lead = 0x00;
// Big5’s decoder’s handler, given ioQueue and byte, runs these steps:
size_t index = 0;
while (true) {
// 1. If byte is end-of-queue and Big5 lead is not 0x00, set Big5 lead to 0x00 and return error.
if (index >= input.length() && big5_lead != 0x00) {
big5_lead = 0x00;
TRY(on_code_point(replacement_code_point));
continue;
}
// 2. If byte is end-of-queue and Big5 lead is 0x00, return finished.
if (index >= input.length() && big5_lead == 0x00)
return {};
u8 const byte = input[index++];
// 3. If Big5 lead is not 0x00, let lead be Big5 lead, let pointer be null, set Big5 lead to 0x00, and then:
if (big5_lead != 0x00) {
auto lead = big5_lead;
Optional<u32> pointer;
big5_lead = 0x00;
// 1. Let offset be 0x40 if byte is less than 0x7F, otherwise 0x62.
u8 const offset = byte < 0x7F ? 0x40 : 0x62;
// 2. If byte is in the range 0x40 to 0x7E, inclusive, or 0xA1 to 0xFE, inclusive, set pointer to (lead − 0x81) × 157 + (byte − offset).
if ((byte >= 0x40 && byte <= 0x7E) || (byte >= 0xA1 && byte <= 0xFE))
pointer = (lead - 0x81) * 157 + (byte - offset);
// 3. If there is a row in the table below whose first column is pointer, return the two code points listed in its second column (the third column is irrelevant):
if (pointer.has_value() && pointer.value() == 1133) {
TRY(on_code_point(0x00CA));
TRY(on_code_point(0x0304));
continue;
}
if (pointer.has_value() && pointer.value() == 1135) {
TRY(on_code_point(0x00CA));
TRY(on_code_point(0x030C));
continue;
}
if (pointer.has_value() && pointer.value() == 1164) {
TRY(on_code_point(0x00EA));
TRY(on_code_point(0x0304));
continue;
}
if (pointer.has_value() && pointer.value() == 1166) {
TRY(on_code_point(0x00EA));
TRY(on_code_point(0x030C));
continue;
}
// 4. Let code point be null if pointer is null, otherwise the index code point for pointer in index Big5.
auto code_pointer = pointer.has_value() ? index_big5_code_point(pointer.value()) : Optional<u32> {};
// 5. If code point is non-null, return a code point whose value is code point.
if (code_pointer.has_value()) {
TRY(on_code_point(code_pointer.value()));
continue;
}
// 6. If byte is an ASCII byte, restore byte to ioQueue.
if (byte <= 0x7F)
index--;
// 7. Return error.
TRY(on_code_point(replacement_code_point));
continue;
}
// 4. If byte is an ASCII byte, return a code point whose value is byte.
if (byte <= 0x7F) {
TRY(on_code_point(byte));
continue;
}
// 5. If byte is in the range 0x81 to 0xFE, inclusive, set Big5 lead to byte and return continue.
if (byte >= 0x81 && byte <= 0xFE) {
big5_lead = byte;
continue;
}
// 6. Return error
TRY(on_code_point(replacement_code_point));
}
}
// https://encoding.spec.whatwg.org/#euc-jp-decoder
ErrorOr<void> EUCJPDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
// EUC-JP’s decoder has an associated EUC-JP jis0212 (initially false) and EUC-JP lead (initially 0x00).
bool jis0212 = false;
u8 euc_jp_lead = 0x00;
// EUC-JP’s decoder’s handler, given ioQueue and byte, runs these steps:
size_t index = 0;
while (true) {
// 1. If byte is end-of-queue and EUC-JP lead is not 0x00, set EUC-JP lead to 0x00, and return error.
if (index >= input.length() && euc_jp_lead != 0x00) {
euc_jp_lead = 0x00;
TRY(on_code_point(replacement_code_point));
continue;
}
// 2. If byte is end-of-queue and EUC-JP lead is 0x00, return finished.
if (index >= input.length() && euc_jp_lead == 0x00)
return {};
u8 const byte = input[index++];
// 3. If EUC-JP lead is 0x8E and byte is in the range 0xA1 to 0xDF, inclusive, set EUC-JP lead to 0x00 and return a code point whose value is 0xFF61 − 0xA1 + byte.
if (euc_jp_lead == 0x8E && byte >= 0xA1 && byte <= 0xDF) {
euc_jp_lead = 0x00;
TRY(on_code_point(0xFF61 - 0xA1 + byte));
continue;
}
// 4. If EUC-JP lead is 0x8F and byte is in the range 0xA1 to 0xFE, inclusive, set EUC-JP jis0212 to true, set EUC-JP lead to byte, and return continue.
if (euc_jp_lead == 0x8F && byte >= 0xA1 && byte <= 0xFE) {
jis0212 = true;
euc_jp_lead = byte;
continue;
}
// 5. If EUC-JP lead is not 0x00, let lead be EUC-JP lead, set EUC-JP lead to 0x00, and then:
if (euc_jp_lead != 0x00) {
auto lead = euc_jp_lead;
euc_jp_lead = 0x00;
// 1. Let code point be null.
Optional<u32> code_point;
// 2. If lead and byte are both in the range 0xA1 to 0xFE, inclusive, then set code point to the index code point for (lead − 0xA1) × 94 + byte − 0xA1 in index jis0208 if EUC-JP jis0212 is false and in index jis0212 otherwise.
if (lead >= 0xA1 && lead <= 0xFE && byte >= 0xA1 && byte <= 0xFE) {
auto pointer = (lead - 0xA1) * 94 + byte - 0xA1;
code_point = jis0212 ? index_jis0212_code_point(pointer) : index_jis0208_code_point(pointer);
}
// 3. Set EUC-JP jis0212 to false.
jis0212 = false;
// 4. If code point is non-null, return a code point whose value is code point.
if (code_point.has_value()) {
TRY(on_code_point(code_point.value()));
continue;
}
// 5. If byte is an ASCII byte, restore byte to ioQueue.
if (byte <= 0x7F)
index--;
// 6. Return error.
TRY(on_code_point(replacement_code_point));
continue;
}
// 6. If byte is an ASCII byte, return a code point whose value is byte.
if (byte <= 0x7F) {
TRY(on_code_point(byte));
continue;
}
// 7. If byte is 0x8E, 0x8F, or in the range 0xA1 to 0xFE, inclusive, set EUC-JP lead to byte and return continue.
if (byte == 0x8E || byte == 0x8F || (byte >= 0xA1 && byte <= 0xFE)) {
euc_jp_lead = byte;
continue;
}
// 8. Return error.
TRY(on_code_point(replacement_code_point));
}
}
enum class ISO2022JPState {
ASCII,
Roman,
Katakana,
LeadByte,
TrailByte,
EscapeStart,
Escape,
};
// https://encoding.spec.whatwg.org/#iso-2022-jp-decoder
ErrorOr<void> ISO2022JPDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
// ISO-2022-JP’s decoder has an associated ISO-2022-JP decoder state (initially ASCII), ISO-2022-JP decoder output state (initially ASCII), ISO-2022-JP lead (initially 0x00), and ISO-2022-JP output (initially false).
auto decoder_state = ISO2022JPState::ASCII;
auto output_state = ISO2022JPState::ASCII;
u8 iso2022_jp_lead = 0x00;
bool iso2022_jp_output = false;
size_t index = 0;
while (true) {
Optional<u8> byte;
if (index < input.length())
byte = input[index++];
// ISO-2022-JP’s decoder’s handler, given ioQueue and byte, runs these steps, switching on ISO-2022-JP decoder state:
switch (decoder_state) {
case ISO2022JPState::ASCII:
// Based on byte:
// 0x1B: Set ISO-2022-JP decoder state to escape start and return continue.
if (byte == 0x1B) {
decoder_state = ISO2022JPState::EscapeStart;
continue;
}
// 0x00 to 0x7F, excluding 0x0E, 0x0F, and 0x1B: Set ISO-2022-JP output to false and return a code point whose value is byte.
if (byte.has_value() && byte.value() <= 0x7F && byte != 0x0E && byte != 0x0F) {
iso2022_jp_output = false;
TRY(on_code_point(byte.value()));
continue;
}
// end-of-queue: Return finished.
if (!byte.has_value())
return {};
// Otherwise: Set ISO-2022-JP output to false and return error.
iso2022_jp_output = false;
TRY(on_code_point(replacement_code_point));
break;
case ISO2022JPState::Roman:
// Based on byte:
// 0x1B: Set ISO-2022-JP decoder state to escape start and return continue.
if (byte == 0x1B) {
decoder_state = ISO2022JPState::EscapeStart;
continue;
}
// 0x5C: Set ISO-2022-JP output to false and return code point U+00A5.
if (byte == 0x5C) {
iso2022_jp_output = false;
TRY(on_code_point(0x00A5));
continue;
}
// 0x7E: Set ISO-2022-JP output to false and return code point U+203E.
if (byte == 0x7E) {
iso2022_jp_output = false;
TRY(on_code_point(0x203E));
continue;
}
// 0x00 to 0x7F, excluding 0x0E, 0x0F, 0x1B, 0x5C, and 0x7E: Set ISO-2022-JP output to false and return a code point whose value is byte.
if (byte.has_value() && byte.value() <= 0x7F && byte != 0x0E && byte != 0x0F) {
iso2022_jp_output = false;
TRY(on_code_point(byte.value()));
continue;
}
// end-of-queue: Return finished.
if (!byte.has_value())
return {};
// Otherwise: Set ISO-2022-JP output to false and return error.
iso2022_jp_output = false;
TRY(on_code_point(replacement_code_point));
break;
case ISO2022JPState::Katakana:
// Based on byte:
// 0x1B: Set ISO-2022-JP decoder state to escape start and return continue.
if (byte == 0x1B) {
decoder_state = ISO2022JPState::EscapeStart;
continue;
}
// 0x21 to 0x5F: Set ISO-2022-JP output to false and return a code point whose value is 0xFF61 − 0x21 + byte.
if (byte.has_value() && byte.value() >= 0x21 && byte.value() <= 0x5F) {
iso2022_jp_output = false;
TRY(on_code_point(0xFF61 - 0x21 + byte.value()));
continue;
}
// end-of-queue: Return finished.
if (!byte.has_value())
return {};
// Otherwise: Set ISO-2022-JP output to false and return error.
iso2022_jp_output = false;
TRY(on_code_point(replacement_code_point));
break;
case ISO2022JPState::LeadByte:
// Based on byte:
// 0x1B: Set ISO-2022-JP decoder state to escape start and return continue.
if (byte == 0x1B) {
decoder_state = ISO2022JPState::EscapeStart;
continue;
}
// 0x21 to 0x7E: Set ISO-2022-JP output to false, ISO-2022-JP lead to byte, ISO-2022-JP decoder state to trail byte, and return continue.
if (byte.has_value() && byte.value() >= 0x21 && byte.value() <= 0x7E) {
iso2022_jp_output = false;
iso2022_jp_lead = byte.value();
decoder_state = ISO2022JPState::TrailByte;
continue;
}
// end-of-queue: Return finished.
if (!byte.has_value())
return {};
// Otherwise: Set ISO-2022-JP output to false and return error.
iso2022_jp_output = false;
TRY(on_code_point(replacement_code_point));
break;
case ISO2022JPState::TrailByte:
// Based on byte:
// 0x1B: Set ISO-2022-JP decoder state to escape start and return error.
if (byte == 0x1B) {
decoder_state = ISO2022JPState::EscapeStart;
TRY(on_code_point(replacement_code_point));
continue;
}
// 0x21 to 0x7E:
if (byte.has_value() && byte.value() >= 0x21 && byte.value() <= 0x7E) {
// 1. Set the ISO-2022-JP decoder state to lead byte.
decoder_state = ISO2022JPState::LeadByte;
// 2. Let pointer be (ISO-2022-JP lead − 0x21) × 94 + byte − 0x21.
u32 pointer = (iso2022_jp_lead - 0x21) * 94 + byte.value() - 0x21;
// 3. Let code point be the index code point for pointer in index jis0208.
auto code_point = index_jis0208_code_point(pointer);
// 4. If code point is null, return error.
if (!code_point.has_value()) {
TRY(on_code_point(replacement_code_point));
continue;
}
// 5. Return a code point whose value is code point.
TRY(on_code_point(code_point.value()));
continue;
}
// end-of-queue: Set the ISO-2022-JP decoder state to lead byte and return error.
if (!byte.has_value()) {
decoder_state = ISO2022JPState::LeadByte;
TRY(on_code_point(replacement_code_point));
continue;
}
// Otherwise: Set ISO-2022-JP decoder state to lead byte and return error.
decoder_state = ISO2022JPState::LeadByte;
TRY(on_code_point(replacement_code_point));
break;
case ISO2022JPState::EscapeStart:
// 1. If byte is either 0x24 or 0x28, set ISO-2022-JP lead to byte, ISO-2022-JP decoder state to escape, and return continue.
if (byte == 0x24 || byte == 0x28) {
iso2022_jp_lead = byte.value();
decoder_state = ISO2022JPState::Escape;
continue;
}
// 2. If byte is not end-of-queue, then restore byte to ioQueue.
if (byte.has_value())
index--;
// 3. Set ISO-2022-JP output to false, ISO-2022-JP decoder state to ISO-2022-JP decoder output state, and return error.
iso2022_jp_output = false;
decoder_state = output_state;
TRY(on_code_point(replacement_code_point));
break;
case ISO2022JPState::Escape: {
// 1. Let lead be ISO-2022-JP lead and set ISO-2022-JP lead to 0x00.
auto lead = iso2022_jp_lead;
iso2022_jp_lead = 0x00;
// 2. Let state be null.
Optional<ISO2022JPState> state;
// 3. If lead is 0x28 and byte is 0x42, set state to ASCII.
if (lead == 0x28 && byte == 0x42)
state = ISO2022JPState::ASCII;
// 4. If lead is 0x28 and byte is 0x4A, set state to Roman.
if (lead == 0x28 && byte == 0x4A)
state = ISO2022JPState::Roman;
// 5. If lead is 0x28 and byte is 0x49, set state to katakana.
if (lead == 0x28 && byte == 0x49)
state = ISO2022JPState::Katakana;
// 6. If lead is 0x24 and byte is either 0x40 or 0x42, set state to lead byte.
if (lead == 0x24 && (byte == 0x40 || byte == 0x42))
state = ISO2022JPState::LeadByte;
// 7. If state is non-null, then:
if (state.has_value()) {
// 1. Set ISO-2022-JP decoder state and ISO-2022-JP decoder output state to state.
decoder_state = state.value();
output_state = state.value();
// 2. Let output be the value of ISO-2022-JP output.
auto output = iso2022_jp_output;
// 3. Set ISO-2022-JP output to true.
iso2022_jp_output = true;
// 4. Return continue, if output is false, and error otherwise.
if (output)
TRY(on_code_point(replacement_code_point));
continue;
}
// 8. If byte is end-of-queue, then restore lead to ioQueue; otherwise, restore « lead, byte » to ioQueue.
index -= byte.has_value() ? 2 : 1;
// 9. Set ISO-2022-JP output to false, ISO-2022-JP decoder state to ISO-2022-JP decoder output state and return error.
iso2022_jp_output = false;
decoder_state = output_state;
TRY(on_code_point(replacement_code_point));
break;
}
}
}
}
// https://encoding.spec.whatwg.org/#shift_jis-decoder
ErrorOr<void> ShiftJISDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
// Shift_JIS’s decoder has an associated Shift_JIS lead (initially 0x00).
u8 shift_jis_lead = 0x00;
// Shift_JIS’s decoder’s handler, given ioQueue and byte, runs these steps:
size_t index = 0;
while (true) {
// 1. If byte is end-of-queue and Shift_JIS lead is not 0x00, set Shift_JIS lead to 0x00 and return error.
if (index >= input.length() && shift_jis_lead != 0x00) {
shift_jis_lead = 0x00;
TRY(on_code_point(replacement_code_point));
continue;
}
// 2. If byte is end-of-queue and Shift_JIS lead is 0x00, return finished.
if (index >= input.length() && shift_jis_lead == 0x00)
return {};
u8 const byte = input[index++];
// 3. If Shift_JIS lead is not 0x00, let lead be Shift_JIS lead, let pointer be null, set Shift_JIS lead to 0x00, and then:
if (shift_jis_lead != 0x00) {
auto lead = shift_jis_lead;
Optional<u32> pointer;
shift_jis_lead = 0x00;
// 1. Let offset be 0x40 if byte is less than 0x7F, otherwise 0x41.
u8 const offset = byte < 0x7F ? 0x40 : 0x41;
// 2. Let lead offset be 0x81 if lead is less than 0xA0, otherwise 0xC1.
u8 const lead_offset = lead < 0xA0 ? 0x81 : 0xC1;
// 3. If byte is in the range 0x40 to 0x7E, inclusive, or 0x80 to 0xFC, inclusive, set pointer to (lead − lead offset) × 188 + byte − offset.
if ((byte >= 0x40 && byte <= 0x7E) || (byte >= 0x80 && byte <= 0xFC))
pointer = (lead - lead_offset) * 188 + byte - offset;
// 4. If pointer is in the range 8836 to 10715, inclusive, return a code point whose value is 0xE000 − 8836 + pointer.
if (pointer.has_value() && pointer.value() >= 8836 && pointer.value() <= 10715) {
TRY(on_code_point(0xE000 - 8836 + pointer.value()));
continue;
}
// 5. Let code point be null if pointer is null, otherwise the index code point for pointer in index jis0208.
auto code_point = pointer.has_value() ? index_jis0208_code_point(pointer.value()) : Optional<u32> {};
// 6. If code point is non-null, return a code point whose value is code point.
if (code_point.has_value()) {
TRY(on_code_point(code_point.value()));
continue;
}
// 7. If byte is an ASCII byte, restore byte to ioQueue.
if (byte <= 0x7F)
index--;
// 8. Return error.
TRY(on_code_point(replacement_code_point));
continue;
}
// 4. If byte is an ASCII byte or 0x80, return a code point whose value is byte.
if (byte <= 0x80) {
TRY(on_code_point(byte));
continue;
}
// 5. If byte is in the range 0xA1 to 0xDF, inclusive, return a code point whose value is 0xFF61 − 0xA1 + byte.
if (byte >= 0xA1 && byte <= 0xDF) {
TRY(on_code_point(0xFF61 - 0xA1 + byte));
continue;
}
// 6. If byte is in the range 0x81 to 0x9F, inclusive, or 0xE0 to 0xFC, inclusive, set Shift_JIS lead to byte and return continue.
if ((byte >= 0x81 && byte <= 0x9F) || (byte >= 0xE0 && byte <= 0xFC)) {
shift_jis_lead = byte;
continue;
}
// 7. Return error.
TRY(on_code_point(replacement_code_point));
}
}
// https://encoding.spec.whatwg.org/#euc-kr-decoder
ErrorOr<void> EUCKRDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
// EUC-KR’s decoder has an associated EUC-KR lead (initially 0x00).
u8 euc_kr_lead = 0x00;
// EUC-KR’s decoder’s handler, given ioQueue and byte, runs these steps:
size_t index = 0;
while (true) {
// 1. If byte is end-of-queue and EUC-KR lead is not 0x00, set EUC-KR lead to 0x00 and return error.
if (index >= input.length() && euc_kr_lead != 0x00) {
euc_kr_lead = 0x00;
TRY(on_code_point(replacement_code_point));
continue;
}
// 2. If byte is end-of-queue and EUC-KR lead is 0x00, return finished.
if (index >= input.length() && euc_kr_lead == 0x00)
return {};
u8 const byte = input[index++];
// 3. If EUC-KR lead is not 0x00, let lead be EUC-KR lead, let pointer be null, set EUC-KR lead to 0x00, and then:
if (euc_kr_lead != 0x00) {
auto lead = euc_kr_lead;
Optional<u32> pointer;
euc_kr_lead = 0x00;
// 1. If byte is in the range 0x41 to 0xFE, inclusive, set pointer to (lead − 0x81) × 190 + (byte − 0x41).
if (byte >= 0x41 && byte <= 0xFE)
pointer = (lead - 0x81) * 190 + (byte - 0x41);
// 2. Let code point be null if pointer is null, otherwise the index code point for pointer in index EUC-KR.
auto code_point = pointer.has_value() ? index_euc_kr_code_point(pointer.value()) : Optional<u32> {};
// 3. If code point is non-null, return a code point whose value is code point.
if (code_point.has_value()) {
TRY(on_code_point(code_point.value()));
continue;
}
// 4. If byte is an ASCII byte, restore byte to ioQueue.
if (byte <= 0x7F)
index--;
// 5. Return error.
TRY(on_code_point(replacement_code_point));
continue;
}
// 4. If byte is an ASCII byte, return a code point whose value is byte.
if (byte <= 0x7F) {
TRY(on_code_point(byte));
continue;
}
// 5. If byte is in the range 0x81 to 0xFE, inclusive, set EUC-KR lead to byte and return continue.
if (byte >= 0x81 && byte <= 0xFE) {
euc_kr_lead = byte;
continue;
}
// 6. Return error.
TRY(on_code_point(replacement_code_point));
}
}
// https://encoding.spec.whatwg.org/#replacement-decoder
ErrorOr<void> ReplacementDecoder::process(StringView input, Function<ErrorOr<void>(u32)> on_code_point)
{
// replacement’s decoder has an associated replacement error returned (initially false).
// replacement’s decoder’s handler, given ioQueue and byte, runs these steps:
// 1. If byte is end-of-queue, return finished.
// 2. If replacement error returned is false, set replacement error returned to true and return error.
// 3. Return finished.
if (!input.is_empty())
return on_code_point(replacement_code_point);
return {};
}
}
Reference in a new issue View git blame Copy permalink