beenull/ladybird
1
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2024-11-26 09:30:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

LibUnicode: Generate separate tables for Unicode properties

Browse source 
Similar to General Categories, this generates separate tables for the
Property list.
This commit is contained in:
Timothy Flynn 2021-08-10 08:27:28 -04:00 committed by Andreas Kling
parent b06c104076
commit 5ac23d244d
Notes: sideshowbarker 2024-07-18 07:06:57 +09:00 
Author: https://github.com/trflynn89
Commit: https://github.com/SerenityOS/serenity/commit/5ac23d244dc
Pull-request: https://github.com/SerenityOS/serenity/pull/9333
4 changed files with 111 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
Tests/LibUnicode/TestUnicodeCharacterTypes.cpp
View file

@ -305,3 +305,73 @@ TEST_CASE(general_category)
EXPECT(!Unicode::code_point_has_general_category(code_point, general_category_ll));
}
}
TEST_CASE(property)
{
auto property = [](StringView name) {
auto property = Unicode::property_from_string(name);
VERIFY(property.has_value());
return *property;
};
auto property_any = property("Any"sv);
auto property_assigned = property("Assigned"sv);
auto property_ascii = property("ASCII"sv);
auto property_white_space = property("White_Space"sv);
auto property_wspace = property("WSpace"sv);
auto property_space = property("space"sv);
EXPECT_EQ(property_white_space, property_wspace);
EXPECT_EQ(property_white_space, property_space);
auto property_emoji_presentation = property("Emoji_Presentation"sv);
auto property_epres = property("EPres"sv);
EXPECT_EQ(property_emoji_presentation, property_epres);
for (u32 code_point = 0; code_point <= 0x10ffff; code_point += 1000)
EXPECT(Unicode::code_point_has_property(code_point, property_any));
for (u32 code_point = 0x101d0; code_point <= 0x101fd; ++code_point) {
EXPECT(Unicode::code_point_has_property(code_point, property_any));
EXPECT(Unicode::code_point_has_property(code_point, property_assigned));
EXPECT(!Unicode::code_point_has_property(code_point, property_ascii));
EXPECT(!Unicode::code_point_has_property(code_point, property_white_space));
EXPECT(!Unicode::code_point_has_property(code_point, property_emoji_presentation));
}
for (u32 code_point = 0x101fe; code_point <= 0x1027f; ++code_point) {
EXPECT(Unicode::code_point_has_property(code_point, property_any));
EXPECT(!Unicode::code_point_has_property(code_point, property_assigned));
EXPECT(!Unicode::code_point_has_property(code_point, property_ascii));
EXPECT(!Unicode::code_point_has_property(code_point, property_white_space));
EXPECT(!Unicode::code_point_has_property(code_point, property_emoji_presentation));
}
for (u32 code_point = 0; code_point <= 0x7f; ++code_point) {
EXPECT(Unicode::code_point_has_property(code_point, property_any));
EXPECT(Unicode::code_point_has_property(code_point, property_assigned));
EXPECT(Unicode::code_point_has_property(code_point, property_ascii));
EXPECT(!Unicode::code_point_has_property(code_point, property_emoji_presentation));
}
for (u32 code_point = 0x9; code_point <= 0xd; ++code_point) {
EXPECT(Unicode::code_point_has_property(code_point, property_any));
EXPECT(Unicode::code_point_has_property(code_point, property_assigned));
EXPECT(Unicode::code_point_has_property(code_point, property_ascii));
EXPECT(Unicode::code_point_has_property(code_point, property_white_space));
EXPECT(!Unicode::code_point_has_property(code_point, property_emoji_presentation));
}
for (u32 code_point = 0x1f3e5; code_point <= 0x1f3f0; ++code_point) {
EXPECT(Unicode::code_point_has_property(code_point, property_any));
EXPECT(Unicode::code_point_has_property(code_point, property_assigned));
EXPECT(Unicode::code_point_has_property(code_point, property_emoji_presentation));
EXPECT(!Unicode::code_point_has_property(code_point, property_ascii));
EXPECT(!Unicode::code_point_has_property(code_point, property_white_space));
}
}

30
Userland/Libraries/LibUnicode/CharacterTypes.cpp
View file

@ -22,11 +22,6 @@ namespace Unicode {
#if ENABLE_UNICODE_DATA
static bool has_property(UnicodeData const& unicode_data, Property property)
{
return (unicode_data.properties & property) == property;
}
static bool is_final_code_point(Utf8View const& string, size_t index, size_t byte_length)
{
// C is preceded by a sequence consisting of a cased letter and then zero or more case-ignorable
@ -40,12 +35,8 @@ static bool is_final_code_point(Utf8View const& string, size_t index, size_t byt
size_t cased_letter_count = 0;
for (auto code_point : preceding_view) {
auto unicode_data = Detail::unicode_data_for_code_point(code_point);
if (!unicode_data.has_value())
return false;
bool is_cased = has_property(*unicode_data, Property::Cased);
bool is_case_ignorable = has_property(*unicode_data, Property::Case_Ignorable);
bool is_cased = code_point_has_property(code_point, Property::Cased);
bool is_case_ignorable = code_point_has_property(code_point, Property::Case_Ignorable);
if (is_cased && !is_case_ignorable)
++cased_letter_count;
@ -57,12 +48,8 @@ static bool is_final_code_point(Utf8View const& string, size_t index, size_t byt
return false;
for (auto code_point : following_view) {
auto unicode_data = Detail::unicode_data_for_code_point(code_point);
if (!unicode_data.has_value())
return false;
bool is_cased = has_property(*unicode_data, Property::Cased);
bool is_case_ignorable = has_property(*unicode_data, Property::Case_Ignorable);
bool is_cased = code_point_has_property(code_point, Property::Cased);
bool is_case_ignorable = code_point_has_property(code_point, Property::Case_Ignorable);
if (is_case_ignorable)
continue;
@ -227,14 +214,7 @@ Optional<Property> property_from_string([[maybe_unused]] StringView const& prope
bool code_point_has_property([[maybe_unused]] u32 code_point, [[maybe_unused]] Property property)
{
#if ENABLE_UNICODE_DATA
if (property == Property::Any)
return is_unicode(code_point);
auto unicode_data = Detail::unicode_data_for_code_point(code_point);
if (!unicode_data.has_value())
return false;
return has_property(*unicode_data, property);
return Detail::code_point_has_property(code_point, property);
#else
return false;
#endif

82
Userland/Libraries/LibUnicode/CodeGenerators/GenerateUnicodeData.cpp
View file

@ -68,7 +68,6 @@ struct CodePointData {
Optional<u32> simple_lowercase_mapping;
Optional<u32> simple_titlecase_mapping;
Vector<u32> special_casing_indices;
Vector<StringView> prop_list;
StringView script;
Vector<StringView> script_extensions;
};
@ -87,12 +86,11 @@ struct UnicodeData {
Vector<Alias> general_category_aliases;
// The Unicode standard defines additional properties (Any, Assigned, ASCII) which are not in
// any UCD file. Assigned is set as the default enum value 0 so "property & Assigned == Assigned"
// is always true. Any is not assigned code points here because this file only parses assigned
// code points, whereas Any will include unassigned code points.
// any UCD file. Assigned code point ranges are derived as this generator is executed.
// https://unicode.org/reports/tr18/#General_Category_Property
PropList prop_list {
{ "Any"sv, {} },
{ "Any"sv, { { 0, 0x10ffff } } },
{ "Assigned"sv, {} },
{ "ASCII"sv, { { 0, 0x7f } } },
};
Vector<Alias> prop_aliases;
@ -309,6 +307,10 @@ static void parse_unicode_data(Core::File& file, UnicodeData& unicode_data)
{
Optional<u32> code_point_range_start;
auto& assigned_code_points = unicode_data.prop_list.find("Assigned"sv)->value;
Optional<u32> assigned_code_point_range_start = 0;
u32 previous_code_point = 0;
auto assign_code_point_property = [&](u32 code_point, auto const& list, auto& property, StringView default_) {
using PropertyType = RemoveCVReference<decltype(property)>;
constexpr bool is_single_item = IsSame<PropertyType, StringView>;
@ -361,18 +363,31 @@ static void parse_unicode_data(Core::File& file, UnicodeData& unicode_data)
data.simple_lowercase_mapping = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[13]);
data.simple_titlecase_mapping = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[14]);
if (!assigned_code_point_range_start.has_value())
assigned_code_point_range_start = data.code_point;
if (data.name.starts_with("<"sv) && data.name.ends_with(", First>")) {
VERIFY(!code_point_range_start.has_value());
VERIFY(!code_point_range_start.has_value() && assigned_code_point_range_start.has_value());
code_point_range_start = data.code_point;
data.name = data.name.substring(1, data.name.length() - 9);
assigned_code_points.append({ *assigned_code_point_range_start, previous_code_point });
assigned_code_point_range_start.clear();
} else if (data.name.starts_with("<"sv) && data.name.ends_with(", Last>")) {
VERIFY(code_point_range_start.has_value());
unicode_data.code_point_ranges.append({ *code_point_range_start, data.code_point });
data.name = data.name.substring(1, data.name.length() - 8);
CodePointRange code_point_range { *code_point_range_start, data.code_point };
unicode_data.code_point_ranges.append(code_point_range);
assigned_code_points.append(code_point_range);
data.name = data.name.substring(1, data.name.length() - 8);
code_point_range_start.clear();
} else if ((data.code_point > 0) && (data.code_point - previous_code_point) != 1) {
VERIFY(assigned_code_point_range_start.has_value());
assigned_code_points.append({ *assigned_code_point_range_start, previous_code_point });
assigned_code_point_range_start = data.code_point;
}
for (auto const& casing : unicode_data.special_casing) {
@ -380,12 +395,13 @@ static void parse_unicode_data(Core::File& file, UnicodeData& unicode_data)
data.special_casing_indices.append(casing.index);
}
assign_code_point_property(data.code_point, unicode_data.prop_list, data.prop_list, "Assigned"sv);
assign_code_point_property(data.code_point, unicode_data.script_list, data.script, "Unknown"sv);
assign_code_point_property(data.code_point, unicode_data.script_extensions, data.script_extensions, {});
unicode_data.largest_special_casing_size = max(unicode_data.largest_special_casing_size, data.special_casing_indices.size());
unicode_data.largest_script_extensions_size = max(unicode_data.largest_script_extensions_size, data.script_extensions.size());
previous_code_point = data.code_point;
unicode_data.code_point_data.append(move(data));
}
}
@ -398,17 +414,15 @@ static void generate_unicode_data_header(Core::File& file, UnicodeData& unicode_
generator.set("special_casing_size", String::number(unicode_data.largest_special_casing_size));
generator.set("script_extensions_size", String::number(unicode_data.largest_script_extensions_size));
auto generate_enum = [&](StringView name, StringView default_, Vector<String> values, Vector<Alias> aliases = {}, bool as_bitmask = false) {
VERIFY(!as_bitmask || (values.size() <= 64));
auto generate_enum = [&](StringView name, StringView default_, Vector<String> values, Vector<Alias> aliases = {}) {
quick_sort(values);
quick_sort(aliases, [](auto& alias1, auto& alias2) { return alias1.alias < alias2.alias; });
generator.set("name", name);
generator.set("underlying", String::formatted("{}UnderlyingType", name));
generator.set("underlying_type", as_bitmask ? "u64"sv : "u8"sv);
generator.append(R"~~~(
using @underlying@ = @underlying_type@;
using @underlying@ = u8;
enum class @name@ : @underlying@ {)~~~");
@ -418,18 +432,10 @@ enum class @name@ : @underlying@ {)~~~");
@default@,)~~~");
}
u8 index = 0;
for (auto const& value : values) {
generator.set("value", value);
if (as_bitmask) {
generator.set("index", String::number(index++));
generator.append(R"~~~(
@value@ = static_cast<@underlying@>(1) << @index@,)~~~");
} else {
generator.append(R"~~~(
generator.append(R"~~~(
@value@,)~~~");
}
}
for (auto const& alias : aliases) {
@ -442,20 +448,6 @@ enum class @name@ : @underlying@ {)~~~");
generator.append(R"~~~(
};
)~~~");
if (as_bitmask) {
generator.append(R"~~~(
constexpr @name@ operator&(@name@ value1, @name@ value2)
{
return static_cast<@name@>(static_cast<@underlying@>(value1) & static_cast<@underlying@>(value2));
}
constexpr @name@ operator|(@name@ value1, @name@ value2)
{
return static_cast<@name@>(static_cast<@underlying@>(value1) | static_cast<@underlying@>(value2));
}
)~~~");
}
};
generator.append(R"~~~(
@ -471,7 +463,7 @@ namespace Unicode {
generate_enum("Locale"sv, "None"sv, move(unicode_data.locales));
generate_enum("Condition"sv, "None"sv, move(unicode_data.conditions));
generate_enum("GeneralCategory"sv, {}, unicode_data.general_categories.keys(), unicode_data.general_category_aliases);
generate_enum("Property"sv, "Assigned"sv, unicode_data.prop_list.keys(), unicode_data.prop_aliases, true);
generate_enum("Property"sv, {}, unicode_data.prop_list.keys(), unicode_data.prop_aliases);
generate_enum("Script"sv, {}, unicode_data.script_list.keys(), unicode_data.script_aliases);
generator.append(R"~~~(
@ -524,8 +516,6 @@ struct UnicodeData {
SpecialCasing const* special_casing[@special_casing_size@] {};
u32 special_casing_size { 0 };
Property properties { Property::Assigned };
Script script { Script::Unknown };
Script script_extensions[@script_extensions_size@];
u32 script_extensions_size { 0 };
@ -538,6 +528,7 @@ Optional<UnicodeData> unicode_data_for_code_point(u32 code_point);
bool code_point_has_general_category(u32 code_point, GeneralCategory general_category);
Optional<GeneralCategory> general_category_from_string(StringView const& general_category);
bool code_point_has_property(u32 code_point, Property property);
Optional<Property> property_from_string(StringView const& property);
Optional<Script> script_from_string(StringView const& script);
@ -639,14 +630,6 @@ static constexpr Array<UnicodeData, @code_point_data_size@> s_unicode_data { {)~
append_field("simple_lowercase_mapping", String::formatted("{:#x}", data.simple_lowercase_mapping.value_or(data.code_point)));
append_field("simple_titlecase_mapping", String::formatted("{:#x}", data.simple_titlecase_mapping.value_or(data.code_point)));
append_list_and_size(data.special_casing_indices, "&s_special_casing[{}]"sv);
bool first = true;
for (auto const& property : data.prop_list) {
generator.append(first ? ", " : " | ");
generator.append(String::formatted("Property::{}", property));
first = false;
}
generator.append(String::formatted(", Script::{}", data.script));
append_list_and_size(data.script_extensions, "Script::{}"sv);
generator.append(" },");
@ -724,6 +707,7 @@ static constexpr Array<Span<CodePointRange const>, @size@> @name@ { {)~~~");
};
append_prop_list("s_general_categories"sv, "s_general_category_{}"sv, unicode_data.general_categories);
append_prop_list("s_properties"sv, "s_property_{}"sv, unicode_data.prop_list);
generator.append(R"~~~(
static HashMap<u32, UnicodeData const*> const& ensure_code_point_map()
@ -812,7 +796,11 @@ Optional<GeneralCategory> general_category_from_string(StringView const& general
generator.append(R"~~~(
return {};
}
)~~~");
append_prop_search("Property"sv, "property"sv, "s_properties"sv);
generator.append(R"~~~(
Optional<Property> property_from_string(StringView const& property)
{
if (property == "Assigned"sv)

2
Userland/Libraries/LibUnicode/Forward.h
View file

@ -13,7 +13,7 @@ namespace Unicode {
enum class Condition : u8;
enum class GeneralCategory : u8;
enum class Locale : u8;
enum class Property : u64;
enum class Property : u8;
enum class Script : u8;
enum class WordBreakProperty : u8;