|
|
@@ -1,5 +1,5 @@
|
|
|
/*
|
|
|
- * Copyright (c) 2022, Tim Flynn <trflynn89@serenityos.org>
|
|
|
+ * Copyright (c) 2022-2024, Tim Flynn <trflynn89@serenityos.org>
|
|
|
*
|
|
|
* SPDX-License-Identifier: BSD-2-Clause
|
|
|
*/
|
|
|
@@ -17,175 +17,54 @@
|
|
|
#include <LibUnicode/Emoji.h>
|
|
|
|
|
|
struct Emoji {
|
|
|
- size_t name { 0 };
|
|
|
- Optional<size_t> image_path;
|
|
|
- Unicode::EmojiGroup group;
|
|
|
- ByteString subgroup;
|
|
|
- u32 display_order { 0 };
|
|
|
+ size_t image_path { 0 };
|
|
|
Vector<u32> code_points;
|
|
|
- ByteString encoded_code_points;
|
|
|
- ByteString status;
|
|
|
size_t code_point_array_index { 0 };
|
|
|
};
|
|
|
|
|
|
struct EmojiData {
|
|
|
UniqueStringStorage unique_strings;
|
|
|
Vector<Emoji> emojis;
|
|
|
- Vector<String> emoji_file_list;
|
|
|
+ Vector<ByteString> emoji_file_list;
|
|
|
};
|
|
|
|
|
|
-static void set_image_path_for_emoji(StringView emoji_resource_path, EmojiData& emoji_data, Emoji& emoji)
|
|
|
-{
|
|
|
- StringBuilder builder;
|
|
|
-
|
|
|
- for (auto code_point : emoji.code_points) {
|
|
|
- if (code_point == 0xfe0f)
|
|
|
- continue;
|
|
|
- if (!builder.is_empty())
|
|
|
- builder.append('_');
|
|
|
- builder.appendff("U+{:X}", code_point);
|
|
|
- }
|
|
|
-
|
|
|
- auto file = ByteString::formatted("{}.png", builder.to_byte_string());
|
|
|
- auto path = ByteString::formatted("{}/{}", emoji_resource_path, file);
|
|
|
- if (!FileSystem::exists(path))
|
|
|
- return;
|
|
|
-
|
|
|
- emoji.image_path = emoji_data.unique_strings.ensure(move(file));
|
|
|
-}
|
|
|
-
|
|
|
-static ErrorOr<void> parse_emoji_test_data(Core::InputBufferedFile& file, EmojiData& emoji_data)
|
|
|
+static ErrorOr<void> parse_emoji_file_list(Core::InputBufferedFile& file, EmojiData& emoji_data)
|
|
|
{
|
|
|
- static constexpr auto group_header = "# group: "sv;
|
|
|
- static constexpr auto subgroup_header = "# subgroup: "sv;
|
|
|
-
|
|
|
+ HashTable<ByteString> seen_emojis;
|
|
|
Array<u8, 1024> buffer;
|
|
|
|
|
|
- Unicode::EmojiGroup group;
|
|
|
- ByteString subgroup;
|
|
|
- u32 display_order { 0 };
|
|
|
-
|
|
|
while (TRY(file.can_read_line())) {
|
|
|
auto line = TRY(file.read_line(buffer));
|
|
|
if (line.is_empty())
|
|
|
continue;
|
|
|
|
|
|
- if (line.starts_with('#')) {
|
|
|
- if (line.starts_with(group_header)) {
|
|
|
- auto name = line.substring_view(group_header.length());
|
|
|
- group = Unicode::emoji_group_from_string(name);
|
|
|
- } else if (line.starts_with(subgroup_header)) {
|
|
|
- subgroup = line.substring_view(subgroup_header.length());
|
|
|
- }
|
|
|
-
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- auto status_index = line.find(';');
|
|
|
- VERIFY(status_index.has_value());
|
|
|
-
|
|
|
- auto emoji_and_name_index = line.find('#', *status_index);
|
|
|
- VERIFY(emoji_and_name_index.has_value());
|
|
|
-
|
|
|
- Emoji emoji {};
|
|
|
- emoji.group = group;
|
|
|
- emoji.subgroup = subgroup;
|
|
|
- emoji.display_order = display_order++;
|
|
|
-
|
|
|
- auto code_points = line.substring_view(0, *status_index).split_view(' ');
|
|
|
- TRY(emoji.code_points.try_ensure_capacity(code_points.size()));
|
|
|
-
|
|
|
- for (auto code_point : code_points) {
|
|
|
- auto value = AK::StringUtils::convert_to_uint_from_hex<u32>(code_point);
|
|
|
- VERIFY(value.has_value());
|
|
|
-
|
|
|
- emoji.code_points.unchecked_append(*value);
|
|
|
+ if (seen_emojis.contains(line)) {
|
|
|
+ warnln("\x1b[1;31mError!\x1b[0m Duplicate emoji \x1b[35m{}\x1b[0m listed in emoji-file-list.txt.", line);
|
|
|
+ return Error::from_errno(EEXIST);
|
|
|
}
|
|
|
|
|
|
- auto emoji_and_name = line.substring_view(*emoji_and_name_index + 1);
|
|
|
-
|
|
|
- auto emoji_and_name_spaces = emoji_and_name.find_all(" "sv);
|
|
|
- VERIFY(emoji_and_name_spaces.size() > 2);
|
|
|
-
|
|
|
- auto name = emoji_and_name.substring_view(emoji_and_name_spaces[2]).trim_whitespace();
|
|
|
- emoji.name = emoji_data.unique_strings.ensure(name.to_titlecase_string());
|
|
|
- emoji.encoded_code_points = emoji_and_name.substring_view(0, emoji_and_name_spaces[1]).trim_whitespace();
|
|
|
- emoji.status = line.substring_view(*status_index + 1, *emoji_and_name_index - *status_index - 1).trim_whitespace();
|
|
|
+ ByteString emoji_file { line.trim_whitespace() };
|
|
|
+ emoji_data.emoji_file_list.append(emoji_file);
|
|
|
+ seen_emojis.set(emoji_file);
|
|
|
|
|
|
- TRY(emoji_data.emojis.try_append(move(emoji)));
|
|
|
- }
|
|
|
-
|
|
|
- return {};
|
|
|
-}
|
|
|
-
|
|
|
-static ErrorOr<void> parse_emoji_serenity_data(Core::InputBufferedFile& file, EmojiData& emoji_data)
|
|
|
-{
|
|
|
- static constexpr auto code_point_header = "U+"sv;
|
|
|
-
|
|
|
- Array<u8, 1024> buffer;
|
|
|
-
|
|
|
- auto display_order = static_cast<u32>(emoji_data.emojis.size()) + 1u;
|
|
|
-
|
|
|
- while (TRY(file.can_read_line())) {
|
|
|
- auto line = TRY(file.read_line(buffer));
|
|
|
- if (line.is_empty())
|
|
|
- continue;
|
|
|
-
|
|
|
- auto index = line.find(code_point_header);
|
|
|
- if (!index.has_value())
|
|
|
- continue;
|
|
|
-
|
|
|
- line = line.substring_view(*index);
|
|
|
- StringBuilder builder;
|
|
|
+ Emoji emoji;
|
|
|
+ emoji.image_path = emoji_data.unique_strings.ensure(emoji_file);
|
|
|
|
|
|
- Emoji emoji {};
|
|
|
- emoji.group = Unicode::EmojiGroup::SerenityOS;
|
|
|
- emoji.display_order = display_order++;
|
|
|
+ auto emoji_basename = LexicalPath::basename(emoji_file, LexicalPath::StripExtension::Yes);
|
|
|
|
|
|
- TRY(line.for_each_split_view(' ', SplitBehavior::Nothing, [&](auto segment) -> ErrorOr<void> {
|
|
|
- if (segment.starts_with(code_point_header)) {
|
|
|
- segment = segment.substring_view(code_point_header.length());
|
|
|
-
|
|
|
- auto code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(segment);
|
|
|
- VERIFY(code_point.has_value());
|
|
|
-
|
|
|
- TRY(emoji.code_points.try_append(*code_point));
|
|
|
- } else {
|
|
|
- if (!builder.is_empty())
|
|
|
- TRY(builder.try_append(' '));
|
|
|
- TRY(builder.try_append(segment));
|
|
|
- }
|
|
|
- return {};
|
|
|
- }));
|
|
|
-
|
|
|
- auto name = builder.to_byte_string();
|
|
|
- if (!any_of(name, is_ascii_lower_alpha))
|
|
|
- name = name.to_titlecase();
|
|
|
-
|
|
|
- emoji.name = emoji_data.unique_strings.ensure(move(name));
|
|
|
- TRY(emoji_data.emojis.try_append(move(emoji)));
|
|
|
- }
|
|
|
-
|
|
|
- return {};
|
|
|
-}
|
|
|
+ emoji_basename.view().for_each_split_view('_', SplitBehavior::Nothing, [&](StringView code_point) {
|
|
|
+ static constexpr auto code_point_header = "U+"sv;
|
|
|
|
|
|
-static ErrorOr<void> parse_emoji_file_list(Core::InputBufferedFile& file, EmojiData& emoji_data)
|
|
|
-{
|
|
|
- HashTable<String> seen_emojis;
|
|
|
- Array<u8, 1024> buffer;
|
|
|
+ VERIFY(code_point.starts_with(code_point_header));
|
|
|
+ code_point = code_point.substring_view(code_point_header.length());
|
|
|
|
|
|
- while (TRY(file.can_read_line())) {
|
|
|
- auto line = TRY(file.read_line(buffer));
|
|
|
- if (line.is_empty())
|
|
|
- continue;
|
|
|
+ auto code_point_value = AK::StringUtils::convert_to_uint_from_hex<u32>(code_point);
|
|
|
+ VERIFY(code_point_value.has_value());
|
|
|
|
|
|
- if (seen_emojis.contains(line)) {
|
|
|
- warnln("\x1b[1;31mError!\x1b[0m Duplicate emoji \x1b[35m{}\x1b[0m listed in emoji-file-list.txt.", line);
|
|
|
- return Error::from_errno(EEXIST);
|
|
|
- }
|
|
|
+ emoji.code_points.append(*code_point_value);
|
|
|
+ });
|
|
|
|
|
|
- emoji_data.emoji_file_list.append(TRY(String::from_utf8(line)));
|
|
|
- seen_emojis.set(emoji_data.emoji_file_list.last());
|
|
|
+ emoji_data.emojis.append(move(emoji));
|
|
|
}
|
|
|
|
|
|
return {};
|
|
|
@@ -220,7 +99,7 @@ static ErrorOr<void> validate_emoji(StringView emoji_resource_path, EmojiData& e
|
|
|
return Error::from_errno(ENOENT);
|
|
|
}
|
|
|
|
|
|
- if (!emoji_data.emoji_file_list.contains_slow(lexical_path.string().view())) {
|
|
|
+ if (!emoji_data.emoji_file_list.contains_slow(lexical_path.string())) {
|
|
|
warnln("\x1b[1;31mError!\x1b[0m Emoji entry for \x1b[35m{}\x1b[0m not found. Please check emoji-file-list.txt.", lexical_path);
|
|
|
return Error::from_errno(ENOENT);
|
|
|
}
|
|
|
@@ -263,9 +142,8 @@ namespace Unicode {
|
|
|
emoji_data.unique_strings.generate(generator);
|
|
|
|
|
|
size_t total_code_point_count { 0 };
|
|
|
- for (auto const& emoji : emoji_data.emojis) {
|
|
|
+ for (auto const& emoji : emoji_data.emojis)
|
|
|
total_code_point_count += emoji.code_points.size();
|
|
|
- }
|
|
|
generator.set("total_code_point_count", ByteString::number(total_code_point_count));
|
|
|
|
|
|
generator.append(R"~~~(
|
|
|
@@ -280,32 +158,16 @@ static constexpr Array<u32, @total_code_point_count@> s_emoji_code_points { {)~~
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- generator.append(" } };"sv);
|
|
|
+ generator.append(" } };\n"sv);
|
|
|
|
|
|
generator.append(R"~~~(
|
|
|
struct EmojiData {
|
|
|
- Emoji to_unicode_emoji() const
|
|
|
- {
|
|
|
- Emoji emoji {};
|
|
|
- emoji.name = decode_string(name);
|
|
|
- if (image_path != 0)
|
|
|
- emoji.image_path = decode_string(image_path);
|
|
|
- emoji.group = static_cast<EmojiGroup>(group);
|
|
|
- emoji.display_order = display_order;
|
|
|
- emoji.code_points = code_points();
|
|
|
-
|
|
|
- return emoji;
|
|
|
- }
|
|
|
-
|
|
|
constexpr ReadonlySpan<u32> code_points() const
|
|
|
{
|
|
|
return ReadonlySpan<u32>(s_emoji_code_points.data() + code_point_start, code_point_count);
|
|
|
}
|
|
|
|
|
|
- @string_index_type@ name { 0 };
|
|
|
@string_index_type@ image_path { 0 };
|
|
|
- u8 group { 0 };
|
|
|
- u32 display_order { 0 };
|
|
|
size_t code_point_start { 0 };
|
|
|
size_t code_point_count { 0 };
|
|
|
};
|
|
|
@@ -316,15 +178,12 @@ struct EmojiData {
|
|
|
static constexpr Array<EmojiData, @emojis_size@> s_emojis { {)~~~");
|
|
|
|
|
|
for (auto const& emoji : emoji_data.emojis) {
|
|
|
- generator.set("name"sv, ByteString::number(emoji.name));
|
|
|
- generator.set("image_path"sv, ByteString::number(emoji.image_path.value_or(0)));
|
|
|
- generator.set("group"sv, ByteString::number(to_underlying(emoji.group)));
|
|
|
- generator.set("display_order"sv, ByteString::number(emoji.display_order));
|
|
|
+ generator.set("image_path"sv, ByteString::number(emoji.image_path));
|
|
|
generator.set("code_point_start"sv, ByteString::number(emoji.code_point_array_index));
|
|
|
generator.set("code_point_count"sv, ByteString::number(emoji.code_points.size()));
|
|
|
|
|
|
generator.append(R"~~~(
|
|
|
- { @name@, @image_path@, @group@, @display_order@, @code_point_start@, @code_point_count@ },)~~~");
|
|
|
+ { @image_path@, @code_point_start@, @code_point_count@ },)~~~");
|
|
|
}
|
|
|
|
|
|
generator.append(R"~~~(
|
|
|
@@ -347,10 +206,10 @@ struct EmojiCodePointComparator {
|
|
|
}
|
|
|
};
|
|
|
|
|
|
-Optional<Emoji> find_emoji_for_code_points(ReadonlySpan<u32> code_points)
|
|
|
+Optional<StringView> emoji_image_for_code_points(ReadonlySpan<u32> code_points)
|
|
|
{
|
|
|
if (auto const* emoji = binary_search(s_emojis, code_points, nullptr, EmojiCodePointComparator {}))
|
|
|
- return emoji->to_unicode_emoji();
|
|
|
+ return decode_string(emoji->image_path);
|
|
|
return {};
|
|
|
}
|
|
|
|
|
|
@@ -361,127 +220,55 @@ Optional<Emoji> find_emoji_for_code_points(ReadonlySpan<u32> code_points)
|
|
|
return {};
|
|
|
}
|
|
|
|
|
|
-static ErrorOr<void> generate_emoji_installation(Core::InputBufferedFile& file, EmojiData const& emoji_data)
|
|
|
-{
|
|
|
- StringBuilder builder;
|
|
|
- SourceGenerator generator { builder };
|
|
|
-
|
|
|
- auto current_group = Unicode::EmojiGroup::Unknown;
|
|
|
- StringView current_subgroup;
|
|
|
-
|
|
|
- for (auto const& emoji : emoji_data.emojis) {
|
|
|
- if (!emoji.image_path.has_value())
|
|
|
- continue;
|
|
|
- if (emoji.group == Unicode::EmojiGroup::SerenityOS)
|
|
|
- continue; // SerenityOS emojis are in emoji-serenity.txt
|
|
|
-
|
|
|
- if (current_group != emoji.group) {
|
|
|
- if (!builder.is_empty())
|
|
|
- generator.append("\n"sv);
|
|
|
-
|
|
|
- generator.set("group"sv, Unicode::emoji_group_to_string(emoji.group));
|
|
|
- generator.append("# group: @group@\n");
|
|
|
-
|
|
|
- current_group = emoji.group;
|
|
|
- }
|
|
|
-
|
|
|
- if (current_subgroup != emoji.subgroup) {
|
|
|
- generator.set("subgroup"sv, emoji.subgroup);
|
|
|
- generator.append("\n# subgroup: @subgroup@\n");
|
|
|
-
|
|
|
- current_subgroup = emoji.subgroup;
|
|
|
- }
|
|
|
-
|
|
|
- generator.set("emoji"sv, emoji.encoded_code_points);
|
|
|
- generator.set("name"sv, emoji_data.unique_strings.get(emoji.name));
|
|
|
- generator.set("status"sv, emoji.status);
|
|
|
-
|
|
|
- generator.append("@emoji@"sv);
|
|
|
- generator.append(" - "sv);
|
|
|
- generator.append(ByteString::join(" "sv, emoji.code_points, "U+{:X}"sv));
|
|
|
- generator.append(" @name@ (@status@)\n"sv);
|
|
|
- }
|
|
|
-
|
|
|
- TRY(file.write_until_depleted(generator.as_string_view().bytes()));
|
|
|
- return {};
|
|
|
-}
|
|
|
-
|
|
|
ErrorOr<int> serenity_main(Main::Arguments arguments)
|
|
|
{
|
|
|
StringView generated_header_path;
|
|
|
StringView generated_implementation_path;
|
|
|
- StringView generated_installation_path;
|
|
|
- StringView emoji_test_path;
|
|
|
- StringView emoji_serenity_path;
|
|
|
StringView emoji_file_list_path;
|
|
|
StringView emoji_resource_path;
|
|
|
|
|
|
Core::ArgsParser args_parser;
|
|
|
args_parser.add_option(generated_header_path, "Path to the Unicode Data header file to generate", "generated-header-path", 'h', "generated-header-path");
|
|
|
args_parser.add_option(generated_implementation_path, "Path to the Unicode Data implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
|
|
|
- args_parser.add_option(generated_installation_path, "Path to the emoji.txt file to generate", "generated-installation-path", 'i', "generated-installation-path");
|
|
|
- args_parser.add_option(emoji_test_path, "Path to emoji-test.txt file", "emoji-test-path", 'e', "emoji-test-path");
|
|
|
- args_parser.add_option(emoji_serenity_path, "Path to emoji-serenity.txt file", "emoji-serenity-path", 's', "emoji-serenity-path");
|
|
|
args_parser.add_option(emoji_file_list_path, "Path to the emoji-file-list.txt file", "emoji-file-list-path", 'f', "emoji-file-list-path");
|
|
|
args_parser.add_option(emoji_resource_path, "Path to the /res/emoji directory", "emoji-resource-path", 'r', "emoji-resource-path");
|
|
|
args_parser.parse(arguments);
|
|
|
|
|
|
VERIFY(!emoji_resource_path.is_empty() && FileSystem::exists(emoji_resource_path));
|
|
|
-
|
|
|
- auto emoji_test_file = TRY(open_file(emoji_test_path, Core::File::OpenMode::Read));
|
|
|
+ VERIFY(!emoji_file_list_path.is_empty() && FileSystem::exists(emoji_file_list_path));
|
|
|
|
|
|
EmojiData emoji_data {};
|
|
|
- TRY(parse_emoji_test_data(*emoji_test_file, emoji_data));
|
|
|
|
|
|
- if (!emoji_serenity_path.is_empty() && !emoji_file_list_path.is_empty()) {
|
|
|
- auto emoji_serenity_file = TRY(open_file(emoji_serenity_path, Core::File::OpenMode::Read));
|
|
|
- TRY(parse_emoji_serenity_data(*emoji_serenity_file, emoji_data));
|
|
|
+ auto emoji_file_list_file = TRY(open_file(emoji_file_list_path, Core::File::OpenMode::Read));
|
|
|
+ TRY(parse_emoji_file_list(*emoji_file_list_file, emoji_data));
|
|
|
|
|
|
- auto emoji_file_list_file = TRY(open_file(emoji_file_list_path, Core::File::OpenMode::Read));
|
|
|
- TRY(parse_emoji_file_list(*emoji_file_list_file, emoji_data));
|
|
|
+ TRY(validate_emoji(emoji_resource_path, emoji_data));
|
|
|
|
|
|
- TRY(validate_emoji(emoji_resource_path, emoji_data));
|
|
|
- }
|
|
|
+ auto generated_header_file = TRY(open_file(generated_header_path, Core::File::OpenMode::Write));
|
|
|
+ TRY(generate_emoji_data_header(*generated_header_file, emoji_data));
|
|
|
|
|
|
- for (auto& emoji : emoji_data.emojis)
|
|
|
- set_image_path_for_emoji(emoji_resource_path, emoji_data, emoji);
|
|
|
+ quick_sort(emoji_data.emojis, [](auto const& lhs, auto const& rhs) {
|
|
|
+ if (lhs.code_points.size() != rhs.code_points.size())
|
|
|
+ return lhs.code_points.size() < rhs.code_points.size();
|
|
|
|
|
|
- if (!generated_installation_path.is_empty()) {
|
|
|
- TRY(Core::Directory::create(LexicalPath { generated_installation_path }.parent(), Core::Directory::CreateDirectories::Yes));
|
|
|
+ for (size_t i = 0; i < lhs.code_points.size(); ++i) {
|
|
|
+ if (lhs.code_points[i] < rhs.code_points[i])
|
|
|
+ return true;
|
|
|
+ if (lhs.code_points[i] > rhs.code_points[i])
|
|
|
+ return false;
|
|
|
+ }
|
|
|
|
|
|
- auto generated_installation_file = TRY(open_file(generated_installation_path, Core::File::OpenMode::Write));
|
|
|
- TRY(generate_emoji_installation(*generated_installation_file, emoji_data));
|
|
|
- }
|
|
|
+ return false;
|
|
|
+ });
|
|
|
|
|
|
- if (!generated_header_path.is_empty()) {
|
|
|
- auto generated_header_file = TRY(open_file(generated_header_path, Core::File::OpenMode::Write));
|
|
|
- TRY(generate_emoji_data_header(*generated_header_file, emoji_data));
|
|
|
+ size_t code_point_array_index { 0 };
|
|
|
+ for (auto& emoji : emoji_data.emojis) {
|
|
|
+ emoji.code_point_array_index = code_point_array_index;
|
|
|
+ code_point_array_index += emoji.code_points.size();
|
|
|
}
|
|
|
|
|
|
- if (!generated_implementation_path.is_empty()) {
|
|
|
- quick_sort(emoji_data.emojis, [](auto const& lhs, auto const& rhs) {
|
|
|
- if (lhs.code_points.size() != rhs.code_points.size())
|
|
|
- return lhs.code_points.size() < rhs.code_points.size();
|
|
|
-
|
|
|
- for (size_t i = 0; i < lhs.code_points.size(); ++i) {
|
|
|
- if (lhs.code_points[i] < rhs.code_points[i])
|
|
|
- return true;
|
|
|
- if (lhs.code_points[i] > rhs.code_points[i])
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- return false;
|
|
|
- });
|
|
|
-
|
|
|
- size_t code_point_array_index { 0 };
|
|
|
- for (auto& emoji : emoji_data.emojis) {
|
|
|
- emoji.code_point_array_index = code_point_array_index;
|
|
|
- code_point_array_index += emoji.code_points.size();
|
|
|
- }
|
|
|
-
|
|
|
- auto generated_implementation_file = TRY(open_file(generated_implementation_path, Core::File::OpenMode::Write));
|
|
|
- TRY(generate_emoji_data_implementation(*generated_implementation_file, emoji_data));
|
|
|
- }
|
|
|
+ auto generated_implementation_file = TRY(open_file(generated_implementation_path, Core::File::OpenMode::Write));
|
|
|
+ TRY(generate_emoji_data_implementation(*generated_implementation_file, emoji_data));
|
|
|
|
|
|
return 0;
|
|
|
}
|
Timothy Flynn