GenerateEmojiData.cpp 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389
  1. /*
  2. * Copyright (c) 2022, Tim Flynn <trflynn89@serenityos.org>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include "GeneratorUtil.h"
  7. #include <AK/AnyOf.h>
  8. #include <AK/DeprecatedString.h>
  9. #include <AK/SourceGenerator.h>
  10. #include <AK/StringUtils.h>
  11. #include <AK/Types.h>
  12. #include <LibCore/ArgsParser.h>
  13. #include <LibCore/DeprecatedFile.h>
  14. #include <LibCore/Directory.h>
  15. #include <LibUnicode/Emoji.h>
  16. struct Emoji {
  17. size_t name { 0 };
  18. Optional<size_t> image_path;
  19. Unicode::EmojiGroup group;
  20. DeprecatedString subgroup;
  21. u32 display_order { 0 };
  22. Vector<u32> code_points;
  23. DeprecatedString encoded_code_points;
  24. DeprecatedString status;
  25. size_t code_point_array_index { 0 };
  26. };
  27. struct EmojiData {
  28. UniqueStringStorage unique_strings;
  29. Vector<Emoji> emojis;
  30. };
  31. static void set_image_path_for_emoji(StringView emoji_base_path, StringView emoji_resource_path, EmojiData& emoji_data, Emoji& emoji)
  32. {
  33. StringBuilder builder;
  34. for (auto code_point : emoji.code_points) {
  35. if (code_point == 0xfe0f)
  36. continue;
  37. if (!builder.is_empty())
  38. builder.append('_');
  39. builder.appendff("U+{:X}", code_point);
  40. }
  41. auto path = DeprecatedString::formatted("{}/{}.png", emoji_resource_path, builder.to_deprecated_string());
  42. if (!Core::DeprecatedFile::exists(path))
  43. return;
  44. auto installed_image_path = path.replace(emoji_base_path, {}, ReplaceMode::FirstOnly);
  45. emoji.image_path = emoji_data.unique_strings.ensure(move(installed_image_path));
  46. }
  47. static ErrorOr<void> parse_emoji_test_data(Core::BufferedFile& file, EmojiData& emoji_data)
  48. {
  49. static constexpr auto group_header = "# group: "sv;
  50. static constexpr auto subgroup_header = "# subgroup: "sv;
  51. Array<u8, 1024> buffer;
  52. Unicode::EmojiGroup group;
  53. DeprecatedString subgroup;
  54. u32 display_order { 0 };
  55. while (TRY(file.can_read_line())) {
  56. auto line = TRY(file.read_line(buffer));
  57. if (line.is_empty())
  58. continue;
  59. if (line.starts_with('#')) {
  60. if (line.starts_with(group_header)) {
  61. auto name = line.substring_view(group_header.length());
  62. group = Unicode::emoji_group_from_string(name);
  63. } else if (line.starts_with(subgroup_header)) {
  64. subgroup = line.substring_view(subgroup_header.length());
  65. }
  66. continue;
  67. }
  68. auto status_index = line.find(';');
  69. VERIFY(status_index.has_value());
  70. auto emoji_and_name_index = line.find('#', *status_index);
  71. VERIFY(emoji_and_name_index.has_value());
  72. Emoji emoji {};
  73. emoji.group = group;
  74. emoji.subgroup = subgroup;
  75. emoji.display_order = display_order++;
  76. auto code_points = line.substring_view(0, *status_index).split_view(' ');
  77. TRY(emoji.code_points.try_ensure_capacity(code_points.size()));
  78. for (auto code_point : code_points) {
  79. auto value = AK::StringUtils::convert_to_uint_from_hex<u32>(code_point);
  80. VERIFY(value.has_value());
  81. emoji.code_points.unchecked_append(*value);
  82. }
  83. auto emoji_and_name = line.substring_view(*emoji_and_name_index + 1);
  84. auto emoji_and_name_spaces = emoji_and_name.find_all(" "sv);
  85. VERIFY(emoji_and_name_spaces.size() > 2);
  86. auto name = emoji_and_name.substring_view(emoji_and_name_spaces[2]).trim_whitespace();
  87. emoji.name = emoji_data.unique_strings.ensure(name.to_titlecase_string());
  88. emoji.encoded_code_points = emoji_and_name.substring_view(0, emoji_and_name_spaces[1]).trim_whitespace();
  89. emoji.status = line.substring_view(*status_index + 1, *emoji_and_name_index - *status_index - 1).trim_whitespace();
  90. TRY(emoji_data.emojis.try_append(move(emoji)));
  91. }
  92. return {};
  93. }
  94. static ErrorOr<void> parse_emoji_serenity_data(Core::BufferedFile& file, EmojiData& emoji_data)
  95. {
  96. static constexpr auto code_point_header = "U+"sv;
  97. Array<u8, 1024> buffer;
  98. auto display_order = static_cast<u32>(emoji_data.emojis.size()) + 1u;
  99. while (TRY(file.can_read_line())) {
  100. auto line = TRY(file.read_line(buffer));
  101. if (line.is_empty())
  102. continue;
  103. auto index = line.find(code_point_header);
  104. if (!index.has_value())
  105. continue;
  106. line = line.substring_view(*index);
  107. StringBuilder builder;
  108. Emoji emoji {};
  109. emoji.group = Unicode::EmojiGroup::SerenityOS;
  110. emoji.display_order = display_order++;
  111. TRY(line.for_each_split_view(' ', SplitBehavior::Nothing, [&](auto segment) -> ErrorOr<void> {
  112. if (segment.starts_with(code_point_header)) {
  113. segment = segment.substring_view(code_point_header.length());
  114. auto code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(segment);
  115. VERIFY(code_point.has_value());
  116. TRY(emoji.code_points.try_append(*code_point));
  117. } else {
  118. if (!builder.is_empty())
  119. TRY(builder.try_append(' '));
  120. TRY(builder.try_append(segment));
  121. }
  122. return {};
  123. }));
  124. auto name = builder.to_deprecated_string();
  125. if (!any_of(name, is_ascii_lower_alpha))
  126. name = name.to_titlecase();
  127. emoji.name = emoji_data.unique_strings.ensure(move(name));
  128. TRY(emoji_data.emojis.try_append(move(emoji)));
  129. }
  130. return {};
  131. }
  132. static ErrorOr<void> generate_emoji_data_header(Core::BufferedFile& file, EmojiData const&)
  133. {
  134. StringBuilder builder;
  135. SourceGenerator generator { builder };
  136. TRY(file.write(generator.as_string_view().bytes()));
  137. return {};
  138. }
  139. static ErrorOr<void> generate_emoji_data_implementation(Core::BufferedFile& file, EmojiData const& emoji_data)
  140. {
  141. StringBuilder builder;
  142. SourceGenerator generator { builder };
  143. generator.set("string_index_type"sv, emoji_data.unique_strings.type_that_fits());
  144. generator.set("emojis_size"sv, DeprecatedString::number(emoji_data.emojis.size()));
  145. generator.append(R"~~~(
  146. #include <AK/Array.h>
  147. #include <AK/BinarySearch.h>
  148. #include <AK/Span.h>
  149. #include <AK/StringView.h>
  150. #include <AK/Types.h>
  151. #include <LibUnicode/Emoji.h>
  152. #include <LibUnicode/EmojiData.h>
  153. namespace Unicode {
  154. )~~~");
  155. emoji_data.unique_strings.generate(generator);
  156. size_t total_code_point_count { 0 };
  157. for (auto const& emoji : emoji_data.emojis) {
  158. total_code_point_count += emoji.code_points.size();
  159. }
  160. generator.set("total_code_point_count", DeprecatedString::number(total_code_point_count));
  161. generator.append(R"~~~(
  162. static constexpr Array<u32, @total_code_point_count@> s_emoji_code_points { {)~~~");
  163. bool first = true;
  164. for (auto const& emoji : emoji_data.emojis) {
  165. for (auto code_point : emoji.code_points) {
  166. generator.append(first ? " "sv : ", "sv);
  167. generator.append(DeprecatedString::formatted("{:#x}", code_point));
  168. first = false;
  169. }
  170. }
  171. generator.append(" } };"sv);
  172. generator.append(R"~~~(
  173. struct EmojiData {
  174. Emoji to_unicode_emoji() const
  175. {
  176. Emoji emoji {};
  177. emoji.name = decode_string(name);
  178. if (image_path != 0)
  179. emoji.image_path = decode_string(image_path);
  180. emoji.group = static_cast<EmojiGroup>(group);
  181. emoji.display_order = display_order;
  182. emoji.code_points = code_points();
  183. return emoji;
  184. }
  185. constexpr ReadonlySpan<u32> code_points() const
  186. {
  187. return ReadonlySpan<u32>(s_emoji_code_points.data() + code_point_start, code_point_count);
  188. }
  189. @string_index_type@ name { 0 };
  190. @string_index_type@ image_path { 0 };
  191. u8 group { 0 };
  192. u32 display_order { 0 };
  193. size_t code_point_start { 0 };
  194. size_t code_point_count { 0 };
  195. };
  196. )~~~");
  197. generator.append(R"~~~(
  198. static constexpr Array<EmojiData, @emojis_size@> s_emojis { {)~~~");
  199. for (auto const& emoji : emoji_data.emojis) {
  200. generator.set("name"sv, DeprecatedString::number(emoji.name));
  201. generator.set("image_path"sv, DeprecatedString::number(emoji.image_path.value_or(0)));
  202. generator.set("group"sv, DeprecatedString::number(to_underlying(emoji.group)));
  203. generator.set("display_order"sv, DeprecatedString::number(emoji.display_order));
  204. generator.set("code_point_start"sv, DeprecatedString::number(emoji.code_point_array_index));
  205. generator.set("code_point_count"sv, DeprecatedString::number(emoji.code_points.size()));
  206. generator.append(R"~~~(
  207. { @name@, @image_path@, @group@, @display_order@, @code_point_start@, @code_point_count@ },)~~~");
  208. }
  209. generator.append(R"~~~(
  210. } };
  211. Optional<Emoji> find_emoji_for_code_points(ReadonlySpan<u32> code_points)
  212. {
  213. for (auto& emoji : s_emojis) {
  214. if (emoji.code_points() == code_points)
  215. return emoji.to_unicode_emoji();
  216. }
  217. return {};
  218. }
  219. }
  220. )~~~");
  221. TRY(file.write(generator.as_string_view().bytes()));
  222. return {};
  223. }
  224. static ErrorOr<void> generate_emoji_installation(Core::BufferedFile& file, EmojiData const& emoji_data)
  225. {
  226. StringBuilder builder;
  227. SourceGenerator generator { builder };
  228. auto current_group = Unicode::EmojiGroup::Unknown;
  229. StringView current_subgroup;
  230. for (auto const& emoji : emoji_data.emojis) {
  231. if (!emoji.image_path.has_value())
  232. continue;
  233. if (emoji.group == Unicode::EmojiGroup::SerenityOS)
  234. continue; // SerenityOS emojis are in emoji-serenity.txt
  235. if (current_group != emoji.group) {
  236. if (!builder.is_empty())
  237. generator.append("\n"sv);
  238. generator.set("group"sv, Unicode::emoji_group_to_string(emoji.group));
  239. generator.append("# group: @group@\n");
  240. current_group = emoji.group;
  241. }
  242. if (current_subgroup != emoji.subgroup) {
  243. generator.set("subgroup"sv, emoji.subgroup);
  244. generator.append("\n# subgroup: @subgroup@\n");
  245. current_subgroup = emoji.subgroup;
  246. }
  247. generator.set("emoji"sv, emoji.encoded_code_points);
  248. generator.set("name"sv, emoji_data.unique_strings.get(emoji.name));
  249. generator.set("status"sv, emoji.status);
  250. generator.append("@emoji@"sv);
  251. generator.append(" - "sv);
  252. generator.append(DeprecatedString::join(" "sv, emoji.code_points, "U+{:X}"sv));
  253. generator.append(" @name@ (@status@)\n"sv);
  254. }
  255. TRY(file.write(generator.as_string_view().bytes()));
  256. return {};
  257. }
  258. ErrorOr<int> serenity_main(Main::Arguments arguments)
  259. {
  260. StringView generated_header_path;
  261. StringView generated_implementation_path;
  262. StringView generated_installation_path;
  263. StringView emoji_test_path;
  264. StringView emoji_serenity_path;
  265. StringView emoji_base_path;
  266. StringView emoji_resource_path;
  267. Core::ArgsParser args_parser;
  268. args_parser.add_option(generated_header_path, "Path to the Unicode Data header file to generate", "generated-header-path", 'h', "generated-header-path");
  269. args_parser.add_option(generated_implementation_path, "Path to the Unicode Data implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
  270. args_parser.add_option(generated_installation_path, "Path to the emoji.txt file to generate", "generated-installation-path", 'i', "generated-installation-path");
  271. args_parser.add_option(emoji_test_path, "Path to emoji-test.txt file", "emoji-test-path", 'e', "emoji-test-path");
  272. args_parser.add_option(emoji_serenity_path, "Path to emoji-serenity.txt file", "emoji-serenity-path", 's', "emoji-serenity-path");
  273. args_parser.add_option(emoji_base_path, "Path to the Base directory", "emoji-base-path", 'b', "emoji-base-path");
  274. args_parser.add_option(emoji_resource_path, "Path to the /res/emoji directory", "emoji-resource-path", 'r', "emoji-resource-path");
  275. args_parser.parse(arguments);
  276. VERIFY(!emoji_base_path.is_empty() && Core::DeprecatedFile::exists(emoji_base_path));
  277. VERIFY(!emoji_resource_path.is_empty() && Core::DeprecatedFile::exists(emoji_resource_path));
  278. auto emoji_test_file = TRY(open_file(emoji_test_path, Core::File::OpenMode::Read));
  279. EmojiData emoji_data {};
  280. TRY(parse_emoji_test_data(*emoji_test_file, emoji_data));
  281. if (!emoji_serenity_path.is_empty()) {
  282. auto emoji_serenity_file = TRY(open_file(emoji_serenity_path, Core::File::OpenMode::Read));
  283. TRY(parse_emoji_serenity_data(*emoji_serenity_file, emoji_data));
  284. }
  285. size_t code_point_array_index { 0 };
  286. for (auto& emoji : emoji_data.emojis) {
  287. emoji.code_point_array_index = code_point_array_index;
  288. code_point_array_index += emoji.code_points.size();
  289. set_image_path_for_emoji(emoji_base_path, emoji_resource_path, emoji_data, emoji);
  290. }
  291. if (!generated_header_path.is_empty()) {
  292. auto generated_header_file = TRY(open_file(generated_header_path, Core::File::OpenMode::Write));
  293. TRY(generate_emoji_data_header(*generated_header_file, emoji_data));
  294. }
  295. if (!generated_implementation_path.is_empty()) {
  296. auto generated_implementation_file = TRY(open_file(generated_implementation_path, Core::File::OpenMode::Write));
  297. TRY(generate_emoji_data_implementation(*generated_implementation_file, emoji_data));
  298. }
  299. if (!generated_installation_path.is_empty()) {
  300. TRY(Core::Directory::create(LexicalPath { generated_installation_path }.parent(), Core::Directory::CreateDirectories::Yes));
  301. auto generated_installation_file = TRY(open_file(generated_installation_path, Core::File::OpenMode::Write));
  302. TRY(generate_emoji_installation(*generated_installation_file, emoji_data));
  303. }
  304. return 0;
  305. }