GenerateEmojiData.cpp 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459
  1. /*
  2. * Copyright (c) 2022, Tim Flynn <trflynn89@serenityos.org>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include "GeneratorUtil.h"
  7. #include <AK/AnyOf.h>
  8. #include <AK/DeprecatedString.h>
  9. #include <AK/QuickSort.h>
  10. #include <AK/SourceGenerator.h>
  11. #include <AK/StringUtils.h>
  12. #include <AK/Types.h>
  13. #include <LibCore/ArgsParser.h>
  14. #include <LibCore/DeprecatedFile.h>
  15. #include <LibCore/DirIterator.h>
  16. #include <LibCore/Directory.h>
  17. #include <LibUnicode/Emoji.h>
  18. struct Emoji {
  19. size_t name { 0 };
  20. Optional<size_t> image_path;
  21. Unicode::EmojiGroup group;
  22. DeprecatedString subgroup;
  23. u32 display_order { 0 };
  24. Vector<u32> code_points;
  25. DeprecatedString encoded_code_points;
  26. DeprecatedString status;
  27. size_t code_point_array_index { 0 };
  28. };
  29. struct EmojiData {
  30. UniqueStringStorage unique_strings;
  31. Vector<Emoji> emojis;
  32. };
  33. static void set_image_path_for_emoji(StringView emoji_resource_path, EmojiData& emoji_data, Emoji& emoji)
  34. {
  35. StringBuilder builder;
  36. for (auto code_point : emoji.code_points) {
  37. if (code_point == 0xfe0f)
  38. continue;
  39. if (!builder.is_empty())
  40. builder.append('_');
  41. builder.appendff("U+{:X}", code_point);
  42. }
  43. auto file = DeprecatedString::formatted("{}.png", builder.to_deprecated_string());
  44. auto path = DeprecatedString::formatted("{}/{}", emoji_resource_path, file);
  45. if (!Core::DeprecatedFile::exists(path))
  46. return;
  47. emoji.image_path = emoji_data.unique_strings.ensure(move(file));
  48. }
  49. static ErrorOr<void> parse_emoji_test_data(Core::BufferedFile& file, EmojiData& emoji_data)
  50. {
  51. static constexpr auto group_header = "# group: "sv;
  52. static constexpr auto subgroup_header = "# subgroup: "sv;
  53. Array<u8, 1024> buffer;
  54. Unicode::EmojiGroup group;
  55. DeprecatedString subgroup;
  56. u32 display_order { 0 };
  57. while (TRY(file.can_read_line())) {
  58. auto line = TRY(file.read_line(buffer));
  59. if (line.is_empty())
  60. continue;
  61. if (line.starts_with('#')) {
  62. if (line.starts_with(group_header)) {
  63. auto name = line.substring_view(group_header.length());
  64. group = Unicode::emoji_group_from_string(name);
  65. } else if (line.starts_with(subgroup_header)) {
  66. subgroup = line.substring_view(subgroup_header.length());
  67. }
  68. continue;
  69. }
  70. auto status_index = line.find(';');
  71. VERIFY(status_index.has_value());
  72. auto emoji_and_name_index = line.find('#', *status_index);
  73. VERIFY(emoji_and_name_index.has_value());
  74. Emoji emoji {};
  75. emoji.group = group;
  76. emoji.subgroup = subgroup;
  77. emoji.display_order = display_order++;
  78. auto code_points = line.substring_view(0, *status_index).split_view(' ');
  79. TRY(emoji.code_points.try_ensure_capacity(code_points.size()));
  80. for (auto code_point : code_points) {
  81. auto value = AK::StringUtils::convert_to_uint_from_hex<u32>(code_point);
  82. VERIFY(value.has_value());
  83. emoji.code_points.unchecked_append(*value);
  84. }
  85. auto emoji_and_name = line.substring_view(*emoji_and_name_index + 1);
  86. auto emoji_and_name_spaces = emoji_and_name.find_all(" "sv);
  87. VERIFY(emoji_and_name_spaces.size() > 2);
  88. auto name = emoji_and_name.substring_view(emoji_and_name_spaces[2]).trim_whitespace();
  89. emoji.name = emoji_data.unique_strings.ensure(name.to_titlecase_string());
  90. emoji.encoded_code_points = emoji_and_name.substring_view(0, emoji_and_name_spaces[1]).trim_whitespace();
  91. emoji.status = line.substring_view(*status_index + 1, *emoji_and_name_index - *status_index - 1).trim_whitespace();
  92. TRY(emoji_data.emojis.try_append(move(emoji)));
  93. }
  94. return {};
  95. }
  96. static ErrorOr<void> parse_emoji_serenity_data(Core::BufferedFile& file, EmojiData& emoji_data)
  97. {
  98. static constexpr auto code_point_header = "U+"sv;
  99. Array<u8, 1024> buffer;
  100. auto display_order = static_cast<u32>(emoji_data.emojis.size()) + 1u;
  101. while (TRY(file.can_read_line())) {
  102. auto line = TRY(file.read_line(buffer));
  103. if (line.is_empty())
  104. continue;
  105. auto index = line.find(code_point_header);
  106. if (!index.has_value())
  107. continue;
  108. line = line.substring_view(*index);
  109. StringBuilder builder;
  110. Emoji emoji {};
  111. emoji.group = Unicode::EmojiGroup::SerenityOS;
  112. emoji.display_order = display_order++;
  113. TRY(line.for_each_split_view(' ', SplitBehavior::Nothing, [&](auto segment) -> ErrorOr<void> {
  114. if (segment.starts_with(code_point_header)) {
  115. segment = segment.substring_view(code_point_header.length());
  116. auto code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(segment);
  117. VERIFY(code_point.has_value());
  118. TRY(emoji.code_points.try_append(*code_point));
  119. } else {
  120. if (!builder.is_empty())
  121. TRY(builder.try_append(' '));
  122. TRY(builder.try_append(segment));
  123. }
  124. return {};
  125. }));
  126. auto name = builder.to_deprecated_string();
  127. if (!any_of(name, is_ascii_lower_alpha))
  128. name = name.to_titlecase();
  129. emoji.name = emoji_data.unique_strings.ensure(move(name));
  130. TRY(emoji_data.emojis.try_append(move(emoji)));
  131. }
  132. return {};
  133. }
  134. static ErrorOr<void> validate_emoji(StringView emoji_resource_path, EmojiData& emoji_data)
  135. {
  136. Core::DirIterator iterator(emoji_resource_path, Core::DirIterator::SkipDots);
  137. while (iterator.has_next()) {
  138. auto filename = iterator.next_path();
  139. auto lexical_path = LexicalPath(filename);
  140. if (lexical_path.extension() != "png")
  141. continue;
  142. auto basename = lexical_path.basename();
  143. if (!basename.starts_with("U+"sv))
  144. continue;
  145. basename = basename.substring_view(0, basename.length() - lexical_path.extension().length() - 1);
  146. Vector<u32> code_points;
  147. TRY(basename.for_each_split_view('_', SplitBehavior::Nothing, [&](auto segment) -> ErrorOr<void> {
  148. auto code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(segment.substring_view(2));
  149. VERIFY(code_point.has_value());
  150. TRY(code_points.try_append(*code_point));
  151. return {};
  152. }));
  153. auto it = emoji_data.emojis.find_if([&](auto const& emoji) {
  154. return emoji.code_points == code_points;
  155. });
  156. if (it == emoji_data.emojis.end()) {
  157. warnln("\x1b[1;31mError!\x1b[0m Emoji data for \x1b[35m{}\x1b[0m not found. Please check emoji-test.txt and emoji-serenity.txt.", filename);
  158. return Error::from_errno(ENOENT);
  159. }
  160. }
  161. return {};
  162. }
  163. static ErrorOr<void> generate_emoji_data_header(Core::BufferedFile& file, EmojiData const&)
  164. {
  165. StringBuilder builder;
  166. SourceGenerator generator { builder };
  167. TRY(file.write_until_depleted(generator.as_string_view().bytes()));
  168. return {};
  169. }
  170. static ErrorOr<void> generate_emoji_data_implementation(Core::BufferedFile& file, EmojiData const& emoji_data)
  171. {
  172. StringBuilder builder;
  173. SourceGenerator generator { builder };
  174. generator.set("string_index_type"sv, emoji_data.unique_strings.type_that_fits());
  175. generator.set("emojis_size"sv, DeprecatedString::number(emoji_data.emojis.size()));
  176. generator.append(R"~~~(
  177. #include <AK/Array.h>
  178. #include <AK/BinarySearch.h>
  179. #include <AK/Span.h>
  180. #include <AK/StringView.h>
  181. #include <AK/Types.h>
  182. #include <LibUnicode/Emoji.h>
  183. #include <LibUnicode/EmojiData.h>
  184. namespace Unicode {
  185. )~~~");
  186. emoji_data.unique_strings.generate(generator);
  187. size_t total_code_point_count { 0 };
  188. for (auto const& emoji : emoji_data.emojis) {
  189. total_code_point_count += emoji.code_points.size();
  190. }
  191. generator.set("total_code_point_count", DeprecatedString::number(total_code_point_count));
  192. generator.append(R"~~~(
  193. static constexpr Array<u32, @total_code_point_count@> s_emoji_code_points { {)~~~");
  194. bool first = true;
  195. for (auto const& emoji : emoji_data.emojis) {
  196. for (auto code_point : emoji.code_points) {
  197. generator.append(first ? " "sv : ", "sv);
  198. generator.append(DeprecatedString::formatted("{:#x}", code_point));
  199. first = false;
  200. }
  201. }
  202. generator.append(" } };"sv);
  203. generator.append(R"~~~(
  204. struct EmojiData {
  205. Emoji to_unicode_emoji() const
  206. {
  207. Emoji emoji {};
  208. emoji.name = decode_string(name);
  209. if (image_path != 0)
  210. emoji.image_path = decode_string(image_path);
  211. emoji.group = static_cast<EmojiGroup>(group);
  212. emoji.display_order = display_order;
  213. emoji.code_points = code_points();
  214. return emoji;
  215. }
  216. constexpr ReadonlySpan<u32> code_points() const
  217. {
  218. return ReadonlySpan<u32>(s_emoji_code_points.data() + code_point_start, code_point_count);
  219. }
  220. @string_index_type@ name { 0 };
  221. @string_index_type@ image_path { 0 };
  222. u8 group { 0 };
  223. u32 display_order { 0 };
  224. size_t code_point_start { 0 };
  225. size_t code_point_count { 0 };
  226. };
  227. )~~~");
  228. generator.append(R"~~~(
  229. static constexpr Array<EmojiData, @emojis_size@> s_emojis { {)~~~");
  230. for (auto const& emoji : emoji_data.emojis) {
  231. generator.set("name"sv, DeprecatedString::number(emoji.name));
  232. generator.set("image_path"sv, DeprecatedString::number(emoji.image_path.value_or(0)));
  233. generator.set("group"sv, DeprecatedString::number(to_underlying(emoji.group)));
  234. generator.set("display_order"sv, DeprecatedString::number(emoji.display_order));
  235. generator.set("code_point_start"sv, DeprecatedString::number(emoji.code_point_array_index));
  236. generator.set("code_point_count"sv, DeprecatedString::number(emoji.code_points.size()));
  237. generator.append(R"~~~(
  238. { @name@, @image_path@, @group@, @display_order@, @code_point_start@, @code_point_count@ },)~~~");
  239. }
  240. generator.append(R"~~~(
  241. } };
  242. struct EmojiCodePointComparator {
  243. constexpr int operator()(ReadonlySpan<u32> code_points, EmojiData const& emoji)
  244. {
  245. auto emoji_code_points = emoji.code_points();
  246. if (code_points.size() != emoji_code_points.size())
  247. return static_cast<int>(code_points.size()) - static_cast<int>(emoji_code_points.size());
  248. for (size_t i = 0; i < code_points.size(); ++i) {
  249. if (code_points[i] != emoji_code_points[i])
  250. return static_cast<int>(code_points[i]) - static_cast<int>(emoji_code_points[i]);
  251. }
  252. return 0;
  253. }
  254. };
  255. Optional<Emoji> find_emoji_for_code_points(ReadonlySpan<u32> code_points)
  256. {
  257. if (auto const* emoji = binary_search(s_emojis, code_points, nullptr, EmojiCodePointComparator {}))
  258. return emoji->to_unicode_emoji();
  259. return {};
  260. }
  261. }
  262. )~~~");
  263. TRY(file.write_until_depleted(generator.as_string_view().bytes()));
  264. return {};
  265. }
  266. static ErrorOr<void> generate_emoji_installation(Core::BufferedFile& file, EmojiData const& emoji_data)
  267. {
  268. StringBuilder builder;
  269. SourceGenerator generator { builder };
  270. auto current_group = Unicode::EmojiGroup::Unknown;
  271. StringView current_subgroup;
  272. for (auto const& emoji : emoji_data.emojis) {
  273. if (!emoji.image_path.has_value())
  274. continue;
  275. if (emoji.group == Unicode::EmojiGroup::SerenityOS)
  276. continue; // SerenityOS emojis are in emoji-serenity.txt
  277. if (current_group != emoji.group) {
  278. if (!builder.is_empty())
  279. generator.append("\n"sv);
  280. generator.set("group"sv, Unicode::emoji_group_to_string(emoji.group));
  281. generator.append("# group: @group@\n");
  282. current_group = emoji.group;
  283. }
  284. if (current_subgroup != emoji.subgroup) {
  285. generator.set("subgroup"sv, emoji.subgroup);
  286. generator.append("\n# subgroup: @subgroup@\n");
  287. current_subgroup = emoji.subgroup;
  288. }
  289. generator.set("emoji"sv, emoji.encoded_code_points);
  290. generator.set("name"sv, emoji_data.unique_strings.get(emoji.name));
  291. generator.set("status"sv, emoji.status);
  292. generator.append("@emoji@"sv);
  293. generator.append(" - "sv);
  294. generator.append(DeprecatedString::join(" "sv, emoji.code_points, "U+{:X}"sv));
  295. generator.append(" @name@ (@status@)\n"sv);
  296. }
  297. TRY(file.write_until_depleted(generator.as_string_view().bytes()));
  298. return {};
  299. }
  300. ErrorOr<int> serenity_main(Main::Arguments arguments)
  301. {
  302. StringView generated_header_path;
  303. StringView generated_implementation_path;
  304. StringView generated_installation_path;
  305. StringView emoji_test_path;
  306. StringView emoji_serenity_path;
  307. StringView emoji_resource_path;
  308. Core::ArgsParser args_parser;
  309. args_parser.add_option(generated_header_path, "Path to the Unicode Data header file to generate", "generated-header-path", 'h', "generated-header-path");
  310. args_parser.add_option(generated_implementation_path, "Path to the Unicode Data implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
  311. args_parser.add_option(generated_installation_path, "Path to the emoji.txt file to generate", "generated-installation-path", 'i', "generated-installation-path");
  312. args_parser.add_option(emoji_test_path, "Path to emoji-test.txt file", "emoji-test-path", 'e', "emoji-test-path");
  313. args_parser.add_option(emoji_serenity_path, "Path to emoji-serenity.txt file", "emoji-serenity-path", 's', "emoji-serenity-path");
  314. args_parser.add_option(emoji_resource_path, "Path to the /res/emoji directory", "emoji-resource-path", 'r', "emoji-resource-path");
  315. args_parser.parse(arguments);
  316. VERIFY(!emoji_resource_path.is_empty() && Core::DeprecatedFile::exists(emoji_resource_path));
  317. auto emoji_test_file = TRY(open_file(emoji_test_path, Core::File::OpenMode::Read));
  318. EmojiData emoji_data {};
  319. TRY(parse_emoji_test_data(*emoji_test_file, emoji_data));
  320. if (!emoji_serenity_path.is_empty()) {
  321. auto emoji_serenity_file = TRY(open_file(emoji_serenity_path, Core::File::OpenMode::Read));
  322. TRY(parse_emoji_serenity_data(*emoji_serenity_file, emoji_data));
  323. TRY(validate_emoji(emoji_resource_path, emoji_data));
  324. }
  325. for (auto& emoji : emoji_data.emojis)
  326. set_image_path_for_emoji(emoji_resource_path, emoji_data, emoji);
  327. if (!generated_installation_path.is_empty()) {
  328. TRY(Core::Directory::create(LexicalPath { generated_installation_path }.parent(), Core::Directory::CreateDirectories::Yes));
  329. auto generated_installation_file = TRY(open_file(generated_installation_path, Core::File::OpenMode::Write));
  330. TRY(generate_emoji_installation(*generated_installation_file, emoji_data));
  331. }
  332. if (!generated_header_path.is_empty()) {
  333. auto generated_header_file = TRY(open_file(generated_header_path, Core::File::OpenMode::Write));
  334. TRY(generate_emoji_data_header(*generated_header_file, emoji_data));
  335. }
  336. if (!generated_implementation_path.is_empty()) {
  337. quick_sort(emoji_data.emojis, [](auto const& lhs, auto const& rhs) {
  338. if (lhs.code_points.size() != rhs.code_points.size())
  339. return lhs.code_points.size() < rhs.code_points.size();
  340. for (size_t i = 0; i < lhs.code_points.size(); ++i) {
  341. if (lhs.code_points[i] < rhs.code_points[i])
  342. return true;
  343. if (lhs.code_points[i] > rhs.code_points[i])
  344. return false;
  345. }
  346. return false;
  347. });
  348. size_t code_point_array_index { 0 };
  349. for (auto& emoji : emoji_data.emojis) {
  350. emoji.code_point_array_index = code_point_array_index;
  351. code_point_array_index += emoji.code_points.size();
  352. }
  353. auto generated_implementation_file = TRY(open_file(generated_implementation_path, Core::File::OpenMode::Write));
  354. TRY(generate_emoji_data_implementation(*generated_implementation_file, emoji_data));
  355. }
  356. return 0;
  357. }