GenerateUnicodeData.cpp 45 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150
  1. /*
  2. * Copyright (c) 2021, Tim Flynn <trflynn89@pm.me>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include "GeneratorUtil.h"
  7. #include <AK/AllOf.h>
  8. #include <AK/Array.h>
  9. #include <AK/CharacterTypes.h>
  10. #include <AK/Find.h>
  11. #include <AK/HashMap.h>
  12. #include <AK/Optional.h>
  13. #include <AK/QuickSort.h>
  14. #include <AK/SourceGenerator.h>
  15. #include <AK/String.h>
  16. #include <AK/StringUtils.h>
  17. #include <AK/Types.h>
  18. #include <AK/Vector.h>
  19. #include <LibCore/ArgsParser.h>
  20. #include <LibCore/File.h>
  21. // Some code points are excluded from UnicodeData.txt, and instead are part of a "range" of code
  22. // points, as indicated by the "name" field. For example:
  23. // 3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
  24. // 4DBF;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
  25. struct CodePointRange {
  26. u32 first;
  27. u32 last;
  28. };
  29. // SpecialCasing source: https://www.unicode.org/Public/13.0.0/ucd/SpecialCasing.txt
  30. // Field descriptions: https://www.unicode.org/reports/tr44/tr44-13.html#SpecialCasing.txt
  31. struct SpecialCasing {
  32. u32 index { 0 };
  33. u32 code_point { 0 };
  34. Vector<u32> lowercase_mapping;
  35. Vector<u32> uppercase_mapping;
  36. Vector<u32> titlecase_mapping;
  37. String locale;
  38. String condition;
  39. };
  40. // PropList source: https://www.unicode.org/Public/13.0.0/ucd/PropList.txt
  41. // Property descriptions: https://www.unicode.org/reports/tr44/tr44-13.html#PropList.txt
  42. using PropList = HashMap<String, Vector<CodePointRange>>;
  43. // Normalization source: https://www.unicode.org/Public/13.0.0/ucd/DerivedNormalizationProps.txt
  44. // Normalization descriptions: https://www.unicode.org/reports/tr44/#DerivedNormalizationProps.txt
  45. enum class QuickCheck {
  46. Yes,
  47. No,
  48. Maybe,
  49. };
  50. struct Normalization {
  51. CodePointRange code_point_range;
  52. Vector<u32> value;
  53. QuickCheck quick_check { QuickCheck::Yes };
  54. };
  55. using NormalizationProps = HashMap<String, Vector<Normalization>>;
  56. struct CodePointName {
  57. CodePointRange code_point_range;
  58. StringView name;
  59. };
  60. // UnicodeData source: https://www.unicode.org/Public/13.0.0/ucd/UnicodeData.txt
  61. // Field descriptions: https://www.unicode.org/reports/tr44/tr44-13.html#UnicodeData.txt
  62. // https://www.unicode.org/reports/tr44/#General_Category_Values
  63. struct CodePointData {
  64. u32 code_point { 0 };
  65. String name;
  66. u8 canonical_combining_class { 0 };
  67. String bidi_class;
  68. String decomposition_type;
  69. Optional<i8> numeric_value_decimal;
  70. Optional<i8> numeric_value_digit;
  71. Optional<i8> numeric_value_numeric;
  72. bool bidi_mirrored { false };
  73. String unicode_1_name;
  74. String iso_comment;
  75. Optional<u32> simple_uppercase_mapping;
  76. Optional<u32> simple_lowercase_mapping;
  77. Optional<u32> simple_titlecase_mapping;
  78. Vector<u32> special_casing_indices;
  79. };
  80. struct UnicodeData {
  81. u32 code_points_with_non_zero_combining_class { 0 };
  82. u32 simple_uppercase_mapping_size { 0 };
  83. u32 simple_lowercase_mapping_size { 0 };
  84. Vector<SpecialCasing> special_casing;
  85. u32 code_points_with_special_casing { 0 };
  86. u32 largest_casing_transform_size { 0 };
  87. u32 largest_special_casing_size { 0 };
  88. Vector<String> conditions;
  89. Vector<CodePointData> code_point_data;
  90. HashMap<u32, String> code_point_display_name_aliases;
  91. Vector<CodePointName> code_point_display_names;
  92. PropList general_categories;
  93. Vector<Alias> general_category_aliases;
  94. // The Unicode standard defines additional properties (Any, Assigned, ASCII) which are not in
  95. // any UCD file. Assigned code point ranges are derived as this generator is executed.
  96. // https://unicode.org/reports/tr18/#General_Category_Property
  97. PropList prop_list {
  98. { "Any"sv, { { 0, 0x10ffff } } },
  99. { "Assigned"sv, {} },
  100. { "ASCII"sv, { { 0, 0x7f } } },
  101. };
  102. Vector<Alias> prop_aliases;
  103. PropList script_list {
  104. { "Unknown"sv, {} },
  105. };
  106. Vector<Alias> script_aliases;
  107. PropList script_extensions;
  108. // FIXME: We are not yet doing anything with this data. It will be needed for String.prototype.normalize.
  109. NormalizationProps normalization_props;
  110. };
  111. static Vector<u32> parse_code_point_list(StringView list)
  112. {
  113. Vector<u32> code_points;
  114. auto segments = list.split_view(' ');
  115. for (auto const& code_point : segments)
  116. code_points.append(AK::StringUtils::convert_to_uint_from_hex<u32>(code_point).value());
  117. return code_points;
  118. }
  119. static CodePointRange parse_code_point_range(StringView list)
  120. {
  121. CodePointRange code_point_range {};
  122. if (list.contains(".."sv)) {
  123. auto segments = list.split_view(".."sv);
  124. VERIFY(segments.size() == 2);
  125. auto begin = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[0]).value();
  126. auto end = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[1]).value();
  127. code_point_range = { begin, end };
  128. } else {
  129. auto code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(list).value();
  130. code_point_range = { code_point, code_point };
  131. }
  132. return code_point_range;
  133. }
  134. static void parse_special_casing(Core::File& file, UnicodeData& unicode_data)
  135. {
  136. while (file.can_read_line()) {
  137. auto line = file.read_line();
  138. if (line.is_empty() || line.starts_with('#'))
  139. continue;
  140. if (auto index = line.find('#'); index.has_value())
  141. line = line.substring(0, *index);
  142. auto segments = line.split(';', true);
  143. VERIFY(segments.size() == 5 || segments.size() == 6);
  144. SpecialCasing casing {};
  145. casing.code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[0]).value();
  146. casing.lowercase_mapping = parse_code_point_list(segments[1]);
  147. casing.titlecase_mapping = parse_code_point_list(segments[2]);
  148. casing.uppercase_mapping = parse_code_point_list(segments[3]);
  149. if (auto condition = segments[4].trim_whitespace(); !condition.is_empty()) {
  150. auto conditions = condition.split(' ', true);
  151. VERIFY(conditions.size() == 1 || conditions.size() == 2);
  152. if (conditions.size() == 2) {
  153. casing.locale = move(conditions[0]);
  154. casing.condition = move(conditions[1]);
  155. } else if (all_of(conditions[0], is_ascii_lower_alpha)) {
  156. casing.locale = move(conditions[0]);
  157. } else {
  158. casing.condition = move(conditions[0]);
  159. }
  160. if (!casing.locale.is_empty())
  161. casing.locale = String::formatted("{:c}{}", to_ascii_uppercase(casing.locale[0]), casing.locale.substring_view(1));
  162. casing.condition = casing.condition.replace("_", "", true);
  163. if (!casing.condition.is_empty() && !unicode_data.conditions.contains_slow(casing.condition))
  164. unicode_data.conditions.append(casing.condition);
  165. }
  166. unicode_data.largest_casing_transform_size = max(unicode_data.largest_casing_transform_size, casing.lowercase_mapping.size());
  167. unicode_data.largest_casing_transform_size = max(unicode_data.largest_casing_transform_size, casing.titlecase_mapping.size());
  168. unicode_data.largest_casing_transform_size = max(unicode_data.largest_casing_transform_size, casing.uppercase_mapping.size());
  169. unicode_data.special_casing.append(move(casing));
  170. }
  171. quick_sort(unicode_data.special_casing, [](auto const& lhs, auto const& rhs) {
  172. if (lhs.code_point != rhs.code_point)
  173. return lhs.code_point < rhs.code_point;
  174. if (lhs.locale.is_empty() && !rhs.locale.is_empty())
  175. return false;
  176. if (!lhs.locale.is_empty() && rhs.locale.is_empty())
  177. return true;
  178. return lhs.locale < rhs.locale;
  179. });
  180. for (u32 i = 0; i < unicode_data.special_casing.size(); ++i)
  181. unicode_data.special_casing[i].index = i;
  182. }
  183. static void parse_prop_list(Core::File& file, PropList& prop_list, bool multi_value_property = false)
  184. {
  185. while (file.can_read_line()) {
  186. auto line = file.read_line();
  187. if (line.is_empty() || line.starts_with('#'))
  188. continue;
  189. if (auto index = line.find('#'); index.has_value())
  190. line = line.substring(0, *index);
  191. auto segments = line.split_view(';', true);
  192. VERIFY(segments.size() == 2);
  193. auto code_point_range = parse_code_point_range(segments[0].trim_whitespace());
  194. Vector<StringView> properties;
  195. if (multi_value_property)
  196. properties = segments[1].trim_whitespace().split_view(' ');
  197. else
  198. properties = { segments[1].trim_whitespace() };
  199. for (auto const& property : properties) {
  200. auto& code_points = prop_list.ensure(property.trim_whitespace());
  201. code_points.append(code_point_range);
  202. }
  203. }
  204. }
  205. static void parse_alias_list(Core::File& file, PropList const& prop_list, Vector<Alias>& prop_aliases)
  206. {
  207. String current_property;
  208. auto append_alias = [&](auto alias, auto property) {
  209. // Note: The alias files contain lines such as "Hyphen = Hyphen", which we should just skip.
  210. if (alias == property)
  211. return;
  212. // FIXME: We will, eventually, need to find where missing properties are located and parse them.
  213. if (!prop_list.contains(property))
  214. return;
  215. prop_aliases.append({ property, alias });
  216. };
  217. while (file.can_read_line()) {
  218. auto line = file.read_line();
  219. if (line.is_empty() || line.starts_with('#')) {
  220. if (line.ends_with("Properties"sv))
  221. current_property = line.substring(2);
  222. continue;
  223. }
  224. // Note: For now, we only care about Binary Property aliases for Unicode property escapes.
  225. if (current_property != "Binary Properties"sv)
  226. continue;
  227. auto segments = line.split_view(';', true);
  228. VERIFY((segments.size() == 2) || (segments.size() == 3));
  229. auto alias = segments[0].trim_whitespace();
  230. auto property = segments[1].trim_whitespace();
  231. append_alias(alias, property);
  232. if (segments.size() == 3) {
  233. alias = segments[2].trim_whitespace();
  234. append_alias(alias, property);
  235. }
  236. }
  237. }
  238. static void parse_name_aliases(Core::File& file, UnicodeData& unicode_data)
  239. {
  240. while (file.can_read_line()) {
  241. auto line = file.read_line();
  242. if (line.is_empty() || line.starts_with('#'))
  243. continue;
  244. auto segments = line.split_view(';', true);
  245. VERIFY(segments.size() == 3);
  246. auto code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[0].trim_whitespace());
  247. auto alias = segments[1].trim_whitespace();
  248. auto reason = segments[2].trim_whitespace();
  249. if (!reason.is_one_of("correction"sv, "control"sv))
  250. continue;
  251. if (!unicode_data.code_point_display_name_aliases.contains(*code_point))
  252. unicode_data.code_point_display_name_aliases.set(*code_point, alias);
  253. }
  254. }
  255. static void parse_value_alias_list(Core::File& file, StringView desired_category, Vector<String> const& value_list, Vector<Alias>& prop_aliases, bool primary_value_is_first = true)
  256. {
  257. VERIFY(file.seek(0));
  258. auto append_alias = [&](auto alias, auto value) {
  259. // Note: The value alias file contains lines such as "Ahom = Ahom", which we should just skip.
  260. if (alias == value)
  261. return;
  262. // FIXME: We will, eventually, need to find where missing properties are located and parse them.
  263. if (!value_list.contains_slow(value))
  264. return;
  265. prop_aliases.append({ value, alias });
  266. };
  267. while (file.can_read_line()) {
  268. auto line = file.read_line();
  269. if (line.is_empty() || line.starts_with('#'))
  270. continue;
  271. if (auto index = line.find('#'); index.has_value())
  272. line = line.substring(0, *index);
  273. auto segments = line.split_view(';', true);
  274. auto category = segments[0].trim_whitespace();
  275. if (category != desired_category)
  276. continue;
  277. VERIFY((segments.size() == 3) || (segments.size() == 4));
  278. auto value = primary_value_is_first ? segments[1].trim_whitespace() : segments[2].trim_whitespace();
  279. auto alias = primary_value_is_first ? segments[2].trim_whitespace() : segments[1].trim_whitespace();
  280. append_alias(alias, value);
  281. if (segments.size() == 4) {
  282. alias = segments[3].trim_whitespace();
  283. append_alias(alias, value);
  284. }
  285. }
  286. }
  287. static void parse_normalization_props(Core::File& file, UnicodeData& unicode_data)
  288. {
  289. while (file.can_read_line()) {
  290. auto line = file.read_line();
  291. if (line.is_empty() || line.starts_with('#'))
  292. continue;
  293. if (auto index = line.find('#'); index.has_value())
  294. line = line.substring(0, *index);
  295. auto segments = line.split_view(';', true);
  296. VERIFY((segments.size() == 2) || (segments.size() == 3));
  297. auto code_point_range = parse_code_point_range(segments[0].trim_whitespace());
  298. auto property = segments[1].trim_whitespace().to_string();
  299. Vector<u32> value;
  300. QuickCheck quick_check = QuickCheck::Yes;
  301. if (segments.size() == 3) {
  302. auto value_or_quick_check = segments[2].trim_whitespace();
  303. if ((value_or_quick_check == "N"sv))
  304. quick_check = QuickCheck::No;
  305. else if ((value_or_quick_check == "M"sv))
  306. quick_check = QuickCheck::Maybe;
  307. else
  308. value = parse_code_point_list(value_or_quick_check);
  309. }
  310. auto& normalizations = unicode_data.normalization_props.ensure(property);
  311. normalizations.append({ code_point_range, move(value), quick_check });
  312. auto& prop_list = unicode_data.prop_list.ensure(property);
  313. prop_list.append(move(code_point_range));
  314. }
  315. }
  316. static void add_canonical_code_point_name(CodePointRange range, StringView name, UnicodeData& unicode_data)
  317. {
  318. // https://www.unicode.org/versions/Unicode14.0.0/ch04.pdf#G142981
  319. // FIXME: Implement the NR1 rules for Hangul syllables.
  320. // These code point ranges are the NR2 set of name replacements defined by Table 4-8.
  321. constexpr Array<CodePointName, 15> s_ideographic_replacements { {
  322. { { 0x3400, 0x4DBF }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  323. { { 0x4E00, 0x9FFC }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  324. { { 0xF900, 0xFA6D }, "CJK COMPATIBILITY IDEOGRAPH-{:X}"sv },
  325. { { 0xFA70, 0xFAD9 }, "CJK COMPATIBILITY IDEOGRAPH-{:X}"sv },
  326. { { 0x17000, 0x187F7 }, "TANGUT IDEOGRAPH-{:X}"sv },
  327. { { 0x18B00, 0x18CD5 }, "KHITAN SMALL SCRIPT CHARACTER-{:X}"sv },
  328. { { 0x18D00, 0x18D08 }, "TANGUT IDEOGRAPH-{:X}"sv },
  329. { { 0x1B170, 0x1B2FB }, "NUSHU CHARACTER-{:X}"sv },
  330. { { 0x20000, 0x2A6DD }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  331. { { 0x2A700, 0x2B734 }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  332. { { 0x2B740, 0x2B81D }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  333. { { 0x2B820, 0x2CEA1 }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  334. { { 0x2CEB0, 0x2EBE0 }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  335. { { 0x2F800, 0x2FA1D }, "CJK COMPATIBILITY IDEOGRAPH-{:X}"sv },
  336. { { 0x30000, 0x3134A }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  337. } };
  338. auto it = find_if(s_ideographic_replacements.begin(), s_ideographic_replacements.end(),
  339. [&](auto const& replacement) {
  340. return replacement.code_point_range.first == range.first;
  341. });
  342. if (it != s_ideographic_replacements.end()) {
  343. unicode_data.code_point_display_names.append(*it);
  344. return;
  345. }
  346. it = find_if(s_ideographic_replacements.begin(), s_ideographic_replacements.end(),
  347. [&](auto const& replacement) {
  348. return (replacement.code_point_range.first <= range.first) && (range.first <= replacement.code_point_range.last);
  349. });
  350. if (it != s_ideographic_replacements.end()) {
  351. // Drop code points that will have been captured by a range defined by the ideographic replacements.
  352. return;
  353. }
  354. if (auto alias = unicode_data.code_point_display_name_aliases.get(range.first); alias.has_value()) {
  355. // NR4 states that control code points have a null string as their name. Our implementation
  356. // uses the control code's alias as its display name.
  357. unicode_data.code_point_display_names.append({ range, *alias });
  358. return;
  359. }
  360. unicode_data.code_point_display_names.append({ range, name });
  361. }
  362. static void parse_unicode_data(Core::File& file, UnicodeData& unicode_data)
  363. {
  364. Optional<u32> code_point_range_start;
  365. auto& assigned_code_points = unicode_data.prop_list.find("Assigned"sv)->value;
  366. Optional<u32> assigned_code_point_range_start = 0;
  367. u32 previous_code_point = 0;
  368. while (file.can_read_line()) {
  369. auto line = file.read_line();
  370. if (line.is_empty())
  371. continue;
  372. auto segments = line.split(';', true);
  373. VERIFY(segments.size() == 15);
  374. CodePointData data {};
  375. data.code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[0]).value();
  376. data.name = move(segments[1]);
  377. data.canonical_combining_class = AK::StringUtils::convert_to_uint<u8>(segments[3]).value();
  378. data.bidi_class = move(segments[4]);
  379. data.decomposition_type = move(segments[5]);
  380. data.numeric_value_decimal = AK::StringUtils::convert_to_int<i8>(segments[6]);
  381. data.numeric_value_digit = AK::StringUtils::convert_to_int<i8>(segments[7]);
  382. data.numeric_value_numeric = AK::StringUtils::convert_to_int<i8>(segments[8]);
  383. data.bidi_mirrored = segments[9] == "Y"sv;
  384. data.unicode_1_name = move(segments[10]);
  385. data.iso_comment = move(segments[11]);
  386. data.simple_uppercase_mapping = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[12]);
  387. data.simple_lowercase_mapping = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[13]);
  388. data.simple_titlecase_mapping = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[14]);
  389. if (!assigned_code_point_range_start.has_value())
  390. assigned_code_point_range_start = data.code_point;
  391. if (data.name.starts_with("<"sv) && data.name.ends_with(", First>")) {
  392. VERIFY(!code_point_range_start.has_value() && assigned_code_point_range_start.has_value());
  393. code_point_range_start = data.code_point;
  394. data.name = data.name.substring(1, data.name.length() - 9);
  395. assigned_code_points.append({ *assigned_code_point_range_start, previous_code_point });
  396. assigned_code_point_range_start.clear();
  397. } else if (data.name.starts_with("<"sv) && data.name.ends_with(", Last>")) {
  398. VERIFY(code_point_range_start.has_value());
  399. CodePointRange code_point_range { *code_point_range_start, data.code_point };
  400. assigned_code_points.append(code_point_range);
  401. data.name = data.name.substring(1, data.name.length() - 8);
  402. code_point_range_start.clear();
  403. add_canonical_code_point_name(code_point_range, data.name, unicode_data);
  404. } else {
  405. add_canonical_code_point_name({ data.code_point, data.code_point }, data.name, unicode_data);
  406. if ((data.code_point > 0) && (data.code_point - previous_code_point) != 1) {
  407. VERIFY(assigned_code_point_range_start.has_value());
  408. assigned_code_points.append({ *assigned_code_point_range_start, previous_code_point });
  409. assigned_code_point_range_start = data.code_point;
  410. }
  411. }
  412. bool has_special_casing { false };
  413. for (auto const& casing : unicode_data.special_casing) {
  414. if (casing.code_point == data.code_point) {
  415. data.special_casing_indices.append(casing.index);
  416. has_special_casing = true;
  417. }
  418. }
  419. unicode_data.code_points_with_non_zero_combining_class += data.canonical_combining_class != 0;
  420. unicode_data.simple_uppercase_mapping_size += data.simple_uppercase_mapping.has_value();
  421. unicode_data.simple_lowercase_mapping_size += data.simple_lowercase_mapping.has_value();
  422. unicode_data.code_points_with_special_casing += has_special_casing;
  423. unicode_data.largest_special_casing_size = max(unicode_data.largest_special_casing_size, data.special_casing_indices.size());
  424. previous_code_point = data.code_point;
  425. unicode_data.code_point_data.append(move(data));
  426. }
  427. }
  428. static void generate_unicode_data_header(Core::File& file, UnicodeData& unicode_data)
  429. {
  430. StringBuilder builder;
  431. SourceGenerator generator { builder };
  432. generator.set("casing_transform_size", String::number(unicode_data.largest_casing_transform_size));
  433. auto generate_enum = [&](StringView name, StringView default_, Vector<String> values, Vector<Alias> aliases = {}) {
  434. quick_sort(values);
  435. quick_sort(aliases, [](auto& alias1, auto& alias2) { return alias1.alias < alias2.alias; });
  436. generator.set("name", name);
  437. generator.set("underlying", String::formatted("{}UnderlyingType", name));
  438. generator.append(R"~~~(
  439. using @underlying@ = u8;
  440. enum class @name@ : @underlying@ {)~~~");
  441. if (!default_.is_empty()) {
  442. generator.set("default", default_);
  443. generator.append(R"~~~(
  444. @default@,)~~~");
  445. }
  446. for (auto const& value : values) {
  447. generator.set("value", value);
  448. generator.append(R"~~~(
  449. @value@,)~~~");
  450. }
  451. for (auto const& alias : aliases) {
  452. generator.set("alias", alias.alias);
  453. generator.set("value", alias.name);
  454. generator.append(R"~~~(
  455. @alias@ = @value@,)~~~");
  456. }
  457. generator.append(R"~~~(
  458. };
  459. )~~~");
  460. };
  461. generator.append(R"~~~(
  462. #pragma once
  463. #include <AK/Types.h>
  464. #include <LibUnicode/Forward.h>
  465. #include <LibUnicode/UnicodeLocale.h>
  466. namespace Unicode {
  467. )~~~");
  468. generate_enum("Condition"sv, "None"sv, move(unicode_data.conditions));
  469. generate_enum("GeneralCategory"sv, {}, unicode_data.general_categories.keys(), unicode_data.general_category_aliases);
  470. generate_enum("Property"sv, {}, unicode_data.prop_list.keys(), unicode_data.prop_aliases);
  471. generate_enum("Script"sv, {}, unicode_data.script_list.keys(), unicode_data.script_aliases);
  472. generator.append(R"~~~(
  473. struct SpecialCasing {
  474. u32 code_point { 0 };
  475. u32 lowercase_mapping[@casing_transform_size@];
  476. u32 lowercase_mapping_size { 0 };
  477. u32 uppercase_mapping[@casing_transform_size@];
  478. u32 uppercase_mapping_size { 0 };
  479. u32 titlecase_mapping[@casing_transform_size@];
  480. u32 titlecase_mapping_size { 0 };
  481. Locale locale { Locale::None };
  482. Condition condition { Condition::None };
  483. };
  484. }
  485. )~~~");
  486. VERIFY(file.write(generator.as_string_view()));
  487. }
  488. static void generate_unicode_data_implementation(Core::File& file, UnicodeData const& unicode_data)
  489. {
  490. StringBuilder builder;
  491. SourceGenerator generator { builder };
  492. generator.set("largest_special_casing_size", String::number(unicode_data.largest_special_casing_size));
  493. generator.set("special_casing_size", String::number(unicode_data.special_casing.size()));
  494. generator.append(R"~~~(
  495. #include <AK/Array.h>
  496. #include <AK/BinarySearch.h>
  497. #include <AK/CharacterTypes.h>
  498. #include <AK/Optional.h>
  499. #include <AK/Span.h>
  500. #include <AK/String.h>
  501. #include <AK/StringView.h>
  502. #include <LibUnicode/CharacterTypes.h>
  503. #include <LibUnicode/UnicodeData.h>
  504. namespace Unicode {
  505. )~~~");
  506. auto append_list_and_size = [&](auto const& list, StringView format) {
  507. if (list.is_empty()) {
  508. generator.append(", {}, 0");
  509. return;
  510. }
  511. bool first = true;
  512. generator.append(", {");
  513. for (auto const& item : list) {
  514. generator.append(first ? " " : ", ");
  515. generator.append(String::formatted(format, item));
  516. first = false;
  517. }
  518. generator.append(String::formatted(" }}, {}", list.size()));
  519. };
  520. generator.append(R"~~~(
  521. static constexpr Array<SpecialCasing, @special_casing_size@> s_special_casing { {)~~~");
  522. for (auto const& casing : unicode_data.special_casing) {
  523. generator.set("code_point", String::formatted("{:#x}", casing.code_point));
  524. generator.append(R"~~~(
  525. { @code_point@)~~~");
  526. constexpr auto format = "0x{:x}"sv;
  527. append_list_and_size(casing.lowercase_mapping, format);
  528. append_list_and_size(casing.uppercase_mapping, format);
  529. append_list_and_size(casing.titlecase_mapping, format);
  530. generator.set("locale", casing.locale.is_empty() ? "None" : casing.locale);
  531. generator.append(", Locale::@locale@");
  532. generator.set("condition", casing.condition.is_empty() ? "None" : casing.condition);
  533. generator.append(", Condition::@condition@");
  534. generator.append(" },");
  535. }
  536. generator.append(R"~~~(
  537. } };
  538. struct CodePointMapping {
  539. u32 code_point { 0 };
  540. u32 mapping { 0 };
  541. };
  542. struct SpecialCaseMapping {
  543. u32 code_point { 0 };
  544. Array<SpecialCasing const*, @largest_special_casing_size@> special_casing {};
  545. u32 special_casing_size { 0 };
  546. };
  547. template<typename MappingType>
  548. struct CodePointComparator {
  549. constexpr int operator()(u32 code_point, MappingType const& mapping)
  550. {
  551. return code_point - mapping.code_point;
  552. }
  553. };
  554. )~~~");
  555. auto append_code_point_mappings = [&](StringView name, StringView mapping_type, u32 size, auto mapping_getter) {
  556. generator.set("name", name);
  557. generator.set("mapping_type", mapping_type);
  558. generator.set("size", String::number(size));
  559. generator.append(R"~~~(
  560. static constexpr Array<@mapping_type@, @size@> s_@name@_mappings { {
  561. )~~~");
  562. constexpr size_t max_mappings_per_row = 20;
  563. size_t mappings_in_current_row = 0;
  564. for (auto const& data : unicode_data.code_point_data) {
  565. auto mapping = mapping_getter(data);
  566. if constexpr (IsSame<decltype(mapping), Optional<u32>>) {
  567. if (!mapping.has_value())
  568. continue;
  569. } else {
  570. if (mapping.is_empty())
  571. continue;
  572. }
  573. if (mappings_in_current_row++ > 0)
  574. generator.append(" ");
  575. generator.set("code_point", String::formatted("{:#x}", data.code_point));
  576. generator.append("{ @code_point@");
  577. if constexpr (IsSame<decltype(mapping), Optional<u32>>) {
  578. generator.set("mapping", String::formatted("{:#x}", *mapping));
  579. generator.append(", @mapping@ },");
  580. } else {
  581. append_list_and_size(data.special_casing_indices, "&s_special_casing[{}]"sv);
  582. generator.append(" },");
  583. }
  584. if (mappings_in_current_row == max_mappings_per_row) {
  585. mappings_in_current_row = 0;
  586. generator.append("\n ");
  587. }
  588. }
  589. generator.append(R"~~~(
  590. } };
  591. )~~~");
  592. };
  593. append_code_point_mappings("combining_class"sv, "CodePointMapping"sv, unicode_data.code_points_with_non_zero_combining_class,
  594. [](auto const& data) -> Optional<u32> {
  595. if (data.canonical_combining_class == 0)
  596. return {};
  597. return data.canonical_combining_class;
  598. });
  599. append_code_point_mappings("uppercase"sv, "CodePointMapping"sv, unicode_data.simple_uppercase_mapping_size, [](auto const& data) { return data.simple_uppercase_mapping; });
  600. append_code_point_mappings("lowercase"sv, "CodePointMapping"sv, unicode_data.simple_lowercase_mapping_size, [](auto const& data) { return data.simple_lowercase_mapping; });
  601. append_code_point_mappings("special_case"sv, "SpecialCaseMapping"sv, unicode_data.code_points_with_special_casing, [](auto const& data) { return data.special_casing_indices; });
  602. generator.append(R"~~~(
  603. struct CodePointRange {
  604. u32 first { 0 };
  605. u32 last { 0 };
  606. };
  607. struct CodePointRangeComparator {
  608. constexpr int operator()(u32 code_point, CodePointRange const& range)
  609. {
  610. return (code_point > range.last) - (code_point < range.first);
  611. }
  612. };
  613. )~~~");
  614. auto append_code_point_range_list = [&](String name, Vector<CodePointRange> const& ranges) {
  615. generator.set("name", name);
  616. generator.set("size", String::number(ranges.size()));
  617. generator.append(R"~~~(
  618. static constexpr Array<CodePointRange, @size@> @name@ { {
  619. )~~~");
  620. constexpr size_t max_ranges_per_row = 20;
  621. size_t ranges_in_current_row = 0;
  622. for (auto const& range : ranges) {
  623. if (ranges_in_current_row++ > 0)
  624. generator.append(" ");
  625. generator.set("first", String::formatted("{:#x}", range.first));
  626. generator.set("last", String::formatted("{:#x}", range.last));
  627. generator.append("{ @first@, @last@ },");
  628. if (ranges_in_current_row == max_ranges_per_row) {
  629. ranges_in_current_row = 0;
  630. generator.append("\n ");
  631. }
  632. }
  633. generator.append(R"~~~(
  634. } };
  635. )~~~");
  636. };
  637. auto append_prop_list = [&](StringView collection_name, StringView property_format, PropList const& property_list) {
  638. for (auto const& property : property_list) {
  639. auto name = String::formatted(property_format, property.key);
  640. append_code_point_range_list(move(name), property.value);
  641. }
  642. auto property_names = property_list.keys();
  643. quick_sort(property_names);
  644. generator.set("name", collection_name);
  645. generator.set("size", String::number(property_names.size()));
  646. generator.append(R"~~~(
  647. static constexpr Array<Span<CodePointRange const>, @size@> @name@ { {)~~~");
  648. for (auto const& property_name : property_names) {
  649. generator.set("name", String::formatted(property_format, property_name));
  650. generator.append(R"~~~(
  651. @name@.span(),)~~~");
  652. }
  653. generator.append(R"~~~(
  654. } };
  655. )~~~");
  656. };
  657. append_prop_list("s_general_categories"sv, "s_general_category_{}"sv, unicode_data.general_categories);
  658. append_prop_list("s_properties"sv, "s_property_{}"sv, unicode_data.prop_list);
  659. append_prop_list("s_scripts"sv, "s_script_{}"sv, unicode_data.script_list);
  660. append_prop_list("s_script_extensions"sv, "s_script_extension_{}"sv, unicode_data.script_extensions);
  661. generator.append(R"~~~(
  662. struct CodePointName {
  663. CodePointRange code_point_range {};
  664. StringView display_name;
  665. };
  666. struct CodePointNameComparator : public CodePointRangeComparator {
  667. constexpr int operator()(u32 code_point, CodePointName const& name)
  668. {
  669. return CodePointRangeComparator::operator()(code_point, name.code_point_range);
  670. }
  671. };
  672. )~~~");
  673. generator.set("code_point_display_names_size", String::number(unicode_data.code_point_display_names.size()));
  674. generator.append(R"~~~(
  675. static constexpr Array<CodePointName, @code_point_display_names_size@> s_code_point_display_names { {
  676. )~~~");
  677. for (auto const& code_point_name : unicode_data.code_point_display_names) {
  678. generator.set("first", String::formatted("{:#x}", code_point_name.code_point_range.first));
  679. generator.set("last", String::formatted("{:#x}", code_point_name.code_point_range.last));
  680. generator.set("name", code_point_name.name);
  681. generator.append(R"~~~( { { @first@, @last@ }, "@name@"sv },
  682. )~~~");
  683. }
  684. generator.append(R"~~~(} };
  685. )~~~");
  686. generator.append(R"~~~(
  687. Optional<String> code_point_display_name(u32 code_point)
  688. {
  689. if (auto const* entry = binary_search(s_code_point_display_names, code_point, nullptr, CodePointNameComparator {})) {
  690. if (entry->display_name.ends_with("{:X}"sv))
  691. return String::formatted(entry->display_name, code_point);
  692. return entry->display_name;
  693. }
  694. return {};
  695. }
  696. )~~~");
  697. auto append_code_point_mapping_search = [&](StringView method, StringView mappings, StringView fallback) {
  698. generator.set("method", method);
  699. generator.set("mappings", mappings);
  700. generator.set("fallback", fallback);
  701. generator.append(R"~~~(
  702. u32 @method@(u32 code_point)
  703. {
  704. auto const* mapping = binary_search(@mappings@, code_point, nullptr, CodePointComparator<CodePointMapping> {});
  705. return mapping ? mapping->mapping : @fallback@;
  706. }
  707. )~~~");
  708. };
  709. append_code_point_mapping_search("canonical_combining_class"sv, "s_combining_class_mappings"sv, "0"sv);
  710. append_code_point_mapping_search("to_unicode_uppercase"sv, "s_uppercase_mappings"sv, "code_point"sv);
  711. append_code_point_mapping_search("to_unicode_lowercase"sv, "s_lowercase_mappings"sv, "code_point"sv);
  712. generator.append(R"~~~(
  713. Span<SpecialCasing const* const> special_case_mapping(u32 code_point)
  714. {
  715. auto const* mapping = binary_search(s_special_case_mappings, code_point, nullptr, CodePointComparator<SpecialCaseMapping> {});
  716. if (mapping == nullptr)
  717. return {};
  718. return mapping->special_casing.span().slice(0, mapping->special_casing_size);
  719. }
  720. )~~~");
  721. auto append_prop_search = [&](StringView enum_title, StringView enum_snake, StringView collection_name) {
  722. generator.set("enum_title", enum_title);
  723. generator.set("enum_snake", enum_snake);
  724. generator.set("collection_name", collection_name);
  725. generator.append(R"~~~(
  726. bool code_point_has_@enum_snake@(u32 code_point, @enum_title@ @enum_snake@)
  727. {
  728. auto index = static_cast<@enum_title@UnderlyingType>(@enum_snake@);
  729. auto const& ranges = @collection_name@.at(index);
  730. auto const* range = binary_search(ranges, code_point, nullptr, CodePointRangeComparator {});
  731. return range != nullptr;
  732. }
  733. )~~~");
  734. };
  735. auto append_from_string = [&](StringView enum_title, StringView enum_snake, PropList const& prop_list, Vector<Alias> const& aliases) {
  736. HashValueMap<StringView> hashes;
  737. hashes.ensure_capacity(prop_list.size() + aliases.size());
  738. for (auto const& prop : prop_list)
  739. hashes.set(prop.key.hash(), prop.key);
  740. for (auto const& alias : aliases)
  741. hashes.set(alias.alias.hash(), alias.alias);
  742. generate_value_from_string(generator, "{}_from_string"sv, enum_title, enum_snake, move(hashes));
  743. };
  744. append_prop_search("GeneralCategory"sv, "general_category"sv, "s_general_categories"sv);
  745. append_from_string("GeneralCategory"sv, "general_category"sv, unicode_data.general_categories, unicode_data.general_category_aliases);
  746. append_prop_search("Property"sv, "property"sv, "s_properties"sv);
  747. append_from_string("Property"sv, "property"sv, unicode_data.prop_list, unicode_data.prop_aliases);
  748. append_prop_search("Script"sv, "script"sv, "s_scripts"sv);
  749. append_prop_search("Script"sv, "script_extension"sv, "s_script_extensions"sv);
  750. append_from_string("Script"sv, "script"sv, unicode_data.script_list, unicode_data.script_aliases);
  751. generator.append(R"~~~(
  752. }
  753. )~~~");
  754. VERIFY(file.write(generator.as_string_view()));
  755. }
  756. static Vector<u32> flatten_code_point_ranges(Vector<CodePointRange> const& code_points)
  757. {
  758. Vector<u32> flattened;
  759. for (auto const& range : code_points) {
  760. flattened.grow_capacity(range.last - range.first);
  761. for (u32 code_point = range.first; code_point <= range.last; ++code_point)
  762. flattened.append(code_point);
  763. }
  764. return flattened;
  765. }
  766. static Vector<CodePointRange> form_code_point_ranges(Vector<u32> code_points)
  767. {
  768. Vector<CodePointRange> ranges;
  769. u32 range_start = code_points[0];
  770. u32 range_end = range_start;
  771. for (size_t i = 1; i < code_points.size(); ++i) {
  772. u32 code_point = code_points[i];
  773. if ((code_point - range_end) == 1) {
  774. range_end = code_point;
  775. } else {
  776. ranges.append({ range_start, range_end });
  777. range_start = code_point;
  778. range_end = code_point;
  779. }
  780. }
  781. ranges.append({ range_start, range_end });
  782. return ranges;
  783. }
  784. static void sort_and_merge_code_point_ranges(Vector<CodePointRange>& code_points)
  785. {
  786. quick_sort(code_points, [](auto const& range1, auto const& range2) {
  787. return range1.first < range2.first;
  788. });
  789. for (size_t i = 0; i < code_points.size() - 1;) {
  790. if (code_points[i].last >= code_points[i + 1].first) {
  791. code_points[i].last = max(code_points[i].last, code_points[i + 1].last);
  792. code_points.remove(i + 1);
  793. } else {
  794. ++i;
  795. }
  796. }
  797. auto all_code_points = flatten_code_point_ranges(code_points);
  798. code_points = form_code_point_ranges(all_code_points);
  799. }
  800. static void populate_general_category_unions(PropList& general_categories)
  801. {
  802. // The Unicode standard defines General Category values which are not in any UCD file. These
  803. // values are simply unions of other values.
  804. // https://www.unicode.org/reports/tr44/#GC_Values_Table
  805. auto populate_union = [&](auto alias, auto categories) {
  806. auto& code_points = general_categories.ensure(alias);
  807. for (auto const& category : categories)
  808. code_points.extend(general_categories.find(category)->value);
  809. sort_and_merge_code_point_ranges(code_points);
  810. };
  811. populate_union("LC"sv, Array { "Ll"sv, "Lu"sv, "Lt"sv });
  812. populate_union("L"sv, Array { "Lu"sv, "Ll"sv, "Lt"sv, "Lm"sv, "Lo"sv });
  813. populate_union("M"sv, Array { "Mn"sv, "Mc"sv, "Me"sv });
  814. populate_union("N"sv, Array { "Nd"sv, "Nl"sv, "No"sv });
  815. populate_union("P"sv, Array { "Pc"sv, "Pd"sv, "Ps"sv, "Pe"sv, "Pi"sv, "Pf"sv, "Po"sv });
  816. populate_union("S"sv, Array { "Sm"sv, "Sc"sv, "Sk"sv, "So"sv });
  817. populate_union("Z"sv, Array { "Zs"sv, "Zl"sv, "Zp"sv });
  818. populate_union("C"sv, Array { "Cc"sv, "Cf"sv, "Cs"sv, "Co"sv, "Cn"sv });
  819. }
  820. static void normalize_script_extensions(PropList& script_extensions, PropList const& script_list, Vector<Alias> const& script_aliases)
  821. {
  822. // The ScriptExtensions UCD file lays out its code point ranges rather uniquely compared to
  823. // other files. The Script listed on each line may either be a full Script string or an aliased
  824. // abbreviation. Further, the extensions may or may not include the base Script list. Normalize
  825. // the extensions here to be keyed by the full Script name and always include the base list.
  826. auto extensions = move(script_extensions);
  827. script_extensions = script_list;
  828. for (auto const& extension : extensions) {
  829. auto it = find_if(script_aliases.begin(), script_aliases.end(), [&](auto const& alias) { return extension.key == alias.alias; });
  830. auto const& key = (it == script_aliases.end()) ? extension.key : it->name;
  831. auto& code_points = script_extensions.find(key)->value;
  832. code_points.extend(extension.value);
  833. sort_and_merge_code_point_ranges(code_points);
  834. }
  835. // Lastly, the Common and Inherited script extensions are special. They must not contain any
  836. // code points which appear in other script extensions. The ScriptExtensions UCD file does not
  837. // list these extensions, therefore this peculiarity must be handled programmatically.
  838. // https://www.unicode.org/reports/tr24/#Assignment_ScriptX_Values
  839. auto code_point_has_other_extension = [&](StringView key, u32 code_point) {
  840. for (auto const& extension : extensions) {
  841. if (extension.key == key)
  842. continue;
  843. if (any_of(extension.value, [&](auto const& r) { return (r.first <= code_point) && (code_point <= r.last); }))
  844. return true;
  845. }
  846. return false;
  847. };
  848. auto get_code_points_without_other_extensions = [&](StringView key) {
  849. auto code_points = flatten_code_point_ranges(script_list.find(key)->value);
  850. code_points.remove_all_matching([&](u32 c) { return code_point_has_other_extension(key, c); });
  851. return code_points;
  852. };
  853. auto common_code_points = get_code_points_without_other_extensions("Common"sv);
  854. script_extensions.set("Common"sv, form_code_point_ranges(common_code_points));
  855. auto inherited_code_points = get_code_points_without_other_extensions("Inherited"sv);
  856. script_extensions.set("Inherited"sv, form_code_point_ranges(inherited_code_points));
  857. }
  858. ErrorOr<int> serenity_main(Main::Arguments arguments)
  859. {
  860. StringView generated_header_path;
  861. StringView generated_implementation_path;
  862. StringView unicode_data_path;
  863. StringView special_casing_path;
  864. StringView derived_general_category_path;
  865. StringView prop_list_path;
  866. StringView derived_core_prop_path;
  867. StringView derived_binary_prop_path;
  868. StringView prop_alias_path;
  869. StringView prop_value_alias_path;
  870. StringView name_alias_path;
  871. StringView scripts_path;
  872. StringView script_extensions_path;
  873. StringView emoji_data_path;
  874. StringView normalization_path;
  875. Core::ArgsParser args_parser;
  876. args_parser.add_option(generated_header_path, "Path to the Unicode Data header file to generate", "generated-header-path", 'h', "generated-header-path");
  877. args_parser.add_option(generated_implementation_path, "Path to the Unicode Data implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
  878. args_parser.add_option(unicode_data_path, "Path to UnicodeData.txt file", "unicode-data-path", 'u', "unicode-data-path");
  879. args_parser.add_option(special_casing_path, "Path to SpecialCasing.txt file", "special-casing-path", 's', "special-casing-path");
  880. args_parser.add_option(derived_general_category_path, "Path to DerivedGeneralCategory.txt file", "derived-general-category-path", 'g', "derived-general-category-path");
  881. args_parser.add_option(prop_list_path, "Path to PropList.txt file", "prop-list-path", 'p', "prop-list-path");
  882. args_parser.add_option(derived_core_prop_path, "Path to DerivedCoreProperties.txt file", "derived-core-prop-path", 'd', "derived-core-prop-path");
  883. args_parser.add_option(derived_binary_prop_path, "Path to DerivedBinaryProperties.txt file", "derived-binary-prop-path", 'b', "derived-binary-prop-path");
  884. args_parser.add_option(prop_alias_path, "Path to PropertyAliases.txt file", "prop-alias-path", 'a', "prop-alias-path");
  885. args_parser.add_option(prop_value_alias_path, "Path to PropertyValueAliases.txt file", "prop-value-alias-path", 'v', "prop-value-alias-path");
  886. args_parser.add_option(name_alias_path, "Path to NameAliases.txt file", "name-alias-path", 'm', "name-alias-path");
  887. args_parser.add_option(scripts_path, "Path to Scripts.txt file", "scripts-path", 'r', "scripts-path");
  888. args_parser.add_option(script_extensions_path, "Path to ScriptExtensions.txt file", "script-extensions-path", 'x', "script-extensions-path");
  889. args_parser.add_option(emoji_data_path, "Path to emoji-data.txt file", "emoji-data-path", 'e', "emoji-data-path");
  890. args_parser.add_option(normalization_path, "Path to DerivedNormalizationProps.txt file", "normalization-path", 'n', "normalization-path");
  891. args_parser.parse(arguments);
  892. auto open_file = [&](StringView path, Core::OpenMode mode = Core::OpenMode::ReadOnly) -> ErrorOr<NonnullRefPtr<Core::File>> {
  893. if (path.is_empty()) {
  894. args_parser.print_usage(stderr, arguments.argv[0]);
  895. return Error::from_string_literal("Must provide all command line options"sv);
  896. }
  897. return Core::File::open(path, mode);
  898. };
  899. auto generated_header_file = TRY(open_file(generated_header_path, Core::OpenMode::ReadWrite));
  900. auto generated_implementation_file = TRY(open_file(generated_implementation_path, Core::OpenMode::ReadWrite));
  901. auto unicode_data_file = TRY(open_file(unicode_data_path));
  902. auto derived_general_category_file = TRY(open_file(derived_general_category_path));
  903. auto special_casing_file = TRY(open_file(special_casing_path));
  904. auto prop_list_file = TRY(open_file(prop_list_path));
  905. auto derived_core_prop_file = TRY(open_file(derived_core_prop_path));
  906. auto derived_binary_prop_file = TRY(open_file(derived_binary_prop_path));
  907. auto prop_alias_file = TRY(open_file(prop_alias_path));
  908. auto prop_value_alias_file = TRY(open_file(prop_value_alias_path));
  909. auto name_alias_file = TRY(open_file(name_alias_path));
  910. auto scripts_file = TRY(open_file(scripts_path));
  911. auto script_extensions_file = TRY(open_file(script_extensions_path));
  912. auto emoji_data_file = TRY(open_file(emoji_data_path));
  913. auto normalization_file = TRY(open_file(normalization_path));
  914. UnicodeData unicode_data {};
  915. parse_special_casing(special_casing_file, unicode_data);
  916. parse_prop_list(derived_general_category_file, unicode_data.general_categories);
  917. parse_prop_list(prop_list_file, unicode_data.prop_list);
  918. parse_prop_list(derived_core_prop_file, unicode_data.prop_list);
  919. parse_prop_list(derived_binary_prop_file, unicode_data.prop_list);
  920. parse_prop_list(emoji_data_file, unicode_data.prop_list);
  921. parse_normalization_props(normalization_file, unicode_data);
  922. parse_alias_list(prop_alias_file, unicode_data.prop_list, unicode_data.prop_aliases);
  923. parse_prop_list(scripts_file, unicode_data.script_list);
  924. parse_prop_list(script_extensions_file, unicode_data.script_extensions, true);
  925. parse_name_aliases(name_alias_file, unicode_data);
  926. populate_general_category_unions(unicode_data.general_categories);
  927. parse_unicode_data(unicode_data_file, unicode_data);
  928. parse_value_alias_list(prop_value_alias_file, "gc"sv, unicode_data.general_categories.keys(), unicode_data.general_category_aliases);
  929. parse_value_alias_list(prop_value_alias_file, "sc"sv, unicode_data.script_list.keys(), unicode_data.script_aliases, false);
  930. normalize_script_extensions(unicode_data.script_extensions, unicode_data.script_list, unicode_data.script_aliases);
  931. generate_unicode_data_header(generated_header_file, unicode_data);
  932. generate_unicode_data_implementation(generated_implementation_file, unicode_data);
  933. return 0;
  934. }