GenerateUnicodeData.cpp 63 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566
  1. /*
  2. * Copyright (c) 2021, Tim Flynn <trflynn89@serenityos.org>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include "GeneratorUtil.h"
  7. #include <AK/AllOf.h>
  8. #include <AK/Array.h>
  9. #include <AK/CharacterTypes.h>
  10. #include <AK/DeprecatedString.h>
  11. #include <AK/Error.h>
  12. #include <AK/Find.h>
  13. #include <AK/HashMap.h>
  14. #include <AK/Optional.h>
  15. #include <AK/QuickSort.h>
  16. #include <AK/SourceGenerator.h>
  17. #include <AK/StringUtils.h>
  18. #include <AK/Types.h>
  19. #include <AK/Vector.h>
  20. #include <LibCore/ArgsParser.h>
  21. // Some code points are excluded from UnicodeData.txt, and instead are part of a "range" of code
  22. // points, as indicated by the "name" field. For example:
  23. // 3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
  24. // 4DBF;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
  25. struct CodePointRange {
  26. u32 first;
  27. u32 last;
  28. };
  29. // https://www.unicode.org/reports/tr44/#SpecialCasing.txt
  30. struct SpecialCasing {
  31. u32 index { 0 };
  32. u32 code_point { 0 };
  33. Vector<u32> lowercase_mapping;
  34. Vector<u32> uppercase_mapping;
  35. Vector<u32> titlecase_mapping;
  36. DeprecatedString locale;
  37. DeprecatedString condition;
  38. };
  39. // https://www.unicode.org/reports/tr44/#CaseFolding.txt
  40. struct CaseFolding {
  41. u32 code_point { 0 };
  42. StringView status { "Common"sv };
  43. Vector<u32> mapping { 0 };
  44. };
  45. // https://www.unicode.org/reports/tr44/#Character_Decomposition_Mappings
  46. struct CodePointDecomposition {
  47. // `tag` is a string since it's used for codegen as an enum value.
  48. DeprecatedString tag { "Canonical"sv };
  49. size_t decomposition_index { 0 };
  50. size_t decomposition_size { 0 };
  51. };
  52. // https://www.unicode.org/reports/tr44/#PropList.txt
  53. using PropList = HashMap<DeprecatedString, Vector<CodePointRange>>;
  54. // https://www.unicode.org/reports/tr44/#DerivedNormalizationProps.txt
  55. enum class QuickCheck {
  56. Yes,
  57. No,
  58. Maybe,
  59. };
  60. struct Normalization {
  61. CodePointRange code_point_range;
  62. Vector<u32> value;
  63. QuickCheck quick_check { QuickCheck::Yes };
  64. };
  65. using NormalizationProps = HashMap<DeprecatedString, Vector<Normalization>>;
  66. struct CodePointName {
  67. CodePointRange code_point_range;
  68. size_t name { 0 };
  69. };
  70. // https://www.unicode.org/reports/tr44/#UnicodeData.txt
  71. struct CodePointData {
  72. u32 code_point { 0 };
  73. DeprecatedString name;
  74. Optional<size_t> abbreviation;
  75. u8 canonical_combining_class { 0 };
  76. DeprecatedString bidi_class;
  77. Optional<CodePointDecomposition> decomposition_mapping;
  78. Optional<i8> numeric_value_decimal;
  79. Optional<i8> numeric_value_digit;
  80. Optional<i8> numeric_value_numeric;
  81. bool bidi_mirrored { false };
  82. DeprecatedString unicode_1_name;
  83. DeprecatedString iso_comment;
  84. Optional<u32> simple_uppercase_mapping;
  85. Optional<u32> simple_lowercase_mapping;
  86. Optional<u32> simple_titlecase_mapping;
  87. Vector<u32> special_casing_indices;
  88. Vector<u32> case_folding_indices;
  89. };
  90. struct BlockName {
  91. CodePointRange code_point_range;
  92. size_t name { 0 };
  93. };
  94. struct UnicodeData {
  95. UniqueStringStorage unique_strings;
  96. u32 code_points_with_non_zero_combining_class { 0 };
  97. u32 code_points_with_decomposition_mapping { 0 };
  98. Vector<u32> decomposition_mappings;
  99. Vector<DeprecatedString> compatibility_tags;
  100. u32 simple_uppercase_mapping_size { 0 };
  101. u32 simple_lowercase_mapping_size { 0 };
  102. u32 simple_titlecase_mapping_size { 0 };
  103. Vector<SpecialCasing> special_casing;
  104. u32 code_points_with_special_casing { 0 };
  105. u32 largest_special_casing_mapping_size { 0 };
  106. u32 largest_special_casing_size { 0 };
  107. Vector<DeprecatedString> conditions;
  108. Vector<DeprecatedString> locales;
  109. Vector<CaseFolding> case_folding;
  110. u32 code_points_with_case_folding { 0 };
  111. u32 largest_case_folding_mapping_size { 0 };
  112. u32 largest_case_folding_size { 0 };
  113. Vector<StringView> statuses;
  114. Vector<CodePointData> code_point_data;
  115. HashMap<u32, size_t> code_point_abbreviations;
  116. HashMap<u32, size_t> code_point_display_name_aliases;
  117. Vector<CodePointName> code_point_display_names;
  118. // https://www.unicode.org/reports/tr44/#General_Category_Values
  119. PropList general_categories;
  120. Vector<Alias> general_category_aliases;
  121. // The Unicode standard defines additional properties (Any, Assigned, ASCII) which are not in
  122. // any UCD file. Assigned code point ranges are derived as this generator is executed.
  123. // https://unicode.org/reports/tr18/#General_Category_Property
  124. PropList prop_list {
  125. { "Any"sv, { { 0, 0x10ffff } } },
  126. { "Assigned"sv, {} },
  127. { "ASCII"sv, { { 0, 0x7f } } },
  128. };
  129. Vector<Alias> prop_aliases;
  130. PropList script_list {
  131. { "Unknown"sv, {} },
  132. };
  133. Vector<Alias> script_aliases;
  134. PropList script_extensions;
  135. Vector<BlockName> block_display_names;
  136. // FIXME: We are not yet doing anything with this data. It will be needed for String.prototype.normalize.
  137. NormalizationProps normalization_props;
  138. PropList grapheme_break_props;
  139. PropList word_break_props;
  140. PropList sentence_break_props;
  141. };
  142. static DeprecatedString sanitize_entry(DeprecatedString const& entry)
  143. {
  144. auto sanitized = entry.replace("-"sv, "_"sv, ReplaceMode::All);
  145. sanitized = sanitized.replace(" "sv, "_"sv, ReplaceMode::All);
  146. StringBuilder builder;
  147. bool next_is_upper = true;
  148. for (auto ch : sanitized) {
  149. if (next_is_upper)
  150. builder.append_code_point(to_ascii_uppercase(ch));
  151. else
  152. builder.append_code_point(ch);
  153. next_is_upper = ch == '_';
  154. }
  155. return builder.to_deprecated_string();
  156. }
  157. static Vector<u32> parse_code_point_list(StringView list)
  158. {
  159. Vector<u32> code_points;
  160. auto segments = list.split_view(' ');
  161. for (auto const& code_point : segments)
  162. code_points.append(AK::StringUtils::convert_to_uint_from_hex<u32>(code_point).value());
  163. return code_points;
  164. }
  165. static CodePointRange parse_code_point_range(StringView list)
  166. {
  167. CodePointRange code_point_range {};
  168. if (list.contains(".."sv)) {
  169. auto segments = list.split_view(".."sv);
  170. VERIFY(segments.size() == 2);
  171. auto begin = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[0]).value();
  172. auto end = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[1]).value();
  173. code_point_range = { begin, end };
  174. } else {
  175. auto code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(list).value();
  176. code_point_range = { code_point, code_point };
  177. }
  178. return code_point_range;
  179. }
  180. static ErrorOr<void> parse_special_casing(Core::InputBufferedFile& file, UnicodeData& unicode_data)
  181. {
  182. Array<u8, 1024> buffer;
  183. while (TRY(file.can_read_line())) {
  184. auto line = TRY(file.read_line(buffer));
  185. if (line.is_empty() || line.starts_with('#'))
  186. continue;
  187. if (auto index = line.find('#'); index.has_value())
  188. line = line.substring_view(0, *index);
  189. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  190. VERIFY(segments.size() == 5 || segments.size() == 6);
  191. SpecialCasing casing {};
  192. casing.code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[0]).value();
  193. casing.lowercase_mapping = parse_code_point_list(segments[1]);
  194. casing.titlecase_mapping = parse_code_point_list(segments[2]);
  195. casing.uppercase_mapping = parse_code_point_list(segments[3]);
  196. if (auto condition = segments[4].trim_whitespace(); !condition.is_empty()) {
  197. auto conditions = condition.split_view(' ', SplitBehavior::KeepEmpty);
  198. VERIFY(conditions.size() == 1 || conditions.size() == 2);
  199. if (conditions.size() == 2) {
  200. casing.locale = conditions[0];
  201. casing.condition = conditions[1];
  202. } else if (all_of(conditions[0], is_ascii_lower_alpha)) {
  203. casing.locale = conditions[0];
  204. } else {
  205. casing.condition = conditions[0];
  206. }
  207. if (!casing.locale.is_empty()) {
  208. casing.locale = DeprecatedString::formatted("{:c}{}", to_ascii_uppercase(casing.locale[0]), casing.locale.substring_view(1));
  209. if (!unicode_data.locales.contains_slow(casing.locale))
  210. unicode_data.locales.append(casing.locale);
  211. }
  212. casing.condition = casing.condition.replace("_"sv, ""sv, ReplaceMode::All);
  213. if (!casing.condition.is_empty() && !unicode_data.conditions.contains_slow(casing.condition))
  214. unicode_data.conditions.append(casing.condition);
  215. }
  216. unicode_data.largest_special_casing_mapping_size = max(unicode_data.largest_special_casing_mapping_size, casing.lowercase_mapping.size());
  217. unicode_data.largest_special_casing_mapping_size = max(unicode_data.largest_special_casing_mapping_size, casing.titlecase_mapping.size());
  218. unicode_data.largest_special_casing_mapping_size = max(unicode_data.largest_special_casing_mapping_size, casing.uppercase_mapping.size());
  219. unicode_data.special_casing.append(move(casing));
  220. }
  221. quick_sort(unicode_data.special_casing, [](auto const& lhs, auto const& rhs) {
  222. if (lhs.code_point != rhs.code_point)
  223. return lhs.code_point < rhs.code_point;
  224. if (lhs.locale.is_empty() && !rhs.locale.is_empty())
  225. return false;
  226. if (!lhs.locale.is_empty() && rhs.locale.is_empty())
  227. return true;
  228. return lhs.locale < rhs.locale;
  229. });
  230. for (u32 i = 0; i < unicode_data.special_casing.size(); ++i)
  231. unicode_data.special_casing[i].index = i;
  232. return {};
  233. }
  234. static ErrorOr<void> parse_case_folding(Core::InputBufferedFile& file, UnicodeData& unicode_data)
  235. {
  236. Array<u8, 1024> buffer;
  237. while (TRY(file.can_read_line())) {
  238. auto line = TRY(file.read_line(buffer));
  239. if (line.is_empty() || line.starts_with('#'))
  240. continue;
  241. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  242. VERIFY(segments.size() == 4);
  243. CaseFolding folding {};
  244. folding.code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[0]).value();
  245. folding.mapping = parse_code_point_list(segments[2]);
  246. switch (segments[1].trim_whitespace()[0]) {
  247. case 'C':
  248. folding.status = "Common"sv;
  249. break;
  250. case 'F':
  251. folding.status = "Full"sv;
  252. break;
  253. case 'S':
  254. folding.status = "Simple"sv;
  255. break;
  256. case 'T':
  257. folding.status = "Special"sv;
  258. break;
  259. }
  260. unicode_data.largest_case_folding_mapping_size = max(unicode_data.largest_case_folding_mapping_size, folding.mapping.size());
  261. if (!unicode_data.statuses.contains_slow(folding.status))
  262. unicode_data.statuses.append(folding.status);
  263. unicode_data.case_folding.append(move(folding));
  264. }
  265. quick_sort(unicode_data.case_folding, [](auto const& lhs, auto const& rhs) {
  266. if (lhs.code_point != rhs.code_point)
  267. return lhs.code_point < rhs.code_point;
  268. return lhs.status < rhs.status;
  269. });
  270. return {};
  271. }
  272. static ErrorOr<void> parse_prop_list(Core::InputBufferedFile& file, PropList& prop_list, bool multi_value_property = false, bool sanitize_property = false)
  273. {
  274. Array<u8, 1024> buffer;
  275. while (TRY(file.can_read_line())) {
  276. auto line = TRY(file.read_line(buffer));
  277. if (line.is_empty() || line.starts_with('#'))
  278. continue;
  279. if (auto index = line.find('#'); index.has_value())
  280. line = line.substring_view(0, *index);
  281. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  282. VERIFY(segments.size() == 2);
  283. auto code_point_range = parse_code_point_range(segments[0].trim_whitespace());
  284. Vector<StringView> properties;
  285. if (multi_value_property)
  286. properties = segments[1].trim_whitespace().split_view(' ');
  287. else
  288. properties = { segments[1].trim_whitespace() };
  289. for (auto& property : properties) {
  290. auto& code_points = prop_list.ensure(sanitize_property ? sanitize_entry(property).trim_whitespace().view() : property.trim_whitespace());
  291. code_points.append(code_point_range);
  292. }
  293. }
  294. return {};
  295. }
  296. static ErrorOr<void> parse_alias_list(Core::InputBufferedFile& file, PropList const& prop_list, Vector<Alias>& prop_aliases)
  297. {
  298. DeprecatedString current_property;
  299. Array<u8, 1024> buffer;
  300. auto append_alias = [&](auto alias, auto property) {
  301. // Note: The alias files contain lines such as "Hyphen = Hyphen", which we should just skip.
  302. if (alias == property)
  303. return;
  304. // FIXME: We will, eventually, need to find where missing properties are located and parse them.
  305. if (!prop_list.contains(property))
  306. return;
  307. prop_aliases.append({ property, alias });
  308. };
  309. while (TRY(file.can_read_line())) {
  310. auto line = TRY(file.read_line(buffer));
  311. if (line.is_empty() || line.starts_with('#')) {
  312. if (line.ends_with("Properties"sv))
  313. current_property = line.substring_view(2);
  314. continue;
  315. }
  316. // Note: For now, we only care about Binary Property aliases for Unicode property escapes.
  317. if (current_property != "Binary Properties"sv)
  318. continue;
  319. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  320. VERIFY((segments.size() == 2) || (segments.size() == 3));
  321. auto alias = segments[0].trim_whitespace();
  322. auto property = segments[1].trim_whitespace();
  323. append_alias(alias, property);
  324. if (segments.size() == 3) {
  325. alias = segments[2].trim_whitespace();
  326. append_alias(alias, property);
  327. }
  328. }
  329. return {};
  330. }
  331. static ErrorOr<void> parse_name_aliases(Core::InputBufferedFile& file, UnicodeData& unicode_data)
  332. {
  333. Array<u8, 1024> buffer;
  334. while (TRY(file.can_read_line())) {
  335. auto line = TRY(file.read_line(buffer));
  336. if (line.is_empty() || line.starts_with('#'))
  337. continue;
  338. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  339. VERIFY(segments.size() == 3);
  340. auto code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[0].trim_whitespace());
  341. auto alias = segments[1].trim_whitespace();
  342. auto reason = segments[2].trim_whitespace();
  343. if (reason == "abbreviation"sv) {
  344. auto index = unicode_data.unique_strings.ensure(alias);
  345. unicode_data.code_point_abbreviations.set(*code_point, index);
  346. } else if (reason.is_one_of("correction"sv, "control"sv)) {
  347. if (!unicode_data.code_point_display_name_aliases.contains(*code_point)) {
  348. auto index = unicode_data.unique_strings.ensure(alias);
  349. unicode_data.code_point_display_name_aliases.set(*code_point, index);
  350. }
  351. }
  352. }
  353. return {};
  354. }
  355. static ErrorOr<void> parse_value_alias_list(Core::InputBufferedFile& file, StringView desired_category, Vector<DeprecatedString> const& value_list, Vector<Alias>& prop_aliases, bool primary_value_is_first = true, bool sanitize_alias = false)
  356. {
  357. TRY(file.seek(0, SeekMode::SetPosition));
  358. Array<u8, 1024> buffer;
  359. auto append_alias = [&](auto alias, auto value) {
  360. // Note: The value alias file contains lines such as "Ahom = Ahom", which we should just skip.
  361. if (alias == value)
  362. return;
  363. // FIXME: We will, eventually, need to find where missing properties are located and parse them.
  364. if (!value_list.contains_slow(value))
  365. return;
  366. prop_aliases.append({ value, alias });
  367. };
  368. while (TRY(file.can_read_line())) {
  369. auto line = TRY(file.read_line(buffer));
  370. if (line.is_empty() || line.starts_with('#'))
  371. continue;
  372. if (auto index = line.find('#'); index.has_value())
  373. line = line.substring_view(0, *index);
  374. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  375. auto category = segments[0].trim_whitespace();
  376. if (category != desired_category)
  377. continue;
  378. VERIFY((segments.size() == 3) || (segments.size() == 4));
  379. auto value = primary_value_is_first ? segments[1].trim_whitespace() : segments[2].trim_whitespace();
  380. auto alias = primary_value_is_first ? segments[2].trim_whitespace() : segments[1].trim_whitespace();
  381. append_alias(sanitize_alias ? sanitize_entry(alias).view() : alias, value);
  382. if (segments.size() == 4) {
  383. alias = segments[3].trim_whitespace();
  384. append_alias(sanitize_alias ? sanitize_entry(alias).view() : alias, value);
  385. }
  386. }
  387. return {};
  388. }
  389. static ErrorOr<void> parse_normalization_props(Core::InputBufferedFile& file, UnicodeData& unicode_data)
  390. {
  391. Array<u8, 1024> buffer;
  392. while (TRY(file.can_read_line())) {
  393. auto line = TRY(file.read_line(buffer));
  394. if (line.is_empty() || line.starts_with('#'))
  395. continue;
  396. if (auto index = line.find('#'); index.has_value())
  397. line = line.substring_view(0, *index);
  398. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  399. VERIFY((segments.size() == 2) || (segments.size() == 3));
  400. auto code_point_range = parse_code_point_range(segments[0].trim_whitespace());
  401. auto property = segments[1].trim_whitespace().to_deprecated_string();
  402. Vector<u32> value;
  403. QuickCheck quick_check = QuickCheck::Yes;
  404. if (segments.size() == 3) {
  405. auto value_or_quick_check = segments[2].trim_whitespace();
  406. if ((value_or_quick_check == "N"sv))
  407. quick_check = QuickCheck::No;
  408. else if ((value_or_quick_check == "M"sv))
  409. quick_check = QuickCheck::Maybe;
  410. else
  411. value = parse_code_point_list(value_or_quick_check);
  412. }
  413. auto& normalizations = unicode_data.normalization_props.ensure(property);
  414. normalizations.append({ code_point_range, move(value), quick_check });
  415. auto& prop_list = unicode_data.prop_list.ensure(property);
  416. prop_list.append(move(code_point_range));
  417. }
  418. return {};
  419. }
  420. static void add_canonical_code_point_name(CodePointRange range, StringView name, UnicodeData& unicode_data)
  421. {
  422. // https://www.unicode.org/versions/Unicode15.0.0/ch04.pdf#G142981
  423. // FIXME: Implement the NR1 rules for Hangul syllables.
  424. struct CodePointNameFormat {
  425. CodePointRange code_point_range;
  426. StringView name;
  427. };
  428. // These code point ranges are the NR2 set of name replacements defined by Table 4-8.
  429. constexpr Array<CodePointNameFormat, 16> s_ideographic_replacements { {
  430. { { 0x3400, 0x4DBF }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  431. { { 0x4E00, 0x9FFF }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  432. { { 0xF900, 0xFA6D }, "CJK COMPATIBILITY IDEOGRAPH-{:X}"sv },
  433. { { 0xFA70, 0xFAD9 }, "CJK COMPATIBILITY IDEOGRAPH-{:X}"sv },
  434. { { 0x17000, 0x187F7 }, "TANGUT IDEOGRAPH-{:X}"sv },
  435. { { 0x18B00, 0x18CD5 }, "KHITAN SMALL SCRIPT CHARACTER-{:X}"sv },
  436. { { 0x18D00, 0x18D08 }, "TANGUT IDEOGRAPH-{:X}"sv },
  437. { { 0x1B170, 0x1B2FB }, "NUSHU CHARACTER-{:X}"sv },
  438. { { 0x20000, 0x2A6DF }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  439. { { 0x2A700, 0x2B739 }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  440. { { 0x2B740, 0x2B81D }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  441. { { 0x2B820, 0x2CEA1 }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  442. { { 0x2CEB0, 0x2EBE0 }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  443. { { 0x2F800, 0x2FA1D }, "CJK COMPATIBILITY IDEOGRAPH-{:X}"sv },
  444. { { 0x30000, 0x3134A }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  445. { { 0x31350, 0x323AF }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  446. } };
  447. auto it = find_if(s_ideographic_replacements.begin(), s_ideographic_replacements.end(),
  448. [&](auto const& replacement) {
  449. return replacement.code_point_range.first == range.first;
  450. });
  451. if (it != s_ideographic_replacements.end()) {
  452. auto index = unicode_data.unique_strings.ensure(it->name);
  453. unicode_data.code_point_display_names.append({ it->code_point_range, index });
  454. return;
  455. }
  456. it = find_if(s_ideographic_replacements.begin(), s_ideographic_replacements.end(),
  457. [&](auto const& replacement) {
  458. return (replacement.code_point_range.first <= range.first) && (range.first <= replacement.code_point_range.last);
  459. });
  460. if (it != s_ideographic_replacements.end()) {
  461. // Drop code points that will have been captured by a range defined by the ideographic replacements.
  462. return;
  463. }
  464. if (auto alias = unicode_data.code_point_display_name_aliases.get(range.first); alias.has_value()) {
  465. // NR4 states that control code points have a null string as their name. Our implementation
  466. // uses the control code's alias as its display name.
  467. unicode_data.code_point_display_names.append({ range, *alias });
  468. return;
  469. }
  470. auto index = unicode_data.unique_strings.ensure(name);
  471. unicode_data.code_point_display_names.append({ range, index });
  472. }
  473. static Optional<CodePointDecomposition> parse_decomposition_mapping(StringView string, UnicodeData& unicode_data)
  474. {
  475. if (string.is_empty())
  476. return {};
  477. CodePointDecomposition mapping;
  478. auto parts = string.split_view(' ');
  479. VERIFY(parts.size() > 0);
  480. if (parts.first().starts_with('<')) {
  481. auto const tag = parts.take_first().trim("<>"sv);
  482. mapping.tag = DeprecatedString::formatted("{:c}{}", to_ascii_uppercase(tag[0]), tag.substring_view(1));
  483. if (!unicode_data.compatibility_tags.contains_slow(mapping.tag))
  484. unicode_data.compatibility_tags.append(mapping.tag);
  485. }
  486. mapping.decomposition_index = unicode_data.decomposition_mappings.size();
  487. mapping.decomposition_size = parts.size();
  488. for (auto part : parts) {
  489. unicode_data.decomposition_mappings.append(AK::StringUtils::convert_to_uint_from_hex<u32>(part).value());
  490. }
  491. return mapping;
  492. }
  493. static ErrorOr<void> parse_block_display_names(Core::InputBufferedFile& file, UnicodeData& unicode_data)
  494. {
  495. Array<u8, 1024> buffer;
  496. while (TRY(file.can_read_line())) {
  497. auto line = TRY(file.read_line(buffer));
  498. if (line.is_empty() || line.starts_with('#'))
  499. continue;
  500. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  501. VERIFY(segments.size() == 2);
  502. auto code_point_range = parse_code_point_range(segments[0].trim_whitespace());
  503. auto display_name = segments[1].trim_whitespace();
  504. auto index = unicode_data.unique_strings.ensure(display_name);
  505. unicode_data.block_display_names.append({ code_point_range, index });
  506. }
  507. TRY(file.seek(0, SeekMode::SetPosition));
  508. return {};
  509. }
  510. static ErrorOr<void> parse_unicode_data(Core::InputBufferedFile& file, UnicodeData& unicode_data)
  511. {
  512. Optional<u32> code_point_range_start;
  513. auto& assigned_code_points = unicode_data.prop_list.find("Assigned"sv)->value;
  514. Optional<u32> assigned_code_point_range_start = 0;
  515. u32 previous_code_point = 0;
  516. Array<u8, 1024> buffer;
  517. while (TRY(file.can_read_line())) {
  518. auto line = TRY(file.read_line(buffer));
  519. if (line.is_empty())
  520. continue;
  521. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  522. VERIFY(segments.size() == 15);
  523. CodePointData data {};
  524. data.code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[0]).value();
  525. data.name = segments[1];
  526. data.canonical_combining_class = AK::StringUtils::convert_to_uint<u8>(segments[3]).value();
  527. data.bidi_class = segments[4];
  528. data.decomposition_mapping = parse_decomposition_mapping(segments[5], unicode_data);
  529. data.numeric_value_decimal = AK::StringUtils::convert_to_int<i8>(segments[6]);
  530. data.numeric_value_digit = AK::StringUtils::convert_to_int<i8>(segments[7]);
  531. data.numeric_value_numeric = AK::StringUtils::convert_to_int<i8>(segments[8]);
  532. data.bidi_mirrored = segments[9] == "Y"sv;
  533. data.unicode_1_name = segments[10];
  534. data.iso_comment = segments[11];
  535. data.simple_uppercase_mapping = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[12]);
  536. data.simple_lowercase_mapping = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[13]);
  537. data.simple_titlecase_mapping = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[14]);
  538. if (auto abbreviation = unicode_data.code_point_abbreviations.get(data.code_point); abbreviation.has_value())
  539. data.abbreviation = *abbreviation;
  540. if (!assigned_code_point_range_start.has_value())
  541. assigned_code_point_range_start = data.code_point;
  542. if (data.name.starts_with("<"sv) && data.name.ends_with(", First>"sv)) {
  543. VERIFY(!code_point_range_start.has_value() && assigned_code_point_range_start.has_value());
  544. code_point_range_start = data.code_point;
  545. data.name = data.name.substring(1, data.name.length() - 9);
  546. assigned_code_points.append({ *assigned_code_point_range_start, previous_code_point });
  547. assigned_code_point_range_start.clear();
  548. } else if (data.name.starts_with("<"sv) && data.name.ends_with(", Last>"sv)) {
  549. VERIFY(code_point_range_start.has_value());
  550. CodePointRange code_point_range { *code_point_range_start, data.code_point };
  551. assigned_code_points.append(code_point_range);
  552. data.name = data.name.substring(1, data.name.length() - 8);
  553. code_point_range_start.clear();
  554. add_canonical_code_point_name(code_point_range, data.name, unicode_data);
  555. } else {
  556. add_canonical_code_point_name({ data.code_point, data.code_point }, data.name, unicode_data);
  557. if ((data.code_point > 0) && (data.code_point - previous_code_point) != 1) {
  558. VERIFY(assigned_code_point_range_start.has_value());
  559. assigned_code_points.append({ *assigned_code_point_range_start, previous_code_point });
  560. assigned_code_point_range_start = data.code_point;
  561. }
  562. }
  563. bool has_special_casing { false };
  564. for (auto const& casing : unicode_data.special_casing) {
  565. if (casing.code_point == data.code_point) {
  566. data.special_casing_indices.append(casing.index);
  567. has_special_casing = true;
  568. }
  569. }
  570. bool has_case_folding { false };
  571. for (size_t i = 0; i < unicode_data.case_folding.size(); ++i) {
  572. if (auto const& folding = unicode_data.case_folding[i]; folding.code_point == data.code_point) {
  573. data.case_folding_indices.append(i);
  574. has_case_folding = true;
  575. }
  576. }
  577. unicode_data.code_points_with_non_zero_combining_class += data.canonical_combining_class != 0;
  578. unicode_data.simple_uppercase_mapping_size += data.simple_uppercase_mapping.has_value();
  579. unicode_data.simple_lowercase_mapping_size += data.simple_lowercase_mapping.has_value();
  580. unicode_data.simple_titlecase_mapping_size += data.simple_titlecase_mapping.has_value();
  581. unicode_data.code_points_with_decomposition_mapping += data.decomposition_mapping.has_value();
  582. unicode_data.code_points_with_special_casing += has_special_casing;
  583. unicode_data.largest_special_casing_size = max(unicode_data.largest_special_casing_size, data.special_casing_indices.size());
  584. unicode_data.code_points_with_case_folding += has_case_folding;
  585. unicode_data.largest_case_folding_size = max(unicode_data.largest_case_folding_size, data.case_folding_indices.size());
  586. previous_code_point = data.code_point;
  587. unicode_data.code_point_data.append(move(data));
  588. }
  589. return {};
  590. }
  591. static ErrorOr<void> generate_unicode_data_header(Core::InputBufferedFile& file, UnicodeData& unicode_data)
  592. {
  593. StringBuilder builder;
  594. SourceGenerator generator { builder };
  595. generator.set("special_casing_mapping_size", DeprecatedString::number(unicode_data.largest_special_casing_mapping_size));
  596. generator.set("case_folding_mapping_size", DeprecatedString::number(unicode_data.largest_case_folding_mapping_size));
  597. auto generate_enum = [&](StringView name, StringView default_, auto values, Vector<Alias> aliases = {}) {
  598. quick_sort(values);
  599. quick_sort(aliases, [](auto& alias1, auto& alias2) { return alias1.alias < alias2.alias; });
  600. generator.set("name", name);
  601. generator.set("underlying", DeprecatedString::formatted("{}UnderlyingType", name));
  602. generator.set("type", ((values.size() + !default_.is_empty()) < 256) ? "u8"sv : "u16"sv);
  603. generator.append(R"~~~(
  604. using @underlying@ = @type@;
  605. enum class @name@ : @underlying@ {)~~~");
  606. if (!default_.is_empty()) {
  607. generator.set("default", default_);
  608. generator.append(R"~~~(
  609. @default@,)~~~");
  610. }
  611. for (auto const& value : values) {
  612. generator.set("value", value);
  613. generator.append(R"~~~(
  614. @value@,)~~~");
  615. }
  616. for (auto const& alias : aliases) {
  617. generator.set("alias", alias.alias);
  618. generator.set("value", alias.name);
  619. generator.append(R"~~~(
  620. @alias@ = @value@,)~~~");
  621. }
  622. generator.append(R"~~~(
  623. };
  624. )~~~");
  625. };
  626. generator.append(R"~~~(
  627. #pragma once
  628. #include <AK/Types.h>
  629. #include <LibUnicode/Forward.h>
  630. namespace Unicode {
  631. )~~~");
  632. generate_enum("Locale"sv, "None"sv, unicode_data.locales);
  633. generate_enum("Condition"sv, "None"sv, move(unicode_data.conditions));
  634. generate_enum("CaseFoldingStatus"sv, {}, move(unicode_data.statuses));
  635. generate_enum("GeneralCategory"sv, {}, unicode_data.general_categories.keys(), unicode_data.general_category_aliases);
  636. generate_enum("Property"sv, {}, unicode_data.prop_list.keys(), unicode_data.prop_aliases);
  637. generate_enum("Script"sv, {}, unicode_data.script_list.keys(), unicode_data.script_aliases);
  638. generate_enum("GraphemeBreakProperty"sv, {}, unicode_data.grapheme_break_props.keys());
  639. generate_enum("WordBreakProperty"sv, {}, unicode_data.word_break_props.keys());
  640. generate_enum("SentenceBreakProperty"sv, {}, unicode_data.sentence_break_props.keys());
  641. generate_enum("CompatibilityFormattingTag"sv, "Canonical"sv, unicode_data.compatibility_tags);
  642. generator.append(R"~~~(
  643. struct SpecialCasing {
  644. u32 code_point { 0 };
  645. u32 lowercase_mapping[@special_casing_mapping_size@];
  646. u32 lowercase_mapping_size { 0 };
  647. u32 uppercase_mapping[@special_casing_mapping_size@];
  648. u32 uppercase_mapping_size { 0 };
  649. u32 titlecase_mapping[@special_casing_mapping_size@];
  650. u32 titlecase_mapping_size { 0 };
  651. Locale locale { Locale::None };
  652. Condition condition { Condition::None };
  653. };
  654. struct CaseFolding {
  655. u32 code_point { 0 };
  656. CaseFoldingStatus status { CaseFoldingStatus::Common };
  657. u32 mapping[@case_folding_mapping_size@];
  658. u32 mapping_size { 0 };
  659. };
  660. struct CodePointDecompositionRaw {
  661. u32 code_point { 0 };
  662. CompatibilityFormattingTag tag { CompatibilityFormattingTag::Canonical };
  663. size_t decomposition_index { 0 };
  664. size_t decomposition_count { 0 };
  665. };
  666. struct CodePointDecomposition {
  667. u32 code_point { 0 };
  668. CompatibilityFormattingTag tag { CompatibilityFormattingTag::Canonical };
  669. ReadonlySpan<u32> decomposition;
  670. };
  671. Optional<Locale> locale_from_string(StringView locale);
  672. ReadonlySpan<SpecialCasing const*> special_case_mapping(u32 code_point);
  673. ReadonlySpan<CaseFolding const*> case_folding_mapping(u32 code_point);
  674. }
  675. )~~~");
  676. TRY(file.write_until_depleted(generator.as_string_view().bytes()));
  677. return {};
  678. }
  679. static ErrorOr<void> generate_unicode_data_implementation(Core::InputBufferedFile& file, UnicodeData const& unicode_data)
  680. {
  681. StringBuilder builder;
  682. SourceGenerator generator { builder };
  683. generator.set("string_index_type"sv, unicode_data.unique_strings.type_that_fits());
  684. generator.set("largest_special_casing_size", DeprecatedString::number(unicode_data.largest_special_casing_size));
  685. generator.set("special_casing_size", DeprecatedString::number(unicode_data.special_casing.size()));
  686. generator.set("largest_case_folding_size", DeprecatedString::number(unicode_data.largest_case_folding_size));
  687. generator.set("case_folding_size", DeprecatedString::number(unicode_data.case_folding.size()));
  688. generator.append(R"~~~(
  689. #include <AK/Array.h>
  690. #include <AK/BinarySearch.h>
  691. #include <AK/CharacterTypes.h>
  692. #include <AK/Optional.h>
  693. #include <AK/Span.h>
  694. #include <AK/DeprecatedString.h>
  695. #include <AK/StringView.h>
  696. #include <LibUnicode/CharacterTypes.h>
  697. #include <LibUnicode/UnicodeData.h>
  698. #include <LibUnicode/Normalize.h>
  699. namespace Unicode {
  700. )~~~");
  701. unicode_data.unique_strings.generate(generator);
  702. auto append_list_and_size = [&](auto const& list, StringView format) {
  703. if (list.is_empty()) {
  704. generator.append(", {}, 0");
  705. return;
  706. }
  707. bool first = true;
  708. generator.append(", {");
  709. for (auto const& item : list) {
  710. generator.append(first ? " "sv : ", "sv);
  711. generator.append(DeprecatedString::formatted(format, item));
  712. first = false;
  713. }
  714. generator.append(DeprecatedString::formatted(" }}, {}", list.size()));
  715. };
  716. generator.append(R"~~~(
  717. static constexpr Array<SpecialCasing, @special_casing_size@> s_special_case { {)~~~");
  718. for (auto const& casing : unicode_data.special_casing) {
  719. generator.set("code_point", DeprecatedString::formatted("{:#x}", casing.code_point));
  720. generator.append(R"~~~(
  721. { @code_point@)~~~");
  722. constexpr auto format = "0x{:x}"sv;
  723. append_list_and_size(casing.lowercase_mapping, format);
  724. append_list_and_size(casing.uppercase_mapping, format);
  725. append_list_and_size(casing.titlecase_mapping, format);
  726. generator.set("locale", casing.locale.is_empty() ? "None" : casing.locale);
  727. generator.append(", Locale::@locale@");
  728. generator.set("condition", casing.condition.is_empty() ? "None" : casing.condition);
  729. generator.append(", Condition::@condition@");
  730. generator.append(" },");
  731. }
  732. generator.append(R"~~~(
  733. } };
  734. static constexpr Array<CaseFolding, @case_folding_size@> s_case_folding { {)~~~");
  735. for (auto const& folding : unicode_data.case_folding) {
  736. generator.set("code_point", DeprecatedString::formatted("{:#x}", folding.code_point));
  737. generator.set("status", folding.status);
  738. generator.append(R"~~~(
  739. { @code_point@, CaseFoldingStatus::@status@)~~~");
  740. append_list_and_size(folding.mapping, "0x{:x}"sv);
  741. generator.append(" },");
  742. }
  743. generator.append(R"~~~(
  744. } };
  745. struct CodePointMapping {
  746. u32 code_point { 0 };
  747. u32 mapping { 0 };
  748. };
  749. struct SpecialCaseMapping {
  750. u32 code_point { 0 };
  751. Array<SpecialCasing const*, @largest_special_casing_size@> special_casing {};
  752. u32 special_casing_size { 0 };
  753. };
  754. struct CaseFoldingMapping {
  755. u32 code_point { 0 };
  756. Array<CaseFolding const*, @largest_case_folding_size@> case_folding {};
  757. u32 case_folding_size { 0 };
  758. };
  759. struct CodePointAbbreviation {
  760. u32 code_point { 0 };
  761. @string_index_type@ abbreviation { 0 };
  762. };
  763. template<typename MappingType>
  764. struct CodePointComparator {
  765. constexpr int operator()(u32 code_point, MappingType const& mapping)
  766. {
  767. return code_point - mapping.code_point;
  768. }
  769. };
  770. struct BlockNameData {
  771. CodePointRange code_point_range {};
  772. @string_index_type@ display_name { 0 };
  773. };
  774. struct BlockNameComparator : public CodePointRangeComparator {
  775. constexpr int operator()(u32 code_point, BlockNameData const& name)
  776. {
  777. return CodePointRangeComparator::operator()(code_point, name.code_point_range);
  778. }
  779. };
  780. struct CodePointName {
  781. CodePointRange code_point_range {};
  782. @string_index_type@ display_name { 0 };
  783. };
  784. struct CodePointNameComparator : public CodePointRangeComparator {
  785. constexpr int operator()(u32 code_point, CodePointName const& name)
  786. {
  787. return CodePointRangeComparator::operator()(code_point, name.code_point_range);
  788. }
  789. };
  790. )~~~");
  791. generator.set("decomposition_mappings_size", DeprecatedString::number(unicode_data.decomposition_mappings.size()));
  792. generator.append("\nstatic constexpr Array<u32, @decomposition_mappings_size@> s_decomposition_mappings_data { ");
  793. generator.append(DeprecatedString::join(", "sv, unicode_data.decomposition_mappings, "{:#x}"sv));
  794. generator.append(" };\n");
  795. auto append_code_point_mappings = [&](StringView name, StringView mapping_type, u32 size, auto mapping_getter) {
  796. generator.set("name", name);
  797. generator.set("mapping_type", mapping_type);
  798. generator.set("size", DeprecatedString::number(size));
  799. generator.append(R"~~~(
  800. static constexpr Array<@mapping_type@, @size@> s_@name@_mappings { {
  801. )~~~");
  802. constexpr size_t max_mappings_per_row = 20;
  803. size_t mappings_in_current_row = 0;
  804. for (auto const& data : unicode_data.code_point_data) {
  805. auto mapping = mapping_getter(data);
  806. if constexpr (requires { mapping.has_value(); }) {
  807. if (!mapping.has_value())
  808. continue;
  809. } else {
  810. if (mapping.is_empty())
  811. continue;
  812. }
  813. if (mappings_in_current_row++ > 0)
  814. generator.append(" ");
  815. generator.set("code_point", DeprecatedString::formatted("{:#x}", data.code_point));
  816. generator.append("{ @code_point@");
  817. if constexpr (IsSame<decltype(mapping), Optional<u32>> || IsSame<decltype(mapping), Optional<size_t>>) {
  818. generator.set("mapping", DeprecatedString::formatted("{:#x}", *mapping));
  819. generator.append(", @mapping@ },");
  820. } else if constexpr (IsSame<decltype(mapping), Optional<CodePointDecomposition>>) {
  821. generator.set("tag", mapping->tag);
  822. generator.set("start", DeprecatedString::number(mapping->decomposition_index));
  823. generator.set("size", DeprecatedString::number(mapping->decomposition_size));
  824. generator.append(", CompatibilityFormattingTag::@tag@, @start@, @size@ },");
  825. } else {
  826. append_list_and_size(mapping, "&s_@name@[{}]"sv);
  827. generator.append(" },");
  828. }
  829. if (mappings_in_current_row == max_mappings_per_row) {
  830. mappings_in_current_row = 0;
  831. generator.append("\n ");
  832. }
  833. }
  834. generator.append(R"~~~(
  835. } };
  836. )~~~");
  837. };
  838. append_code_point_mappings("combining_class"sv, "CodePointMapping"sv, unicode_data.code_points_with_non_zero_combining_class,
  839. [](auto const& data) -> Optional<u32> {
  840. if (data.canonical_combining_class == 0)
  841. return {};
  842. return data.canonical_combining_class;
  843. });
  844. append_code_point_mappings("uppercase"sv, "CodePointMapping"sv, unicode_data.simple_uppercase_mapping_size, [](auto const& data) { return data.simple_uppercase_mapping; });
  845. append_code_point_mappings("lowercase"sv, "CodePointMapping"sv, unicode_data.simple_lowercase_mapping_size, [](auto const& data) { return data.simple_lowercase_mapping; });
  846. append_code_point_mappings("titlecase"sv, "CodePointMapping"sv, unicode_data.simple_titlecase_mapping_size, [](auto const& data) { return data.simple_titlecase_mapping; });
  847. append_code_point_mappings("special_case"sv, "SpecialCaseMapping"sv, unicode_data.code_points_with_special_casing, [](auto const& data) { return data.special_casing_indices; });
  848. append_code_point_mappings("case_folding"sv, "CaseFoldingMapping"sv, unicode_data.code_points_with_case_folding, [](auto const& data) { return data.case_folding_indices; });
  849. append_code_point_mappings("abbreviation"sv, "CodePointAbbreviation"sv, unicode_data.code_point_abbreviations.size(), [](auto const& data) { return data.abbreviation; });
  850. append_code_point_mappings("decomposition"sv, "CodePointDecompositionRaw"sv, unicode_data.code_points_with_decomposition_mapping,
  851. [](auto const& data) {
  852. return data.decomposition_mapping;
  853. });
  854. auto append_code_point_range_list = [&](DeprecatedString name, Vector<CodePointRange> const& ranges) {
  855. generator.set("name", name);
  856. generator.set("size", DeprecatedString::number(ranges.size()));
  857. generator.append(R"~~~(
  858. static constexpr Array<CodePointRange, @size@> @name@ { {
  859. )~~~");
  860. constexpr size_t max_ranges_per_row = 20;
  861. size_t ranges_in_current_row = 0;
  862. for (auto const& range : ranges) {
  863. if (ranges_in_current_row++ > 0)
  864. generator.append(" ");
  865. generator.set("first", DeprecatedString::formatted("{:#x}", range.first));
  866. generator.set("last", DeprecatedString::formatted("{:#x}", range.last));
  867. generator.append("{ @first@, @last@ },");
  868. if (ranges_in_current_row == max_ranges_per_row) {
  869. ranges_in_current_row = 0;
  870. generator.append("\n ");
  871. }
  872. }
  873. generator.append(R"~~~(
  874. } };
  875. )~~~");
  876. };
  877. auto append_prop_list = [&](StringView collection_name, StringView property_format, PropList const& property_list) {
  878. for (auto const& property : property_list) {
  879. auto name = DeprecatedString::formatted(property_format, property.key);
  880. append_code_point_range_list(move(name), property.value);
  881. }
  882. auto property_names = property_list.keys();
  883. quick_sort(property_names);
  884. generator.set("name", collection_name);
  885. generator.set("size", DeprecatedString::number(property_names.size()));
  886. generator.append(R"~~~(
  887. static constexpr Array<ReadonlySpan<CodePointRange>, @size@> @name@ { {)~~~");
  888. for (auto const& property_name : property_names) {
  889. generator.set("name", DeprecatedString::formatted(property_format, property_name));
  890. generator.append(R"~~~(
  891. @name@.span(),)~~~");
  892. }
  893. generator.append(R"~~~(
  894. } };
  895. )~~~");
  896. };
  897. append_prop_list("s_general_categories"sv, "s_general_category_{}"sv, unicode_data.general_categories);
  898. append_prop_list("s_properties"sv, "s_property_{}"sv, unicode_data.prop_list);
  899. append_prop_list("s_scripts"sv, "s_script_{}"sv, unicode_data.script_list);
  900. append_prop_list("s_script_extensions"sv, "s_script_extension_{}"sv, unicode_data.script_extensions);
  901. append_prop_list("s_grapheme_break_properties"sv, "s_grapheme_break_property_{}"sv, unicode_data.grapheme_break_props);
  902. append_prop_list("s_word_break_properties"sv, "s_word_break_property_{}"sv, unicode_data.word_break_props);
  903. append_prop_list("s_sentence_break_properties"sv, "s_sentence_break_property_{}"sv, unicode_data.sentence_break_props);
  904. auto append_code_point_display_names = [&](StringView type, StringView name, auto const& display_names) {
  905. constexpr size_t max_values_per_row = 30;
  906. size_t values_in_current_row = 0;
  907. generator.set("type", type);
  908. generator.set("name", name);
  909. generator.set("size", DeprecatedString::number(display_names.size()));
  910. generator.append(R"~~~(
  911. static constexpr Array<@type@, @size@> @name@ { {
  912. )~~~");
  913. for (auto const& display_name : display_names) {
  914. if (values_in_current_row++ > 0)
  915. generator.append(", ");
  916. generator.set("first", DeprecatedString::formatted("{:#x}", display_name.code_point_range.first));
  917. generator.set("last", DeprecatedString::formatted("{:#x}", display_name.code_point_range.last));
  918. generator.set("name", DeprecatedString::number(display_name.name));
  919. generator.append("{ { @first@, @last@ }, @name@ }");
  920. if (values_in_current_row == max_values_per_row) {
  921. values_in_current_row = 0;
  922. generator.append(",\n ");
  923. }
  924. }
  925. generator.append(R"~~~(
  926. } };
  927. )~~~");
  928. };
  929. append_code_point_display_names("BlockNameData"sv, "s_block_display_names"sv, unicode_data.block_display_names);
  930. append_code_point_display_names("CodePointName"sv, "s_code_point_display_names"sv, unicode_data.code_point_display_names);
  931. generator.append(R"~~~(
  932. Optional<StringView> code_point_block_display_name(u32 code_point)
  933. {
  934. if (auto const* entry = binary_search(s_block_display_names, code_point, nullptr, BlockNameComparator {}))
  935. return decode_string(entry->display_name);
  936. return {};
  937. }
  938. ReadonlySpan<BlockName> block_display_names()
  939. {
  940. static auto display_names = []() {
  941. Array<BlockName, s_block_display_names.size()> display_names;
  942. for (size_t i = 0; i < s_block_display_names.size(); ++i) {
  943. auto const& display_name = s_block_display_names[i];
  944. display_names[i] = { display_name.code_point_range, decode_string(display_name.display_name) };
  945. }
  946. return display_names;
  947. }();
  948. return display_names.span();
  949. }
  950. Optional<DeprecatedString> code_point_display_name(u32 code_point)
  951. {
  952. if (auto const* entry = binary_search(s_code_point_display_names, code_point, nullptr, CodePointNameComparator {})) {
  953. auto display_name = decode_string(entry->display_name);
  954. if (display_name.ends_with("{:X}"sv))
  955. return DeprecatedString::formatted(display_name, code_point);
  956. return display_name;
  957. }
  958. return {};
  959. }
  960. )~~~");
  961. auto append_code_point_mapping_search = [&](StringView method, StringView mappings, StringView fallback) {
  962. generator.set("method", method);
  963. generator.set("mappings", mappings);
  964. generator.set("fallback", fallback);
  965. generator.append(R"~~~(
  966. u32 @method@(u32 code_point)
  967. {
  968. auto const* mapping = binary_search(@mappings@, code_point, nullptr, CodePointComparator<CodePointMapping> {});
  969. return mapping ? mapping->mapping : @fallback@;
  970. }
  971. )~~~");
  972. };
  973. append_code_point_mapping_search("canonical_combining_class"sv, "s_combining_class_mappings"sv, "0"sv);
  974. append_code_point_mapping_search("to_unicode_uppercase"sv, "s_uppercase_mappings"sv, "code_point"sv);
  975. append_code_point_mapping_search("to_unicode_lowercase"sv, "s_lowercase_mappings"sv, "code_point"sv);
  976. append_code_point_mapping_search("to_unicode_titlecase"sv, "s_titlecase_mappings"sv, "code_point"sv);
  977. generator.append(R"~~~(
  978. ReadonlySpan<SpecialCasing const*> special_case_mapping(u32 code_point)
  979. {
  980. auto const* mapping = binary_search(s_special_case_mappings, code_point, nullptr, CodePointComparator<SpecialCaseMapping> {});
  981. if (mapping == nullptr)
  982. return {};
  983. return mapping->special_casing.span().slice(0, mapping->special_casing_size);
  984. }
  985. ReadonlySpan<CaseFolding const*> case_folding_mapping(u32 code_point)
  986. {
  987. auto const* mapping = binary_search(s_case_folding_mappings, code_point, nullptr, CodePointComparator<CaseFoldingMapping> {});
  988. if (mapping == nullptr)
  989. return {};
  990. return mapping->case_folding.span().slice(0, mapping->case_folding_size);
  991. }
  992. Optional<StringView> code_point_abbreviation(u32 code_point)
  993. {
  994. auto const* mapping = binary_search(s_abbreviation_mappings, code_point, nullptr, CodePointComparator<CodePointAbbreviation> {});
  995. if (mapping == nullptr)
  996. return {};
  997. if (mapping->abbreviation == 0)
  998. return {};
  999. return decode_string(mapping->abbreviation);
  1000. }
  1001. Optional<CodePointDecomposition const> code_point_decomposition(u32 code_point)
  1002. {
  1003. auto const* mapping = binary_search(s_decomposition_mappings, code_point, nullptr, CodePointComparator<CodePointDecompositionRaw> {});
  1004. if (mapping == nullptr)
  1005. return {};
  1006. return CodePointDecomposition { mapping->code_point, mapping->tag, ReadonlySpan<u32> { s_decomposition_mappings_data.data() + mapping->decomposition_index, mapping->decomposition_count } };
  1007. }
  1008. Optional<CodePointDecomposition const> code_point_decomposition_by_index(size_t index)
  1009. {
  1010. if (index >= s_decomposition_mappings.size())
  1011. return {};
  1012. auto const& mapping = s_decomposition_mappings[index];
  1013. return CodePointDecomposition { mapping.code_point, mapping.tag, ReadonlySpan<u32> { s_decomposition_mappings_data.data() + mapping.decomposition_index, mapping.decomposition_count } };
  1014. }
  1015. )~~~");
  1016. auto append_prop_search = [&](StringView enum_title, StringView enum_snake, StringView collection_name) {
  1017. generator.set("enum_title", enum_title);
  1018. generator.set("enum_snake", enum_snake);
  1019. generator.set("collection_name", collection_name);
  1020. generator.append(R"~~~(
  1021. bool code_point_has_@enum_snake@(u32 code_point, @enum_title@ @enum_snake@)
  1022. {
  1023. auto index = static_cast<@enum_title@UnderlyingType>(@enum_snake@);
  1024. auto const& ranges = @collection_name@.at(index);
  1025. auto const* range = binary_search(ranges, code_point, nullptr, CodePointRangeComparator {});
  1026. return range != nullptr;
  1027. }
  1028. )~~~");
  1029. };
  1030. auto append_from_string = [&](StringView enum_title, StringView enum_snake, auto const& prop_list, Vector<Alias> const& aliases) -> ErrorOr<void> {
  1031. HashValueMap<StringView> hashes;
  1032. TRY(hashes.try_ensure_capacity(prop_list.size() + aliases.size()));
  1033. ValueFromStringOptions options {};
  1034. for (auto const& prop : prop_list) {
  1035. if constexpr (IsSame<RemoveCVReference<decltype(prop)>, DeprecatedString>) {
  1036. hashes.set(CaseInsensitiveASCIIStringViewTraits::hash(prop), prop);
  1037. options.sensitivity = CaseSensitivity::CaseInsensitive;
  1038. } else {
  1039. hashes.set(prop.key.hash(), prop.key);
  1040. }
  1041. }
  1042. for (auto const& alias : aliases)
  1043. hashes.set(alias.alias.hash(), alias.alias);
  1044. generate_value_from_string(generator, "{}_from_string"sv, enum_title, enum_snake, move(hashes), options);
  1045. return {};
  1046. };
  1047. TRY(append_from_string("Locale"sv, "locale"sv, unicode_data.locales, {}));
  1048. append_prop_search("GeneralCategory"sv, "general_category"sv, "s_general_categories"sv);
  1049. TRY(append_from_string("GeneralCategory"sv, "general_category"sv, unicode_data.general_categories, unicode_data.general_category_aliases));
  1050. append_prop_search("Property"sv, "property"sv, "s_properties"sv);
  1051. TRY(append_from_string("Property"sv, "property"sv, unicode_data.prop_list, unicode_data.prop_aliases));
  1052. append_prop_search("Script"sv, "script"sv, "s_scripts"sv);
  1053. append_prop_search("Script"sv, "script_extension"sv, "s_script_extensions"sv);
  1054. TRY(append_from_string("Script"sv, "script"sv, unicode_data.script_list, unicode_data.script_aliases));
  1055. append_prop_search("GraphemeBreakProperty"sv, "grapheme_break_property"sv, "s_grapheme_break_properties"sv);
  1056. append_prop_search("WordBreakProperty"sv, "word_break_property"sv, "s_word_break_properties"sv);
  1057. append_prop_search("SentenceBreakProperty"sv, "sentence_break_property"sv, "s_sentence_break_properties"sv);
  1058. generator.append(R"~~~(
  1059. }
  1060. )~~~");
  1061. TRY(file.write_until_depleted(generator.as_string_view().bytes()));
  1062. return {};
  1063. }
  1064. static Vector<u32> flatten_code_point_ranges(Vector<CodePointRange> const& code_points)
  1065. {
  1066. Vector<u32> flattened;
  1067. for (auto const& range : code_points) {
  1068. flattened.grow_capacity(range.last - range.first);
  1069. for (u32 code_point = range.first; code_point <= range.last; ++code_point)
  1070. flattened.append(code_point);
  1071. }
  1072. return flattened;
  1073. }
  1074. static Vector<CodePointRange> form_code_point_ranges(Vector<u32> code_points)
  1075. {
  1076. Vector<CodePointRange> ranges;
  1077. u32 range_start = code_points[0];
  1078. u32 range_end = range_start;
  1079. for (size_t i = 1; i < code_points.size(); ++i) {
  1080. u32 code_point = code_points[i];
  1081. if ((code_point - range_end) == 1) {
  1082. range_end = code_point;
  1083. } else {
  1084. ranges.append({ range_start, range_end });
  1085. range_start = code_point;
  1086. range_end = code_point;
  1087. }
  1088. }
  1089. ranges.append({ range_start, range_end });
  1090. return ranges;
  1091. }
  1092. static void sort_and_merge_code_point_ranges(Vector<CodePointRange>& code_points)
  1093. {
  1094. quick_sort(code_points, [](auto const& range1, auto const& range2) {
  1095. return range1.first < range2.first;
  1096. });
  1097. for (size_t i = 0; i < code_points.size() - 1;) {
  1098. if (code_points[i].last >= code_points[i + 1].first) {
  1099. code_points[i].last = max(code_points[i].last, code_points[i + 1].last);
  1100. code_points.remove(i + 1);
  1101. } else {
  1102. ++i;
  1103. }
  1104. }
  1105. auto all_code_points = flatten_code_point_ranges(code_points);
  1106. code_points = form_code_point_ranges(all_code_points);
  1107. }
  1108. static void populate_general_category_unions(PropList& general_categories)
  1109. {
  1110. // The Unicode standard defines General Category values which are not in any UCD file. These
  1111. // values are simply unions of other values.
  1112. // https://www.unicode.org/reports/tr44/#GC_Values_Table
  1113. auto populate_union = [&](auto alias, auto categories) {
  1114. auto& code_points = general_categories.ensure(alias);
  1115. for (auto const& category : categories)
  1116. code_points.extend(general_categories.find(category)->value);
  1117. sort_and_merge_code_point_ranges(code_points);
  1118. };
  1119. populate_union("LC"sv, Array { "Ll"sv, "Lu"sv, "Lt"sv });
  1120. populate_union("L"sv, Array { "Lu"sv, "Ll"sv, "Lt"sv, "Lm"sv, "Lo"sv });
  1121. populate_union("M"sv, Array { "Mn"sv, "Mc"sv, "Me"sv });
  1122. populate_union("N"sv, Array { "Nd"sv, "Nl"sv, "No"sv });
  1123. populate_union("P"sv, Array { "Pc"sv, "Pd"sv, "Ps"sv, "Pe"sv, "Pi"sv, "Pf"sv, "Po"sv });
  1124. populate_union("S"sv, Array { "Sm"sv, "Sc"sv, "Sk"sv, "So"sv });
  1125. populate_union("Z"sv, Array { "Zs"sv, "Zl"sv, "Zp"sv });
  1126. populate_union("C"sv, Array { "Cc"sv, "Cf"sv, "Cs"sv, "Co"sv, "Cn"sv });
  1127. }
  1128. static ErrorOr<void> normalize_script_extensions(PropList& script_extensions, PropList const& script_list, Vector<Alias> const& script_aliases)
  1129. {
  1130. // The ScriptExtensions UCD file lays out its code point ranges rather uniquely compared to
  1131. // other files. The Script listed on each line may either be a full Script string or an aliased
  1132. // abbreviation. Further, the extensions may or may not include the base Script list. Normalize
  1133. // the extensions here to be keyed by the full Script name and always include the base list.
  1134. auto extensions = move(script_extensions);
  1135. script_extensions = TRY(script_list.clone());
  1136. for (auto const& extension : extensions) {
  1137. auto it = find_if(script_aliases.begin(), script_aliases.end(), [&](auto const& alias) { return extension.key == alias.alias; });
  1138. auto const& key = (it == script_aliases.end()) ? extension.key : it->name;
  1139. auto& code_points = script_extensions.find(key)->value;
  1140. code_points.extend(extension.value);
  1141. sort_and_merge_code_point_ranges(code_points);
  1142. }
  1143. // Lastly, the Common and Inherited script extensions are special. They must not contain any
  1144. // code points which appear in other script extensions. The ScriptExtensions UCD file does not
  1145. // list these extensions, therefore this peculiarity must be handled programmatically.
  1146. // https://www.unicode.org/reports/tr24/#Assignment_ScriptX_Values
  1147. auto code_point_has_other_extension = [&](StringView key, u32 code_point) {
  1148. for (auto const& extension : extensions) {
  1149. if (extension.key == key)
  1150. continue;
  1151. if (any_of(extension.value, [&](auto const& r) { return (r.first <= code_point) && (code_point <= r.last); }))
  1152. return true;
  1153. }
  1154. return false;
  1155. };
  1156. auto get_code_points_without_other_extensions = [&](StringView key) {
  1157. auto code_points = flatten_code_point_ranges(script_list.find(key)->value);
  1158. code_points.remove_all_matching([&](u32 c) { return code_point_has_other_extension(key, c); });
  1159. return code_points;
  1160. };
  1161. auto common_code_points = get_code_points_without_other_extensions("Common"sv);
  1162. script_extensions.set("Common"sv, form_code_point_ranges(common_code_points));
  1163. auto inherited_code_points = get_code_points_without_other_extensions("Inherited"sv);
  1164. script_extensions.set("Inherited"sv, form_code_point_ranges(inherited_code_points));
  1165. return {};
  1166. }
  1167. ErrorOr<int> serenity_main(Main::Arguments arguments)
  1168. {
  1169. StringView generated_header_path;
  1170. StringView generated_implementation_path;
  1171. StringView unicode_data_path;
  1172. StringView special_casing_path;
  1173. StringView case_folding_path;
  1174. StringView derived_general_category_path;
  1175. StringView prop_list_path;
  1176. StringView derived_core_prop_path;
  1177. StringView derived_binary_prop_path;
  1178. StringView prop_alias_path;
  1179. StringView prop_value_alias_path;
  1180. StringView name_alias_path;
  1181. StringView scripts_path;
  1182. StringView script_extensions_path;
  1183. StringView blocks_path;
  1184. StringView emoji_data_path;
  1185. StringView normalization_path;
  1186. StringView grapheme_break_path;
  1187. StringView word_break_path;
  1188. StringView sentence_break_path;
  1189. Core::ArgsParser args_parser;
  1190. args_parser.add_option(generated_header_path, "Path to the Unicode Data header file to generate", "generated-header-path", 'h', "generated-header-path");
  1191. args_parser.add_option(generated_implementation_path, "Path to the Unicode Data implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
  1192. args_parser.add_option(unicode_data_path, "Path to UnicodeData.txt file", "unicode-data-path", 'u', "unicode-data-path");
  1193. args_parser.add_option(special_casing_path, "Path to SpecialCasing.txt file", "special-casing-path", 's', "special-casing-path");
  1194. args_parser.add_option(case_folding_path, "Path to CaseFolding.txt file", "case-folding-path", 'o', "case-folding-path");
  1195. args_parser.add_option(derived_general_category_path, "Path to DerivedGeneralCategory.txt file", "derived-general-category-path", 'g', "derived-general-category-path");
  1196. args_parser.add_option(prop_list_path, "Path to PropList.txt file", "prop-list-path", 'p', "prop-list-path");
  1197. args_parser.add_option(derived_core_prop_path, "Path to DerivedCoreProperties.txt file", "derived-core-prop-path", 'd', "derived-core-prop-path");
  1198. args_parser.add_option(derived_binary_prop_path, "Path to DerivedBinaryProperties.txt file", "derived-binary-prop-path", 'b', "derived-binary-prop-path");
  1199. args_parser.add_option(prop_alias_path, "Path to PropertyAliases.txt file", "prop-alias-path", 'a', "prop-alias-path");
  1200. args_parser.add_option(prop_value_alias_path, "Path to PropertyValueAliases.txt file", "prop-value-alias-path", 'v', "prop-value-alias-path");
  1201. args_parser.add_option(name_alias_path, "Path to NameAliases.txt file", "name-alias-path", 'm', "name-alias-path");
  1202. args_parser.add_option(scripts_path, "Path to Scripts.txt file", "scripts-path", 'r', "scripts-path");
  1203. args_parser.add_option(script_extensions_path, "Path to ScriptExtensions.txt file", "script-extensions-path", 'x', "script-extensions-path");
  1204. args_parser.add_option(blocks_path, "Path to Blocks.txt file", "blocks-path", 'k', "blocks-path");
  1205. args_parser.add_option(emoji_data_path, "Path to emoji-data.txt file", "emoji-data-path", 'e', "emoji-data-path");
  1206. args_parser.add_option(normalization_path, "Path to DerivedNormalizationProps.txt file", "normalization-path", 'n', "normalization-path");
  1207. args_parser.add_option(grapheme_break_path, "Path to GraphemeBreakProperty.txt file", "grapheme-break-path", 'f', "grapheme-break-path");
  1208. args_parser.add_option(word_break_path, "Path to WordBreakProperty.txt file", "word-break-path", 'w', "word-break-path");
  1209. args_parser.add_option(sentence_break_path, "Path to SentenceBreakProperty.txt file", "sentence-break-path", 'i', "sentence-break-path");
  1210. args_parser.parse(arguments);
  1211. auto generated_header_file = TRY(open_file(generated_header_path, Core::File::OpenMode::Write));
  1212. auto generated_implementation_file = TRY(open_file(generated_implementation_path, Core::File::OpenMode::Write));
  1213. auto unicode_data_file = TRY(open_file(unicode_data_path, Core::File::OpenMode::Read));
  1214. auto derived_general_category_file = TRY(open_file(derived_general_category_path, Core::File::OpenMode::Read));
  1215. auto special_casing_file = TRY(open_file(special_casing_path, Core::File::OpenMode::Read));
  1216. auto case_folding_file = TRY(open_file(case_folding_path, Core::File::OpenMode::Read));
  1217. auto prop_list_file = TRY(open_file(prop_list_path, Core::File::OpenMode::Read));
  1218. auto derived_core_prop_file = TRY(open_file(derived_core_prop_path, Core::File::OpenMode::Read));
  1219. auto derived_binary_prop_file = TRY(open_file(derived_binary_prop_path, Core::File::OpenMode::Read));
  1220. auto prop_alias_file = TRY(open_file(prop_alias_path, Core::File::OpenMode::Read));
  1221. auto prop_value_alias_file = TRY(open_file(prop_value_alias_path, Core::File::OpenMode::Read));
  1222. auto name_alias_file = TRY(open_file(name_alias_path, Core::File::OpenMode::Read));
  1223. auto scripts_file = TRY(open_file(scripts_path, Core::File::OpenMode::Read));
  1224. auto script_extensions_file = TRY(open_file(script_extensions_path, Core::File::OpenMode::Read));
  1225. auto blocks_file = TRY(open_file(blocks_path, Core::File::OpenMode::Read));
  1226. auto emoji_data_file = TRY(open_file(emoji_data_path, Core::File::OpenMode::Read));
  1227. auto normalization_file = TRY(open_file(normalization_path, Core::File::OpenMode::Read));
  1228. auto grapheme_break_file = TRY(open_file(grapheme_break_path, Core::File::OpenMode::Read));
  1229. auto word_break_file = TRY(open_file(word_break_path, Core::File::OpenMode::Read));
  1230. auto sentence_break_file = TRY(open_file(sentence_break_path, Core::File::OpenMode::Read));
  1231. UnicodeData unicode_data {};
  1232. TRY(parse_special_casing(*special_casing_file, unicode_data));
  1233. TRY(parse_case_folding(*case_folding_file, unicode_data));
  1234. TRY(parse_prop_list(*derived_general_category_file, unicode_data.general_categories));
  1235. TRY(parse_prop_list(*prop_list_file, unicode_data.prop_list));
  1236. TRY(parse_prop_list(*derived_core_prop_file, unicode_data.prop_list));
  1237. TRY(parse_prop_list(*derived_binary_prop_file, unicode_data.prop_list));
  1238. TRY(parse_prop_list(*emoji_data_file, unicode_data.prop_list));
  1239. TRY(parse_normalization_props(*normalization_file, unicode_data));
  1240. TRY(parse_alias_list(*prop_alias_file, unicode_data.prop_list, unicode_data.prop_aliases));
  1241. TRY(parse_prop_list(*scripts_file, unicode_data.script_list));
  1242. TRY(parse_prop_list(*script_extensions_file, unicode_data.script_extensions, true));
  1243. TRY(parse_block_display_names(*blocks_file, unicode_data));
  1244. TRY(parse_name_aliases(*name_alias_file, unicode_data));
  1245. TRY(parse_prop_list(*grapheme_break_file, unicode_data.grapheme_break_props));
  1246. TRY(parse_prop_list(*word_break_file, unicode_data.word_break_props));
  1247. TRY(parse_prop_list(*sentence_break_file, unicode_data.sentence_break_props));
  1248. populate_general_category_unions(unicode_data.general_categories);
  1249. TRY(parse_unicode_data(*unicode_data_file, unicode_data));
  1250. TRY(parse_value_alias_list(*prop_value_alias_file, "gc"sv, unicode_data.general_categories.keys(), unicode_data.general_category_aliases));
  1251. TRY(parse_value_alias_list(*prop_value_alias_file, "sc"sv, unicode_data.script_list.keys(), unicode_data.script_aliases, false));
  1252. TRY(normalize_script_extensions(unicode_data.script_extensions, unicode_data.script_list, unicode_data.script_aliases));
  1253. TRY(generate_unicode_data_header(*generated_header_file, unicode_data));
  1254. TRY(generate_unicode_data_implementation(*generated_implementation_file, unicode_data));
  1255. return 0;
  1256. }