GenerateUnicodeData.cpp 58 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452
  1. /*
  2. * Copyright (c) 2021, Tim Flynn <trflynn89@serenityos.org>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include "GeneratorUtil.h"
  7. #include <AK/AllOf.h>
  8. #include <AK/Array.h>
  9. #include <AK/CharacterTypes.h>
  10. #include <AK/Find.h>
  11. #include <AK/HashMap.h>
  12. #include <AK/Optional.h>
  13. #include <AK/QuickSort.h>
  14. #include <AK/SourceGenerator.h>
  15. #include <AK/String.h>
  16. #include <AK/StringUtils.h>
  17. #include <AK/Types.h>
  18. #include <AK/Vector.h>
  19. #include <LibCore/ArgsParser.h>
  20. #include <LibCore/Stream.h>
  21. using StringIndexType = u16;
  22. constexpr auto s_string_index_type = "u16"sv;
  23. // Some code points are excluded from UnicodeData.txt, and instead are part of a "range" of code
  24. // points, as indicated by the "name" field. For example:
  25. // 3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
  26. // 4DBF;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
  27. struct CodePointRange {
  28. u32 first;
  29. u32 last;
  30. };
  31. // SpecialCasing source: https://www.unicode.org/Public/13.0.0/ucd/SpecialCasing.txt
  32. // Field descriptions: https://www.unicode.org/reports/tr44/tr44-13.html#SpecialCasing.txt
  33. struct SpecialCasing {
  34. u32 index { 0 };
  35. u32 code_point { 0 };
  36. Vector<u32> lowercase_mapping;
  37. Vector<u32> uppercase_mapping;
  38. Vector<u32> titlecase_mapping;
  39. String locale;
  40. String condition;
  41. };
  42. // Field descriptions: https://www.unicode.org/reports/tr44/#Character_Decomposition_Mappings
  43. struct CodePointDecomposition {
  44. // `tag` is a string since it's used for codegen as an enum value.
  45. String tag { "Canonical"sv };
  46. size_t decomposition_index { 0 };
  47. size_t decomposition_size { 0 };
  48. };
  49. // PropList source: https://www.unicode.org/Public/13.0.0/ucd/PropList.txt
  50. // Property descriptions: https://www.unicode.org/reports/tr44/tr44-13.html#PropList.txt
  51. using PropList = HashMap<String, Vector<CodePointRange>>;
  52. // Normalization source: https://www.unicode.org/Public/13.0.0/ucd/DerivedNormalizationProps.txt
  53. // Normalization descriptions: https://www.unicode.org/reports/tr44/#DerivedNormalizationProps.txt
  54. enum class QuickCheck {
  55. Yes,
  56. No,
  57. Maybe,
  58. };
  59. struct Normalization {
  60. CodePointRange code_point_range;
  61. Vector<u32> value;
  62. QuickCheck quick_check { QuickCheck::Yes };
  63. };
  64. using NormalizationProps = HashMap<String, Vector<Normalization>>;
  65. struct CodePointName {
  66. CodePointRange code_point_range;
  67. StringIndexType name { 0 };
  68. };
  69. // UnicodeData source: https://www.unicode.org/Public/13.0.0/ucd/UnicodeData.txt
  70. // Field descriptions: https://www.unicode.org/reports/tr44/tr44-13.html#UnicodeData.txt
  71. // https://www.unicode.org/reports/tr44/#General_Category_Values
  72. struct CodePointData {
  73. u32 code_point { 0 };
  74. String name;
  75. Optional<StringIndexType> abbreviation;
  76. u8 canonical_combining_class { 0 };
  77. String bidi_class;
  78. Optional<CodePointDecomposition> decomposition_mapping;
  79. Optional<i8> numeric_value_decimal;
  80. Optional<i8> numeric_value_digit;
  81. Optional<i8> numeric_value_numeric;
  82. bool bidi_mirrored { false };
  83. String unicode_1_name;
  84. String iso_comment;
  85. Optional<u32> simple_uppercase_mapping;
  86. Optional<u32> simple_lowercase_mapping;
  87. Optional<u32> simple_titlecase_mapping;
  88. Vector<u32> special_casing_indices;
  89. };
  90. struct BlockName {
  91. CodePointRange code_point_range;
  92. StringIndexType name { 0 };
  93. };
  94. struct UnicodeData {
  95. UniqueStringStorage<StringIndexType> unique_strings;
  96. u32 code_points_with_non_zero_combining_class { 0 };
  97. u32 code_points_with_decomposition_mapping { 0 };
  98. Vector<u32> decomposition_mappings;
  99. Vector<String> compatibility_tags;
  100. u32 simple_uppercase_mapping_size { 0 };
  101. u32 simple_lowercase_mapping_size { 0 };
  102. Vector<SpecialCasing> special_casing;
  103. u32 code_points_with_special_casing { 0 };
  104. u32 largest_casing_transform_size { 0 };
  105. u32 largest_special_casing_size { 0 };
  106. Vector<String> conditions;
  107. Vector<String> locales;
  108. Vector<CodePointData> code_point_data;
  109. HashMap<u32, StringIndexType> code_point_abbreviations;
  110. HashMap<u32, StringIndexType> code_point_display_name_aliases;
  111. Vector<CodePointName> code_point_display_names;
  112. PropList general_categories;
  113. Vector<Alias> general_category_aliases;
  114. // The Unicode standard defines additional properties (Any, Assigned, ASCII) which are not in
  115. // any UCD file. Assigned code point ranges are derived as this generator is executed.
  116. // https://unicode.org/reports/tr18/#General_Category_Property
  117. PropList prop_list {
  118. { "Any"sv, { { 0, 0x10ffff } } },
  119. { "Assigned"sv, {} },
  120. { "ASCII"sv, { { 0, 0x7f } } },
  121. };
  122. Vector<Alias> prop_aliases;
  123. PropList script_list {
  124. { "Unknown"sv, {} },
  125. };
  126. Vector<Alias> script_aliases;
  127. PropList script_extensions;
  128. PropList block_list {
  129. { "No_Block"sv, {} },
  130. };
  131. Vector<Alias> block_aliases;
  132. Vector<BlockName> block_display_names;
  133. // FIXME: We are not yet doing anything with this data. It will be needed for String.prototype.normalize.
  134. NormalizationProps normalization_props;
  135. PropList grapheme_break_props;
  136. PropList word_break_props;
  137. PropList sentence_break_props;
  138. };
  139. static String sanitize_entry(String const& entry)
  140. {
  141. auto sanitized = entry.replace("-"sv, "_"sv, ReplaceMode::All);
  142. sanitized = sanitized.replace(" "sv, "_"sv, ReplaceMode::All);
  143. StringBuilder builder;
  144. bool next_is_upper = true;
  145. for (auto ch : sanitized) {
  146. if (next_is_upper)
  147. builder.append_code_point(to_ascii_uppercase(ch));
  148. else
  149. builder.append_code_point(ch);
  150. next_is_upper = ch == '_';
  151. }
  152. return builder.to_string();
  153. }
  154. static Vector<u32> parse_code_point_list(StringView list)
  155. {
  156. Vector<u32> code_points;
  157. auto segments = list.split_view(' ');
  158. for (auto const& code_point : segments)
  159. code_points.append(AK::StringUtils::convert_to_uint_from_hex<u32>(code_point).value());
  160. return code_points;
  161. }
  162. static CodePointRange parse_code_point_range(StringView list)
  163. {
  164. CodePointRange code_point_range {};
  165. if (list.contains(".."sv)) {
  166. auto segments = list.split_view(".."sv);
  167. VERIFY(segments.size() == 2);
  168. auto begin = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[0]).value();
  169. auto end = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[1]).value();
  170. code_point_range = { begin, end };
  171. } else {
  172. auto code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(list).value();
  173. code_point_range = { code_point, code_point };
  174. }
  175. return code_point_range;
  176. }
  177. static ErrorOr<void> parse_special_casing(Core::Stream::BufferedFile& file, UnicodeData& unicode_data)
  178. {
  179. Array<u8, 1024> buffer;
  180. while (TRY(file.can_read_line())) {
  181. auto line = TRY(file.read_line(buffer));
  182. if (line.is_empty() || line.starts_with('#'))
  183. continue;
  184. if (auto index = line.find('#'); index.has_value())
  185. line = line.substring_view(0, *index);
  186. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  187. VERIFY(segments.size() == 5 || segments.size() == 6);
  188. SpecialCasing casing {};
  189. casing.code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[0]).value();
  190. casing.lowercase_mapping = parse_code_point_list(segments[1]);
  191. casing.titlecase_mapping = parse_code_point_list(segments[2]);
  192. casing.uppercase_mapping = parse_code_point_list(segments[3]);
  193. if (auto condition = segments[4].trim_whitespace(); !condition.is_empty()) {
  194. auto conditions = condition.split_view(' ', SplitBehavior::KeepEmpty);
  195. VERIFY(conditions.size() == 1 || conditions.size() == 2);
  196. if (conditions.size() == 2) {
  197. casing.locale = conditions[0];
  198. casing.condition = conditions[1];
  199. } else if (all_of(conditions[0], is_ascii_lower_alpha)) {
  200. casing.locale = conditions[0];
  201. } else {
  202. casing.condition = conditions[0];
  203. }
  204. if (!casing.locale.is_empty()) {
  205. casing.locale = String::formatted("{:c}{}", to_ascii_uppercase(casing.locale[0]), casing.locale.substring_view(1));
  206. if (!unicode_data.locales.contains_slow(casing.locale))
  207. unicode_data.locales.append(casing.locale);
  208. }
  209. casing.condition = casing.condition.replace("_"sv, ""sv, ReplaceMode::All);
  210. if (!casing.condition.is_empty() && !unicode_data.conditions.contains_slow(casing.condition))
  211. unicode_data.conditions.append(casing.condition);
  212. }
  213. unicode_data.largest_casing_transform_size = max(unicode_data.largest_casing_transform_size, casing.lowercase_mapping.size());
  214. unicode_data.largest_casing_transform_size = max(unicode_data.largest_casing_transform_size, casing.titlecase_mapping.size());
  215. unicode_data.largest_casing_transform_size = max(unicode_data.largest_casing_transform_size, casing.uppercase_mapping.size());
  216. unicode_data.special_casing.append(move(casing));
  217. }
  218. quick_sort(unicode_data.special_casing, [](auto const& lhs, auto const& rhs) {
  219. if (lhs.code_point != rhs.code_point)
  220. return lhs.code_point < rhs.code_point;
  221. if (lhs.locale.is_empty() && !rhs.locale.is_empty())
  222. return false;
  223. if (!lhs.locale.is_empty() && rhs.locale.is_empty())
  224. return true;
  225. return lhs.locale < rhs.locale;
  226. });
  227. for (u32 i = 0; i < unicode_data.special_casing.size(); ++i)
  228. unicode_data.special_casing[i].index = i;
  229. return {};
  230. }
  231. static ErrorOr<void> parse_prop_list(Core::Stream::BufferedFile& file, PropList& prop_list, bool multi_value_property = false, bool sanitize_property = false)
  232. {
  233. Array<u8, 1024> buffer;
  234. while (TRY(file.can_read_line())) {
  235. auto line = TRY(file.read_line(buffer));
  236. if (line.is_empty() || line.starts_with('#'))
  237. continue;
  238. if (auto index = line.find('#'); index.has_value())
  239. line = line.substring_view(0, *index);
  240. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  241. VERIFY(segments.size() == 2);
  242. auto code_point_range = parse_code_point_range(segments[0].trim_whitespace());
  243. Vector<StringView> properties;
  244. if (multi_value_property)
  245. properties = segments[1].trim_whitespace().split_view(' ');
  246. else
  247. properties = { segments[1].trim_whitespace() };
  248. for (auto& property : properties) {
  249. auto& code_points = prop_list.ensure(sanitize_property ? sanitize_entry(property).trim_whitespace().view() : property.trim_whitespace());
  250. code_points.append(code_point_range);
  251. }
  252. }
  253. return {};
  254. }
  255. static ErrorOr<void> parse_alias_list(Core::Stream::BufferedFile& file, PropList const& prop_list, Vector<Alias>& prop_aliases)
  256. {
  257. String current_property;
  258. Array<u8, 1024> buffer;
  259. auto append_alias = [&](auto alias, auto property) {
  260. // Note: The alias files contain lines such as "Hyphen = Hyphen", which we should just skip.
  261. if (alias == property)
  262. return;
  263. // FIXME: We will, eventually, need to find where missing properties are located and parse them.
  264. if (!prop_list.contains(property))
  265. return;
  266. prop_aliases.append({ property, alias });
  267. };
  268. while (TRY(file.can_read_line())) {
  269. auto line = TRY(file.read_line(buffer));
  270. if (line.is_empty() || line.starts_with('#')) {
  271. if (line.ends_with("Properties"sv))
  272. current_property = line.substring_view(2);
  273. continue;
  274. }
  275. // Note: For now, we only care about Binary Property aliases for Unicode property escapes.
  276. if (current_property != "Binary Properties"sv)
  277. continue;
  278. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  279. VERIFY((segments.size() == 2) || (segments.size() == 3));
  280. auto alias = segments[0].trim_whitespace();
  281. auto property = segments[1].trim_whitespace();
  282. append_alias(alias, property);
  283. if (segments.size() == 3) {
  284. alias = segments[2].trim_whitespace();
  285. append_alias(alias, property);
  286. }
  287. }
  288. return {};
  289. }
  290. static ErrorOr<void> parse_name_aliases(Core::Stream::BufferedFile& file, UnicodeData& unicode_data)
  291. {
  292. Array<u8, 1024> buffer;
  293. while (TRY(file.can_read_line())) {
  294. auto line = TRY(file.read_line(buffer));
  295. if (line.is_empty() || line.starts_with('#'))
  296. continue;
  297. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  298. VERIFY(segments.size() == 3);
  299. auto code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[0].trim_whitespace());
  300. auto alias = segments[1].trim_whitespace();
  301. auto reason = segments[2].trim_whitespace();
  302. if (reason == "abbreviation"sv) {
  303. auto index = unicode_data.unique_strings.ensure(alias);
  304. unicode_data.code_point_abbreviations.set(*code_point, index);
  305. } else if (reason.is_one_of("correction"sv, "control"sv)) {
  306. if (!unicode_data.code_point_display_name_aliases.contains(*code_point)) {
  307. auto index = unicode_data.unique_strings.ensure(alias);
  308. unicode_data.code_point_display_name_aliases.set(*code_point, index);
  309. }
  310. }
  311. }
  312. return {};
  313. }
  314. static ErrorOr<void> parse_value_alias_list(Core::Stream::BufferedFile& file, StringView desired_category, Vector<String> const& value_list, Vector<Alias>& prop_aliases, bool primary_value_is_first = true, bool sanitize_alias = false)
  315. {
  316. TRY(file.seek(0, Core::Stream::SeekMode::SetPosition));
  317. Array<u8, 1024> buffer;
  318. auto append_alias = [&](auto alias, auto value) {
  319. // Note: The value alias file contains lines such as "Ahom = Ahom", which we should just skip.
  320. if (alias == value)
  321. return;
  322. // FIXME: We will, eventually, need to find where missing properties are located and parse them.
  323. if (!value_list.contains_slow(value))
  324. return;
  325. prop_aliases.append({ value, alias });
  326. };
  327. while (TRY(file.can_read_line())) {
  328. auto line = TRY(file.read_line(buffer));
  329. if (line.is_empty() || line.starts_with('#'))
  330. continue;
  331. if (auto index = line.find('#'); index.has_value())
  332. line = line.substring_view(0, *index);
  333. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  334. auto category = segments[0].trim_whitespace();
  335. if (category != desired_category)
  336. continue;
  337. VERIFY((segments.size() == 3) || (segments.size() == 4));
  338. auto value = primary_value_is_first ? segments[1].trim_whitespace() : segments[2].trim_whitespace();
  339. auto alias = primary_value_is_first ? segments[2].trim_whitespace() : segments[1].trim_whitespace();
  340. append_alias(sanitize_alias ? sanitize_entry(alias).view() : alias, value);
  341. if (segments.size() == 4) {
  342. alias = segments[3].trim_whitespace();
  343. append_alias(sanitize_alias ? sanitize_entry(alias).view() : alias, value);
  344. }
  345. }
  346. return {};
  347. }
  348. static ErrorOr<void> parse_normalization_props(Core::Stream::BufferedFile& file, UnicodeData& unicode_data)
  349. {
  350. Array<u8, 1024> buffer;
  351. while (TRY(file.can_read_line())) {
  352. auto line = TRY(file.read_line(buffer));
  353. if (line.is_empty() || line.starts_with('#'))
  354. continue;
  355. if (auto index = line.find('#'); index.has_value())
  356. line = line.substring_view(0, *index);
  357. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  358. VERIFY((segments.size() == 2) || (segments.size() == 3));
  359. auto code_point_range = parse_code_point_range(segments[0].trim_whitespace());
  360. auto property = segments[1].trim_whitespace().to_string();
  361. Vector<u32> value;
  362. QuickCheck quick_check = QuickCheck::Yes;
  363. if (segments.size() == 3) {
  364. auto value_or_quick_check = segments[2].trim_whitespace();
  365. if ((value_or_quick_check == "N"sv))
  366. quick_check = QuickCheck::No;
  367. else if ((value_or_quick_check == "M"sv))
  368. quick_check = QuickCheck::Maybe;
  369. else
  370. value = parse_code_point_list(value_or_quick_check);
  371. }
  372. auto& normalizations = unicode_data.normalization_props.ensure(property);
  373. normalizations.append({ code_point_range, move(value), quick_check });
  374. auto& prop_list = unicode_data.prop_list.ensure(property);
  375. prop_list.append(move(code_point_range));
  376. }
  377. return {};
  378. }
  379. static void add_canonical_code_point_name(CodePointRange range, StringView name, UnicodeData& unicode_data)
  380. {
  381. // https://www.unicode.org/versions/Unicode15.0.0/ch04.pdf#G142981
  382. // FIXME: Implement the NR1 rules for Hangul syllables.
  383. struct CodePointNameFormat {
  384. CodePointRange code_point_range;
  385. StringView name;
  386. };
  387. // These code point ranges are the NR2 set of name replacements defined by Table 4-8.
  388. constexpr Array<CodePointNameFormat, 16> s_ideographic_replacements { {
  389. { { 0x3400, 0x4DBF }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  390. { { 0x4E00, 0x9FFF }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  391. { { 0xF900, 0xFA6D }, "CJK COMPATIBILITY IDEOGRAPH-{:X}"sv },
  392. { { 0xFA70, 0xFAD9 }, "CJK COMPATIBILITY IDEOGRAPH-{:X}"sv },
  393. { { 0x17000, 0x187F7 }, "TANGUT IDEOGRAPH-{:X}"sv },
  394. { { 0x18B00, 0x18CD5 }, "KHITAN SMALL SCRIPT CHARACTER-{:X}"sv },
  395. { { 0x18D00, 0x18D08 }, "TANGUT IDEOGRAPH-{:X}"sv },
  396. { { 0x1B170, 0x1B2FB }, "NUSHU CHARACTER-{:X}"sv },
  397. { { 0x20000, 0x2A6DF }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  398. { { 0x2A700, 0x2B739 }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  399. { { 0x2B740, 0x2B81D }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  400. { { 0x2B820, 0x2CEA1 }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  401. { { 0x2CEB0, 0x2EBE0 }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  402. { { 0x2F800, 0x2FA1D }, "CJK COMPATIBILITY IDEOGRAPH-{:X}"sv },
  403. { { 0x30000, 0x3134A }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  404. { { 0x31350, 0x323AF }, "CJK UNIFIED IDEOGRAPH-{:X}"sv },
  405. } };
  406. auto it = find_if(s_ideographic_replacements.begin(), s_ideographic_replacements.end(),
  407. [&](auto const& replacement) {
  408. return replacement.code_point_range.first == range.first;
  409. });
  410. if (it != s_ideographic_replacements.end()) {
  411. auto index = unicode_data.unique_strings.ensure(it->name);
  412. unicode_data.code_point_display_names.append({ it->code_point_range, index });
  413. return;
  414. }
  415. it = find_if(s_ideographic_replacements.begin(), s_ideographic_replacements.end(),
  416. [&](auto const& replacement) {
  417. return (replacement.code_point_range.first <= range.first) && (range.first <= replacement.code_point_range.last);
  418. });
  419. if (it != s_ideographic_replacements.end()) {
  420. // Drop code points that will have been captured by a range defined by the ideographic replacements.
  421. return;
  422. }
  423. if (auto alias = unicode_data.code_point_display_name_aliases.get(range.first); alias.has_value()) {
  424. // NR4 states that control code points have a null string as their name. Our implementation
  425. // uses the control code's alias as its display name.
  426. unicode_data.code_point_display_names.append({ range, *alias });
  427. return;
  428. }
  429. auto index = unicode_data.unique_strings.ensure(name);
  430. unicode_data.code_point_display_names.append({ range, index });
  431. }
  432. static Optional<CodePointDecomposition> parse_decomposition_mapping(StringView string, UnicodeData& unicode_data)
  433. {
  434. if (string.is_empty())
  435. return {};
  436. CodePointDecomposition mapping;
  437. auto parts = string.split_view(' ');
  438. VERIFY(parts.size() > 0);
  439. if (parts.first().starts_with('<')) {
  440. auto const tag = parts.take_first().trim("<>"sv);
  441. mapping.tag = String::formatted("{:c}{}", to_ascii_uppercase(tag[0]), tag.substring_view(1));
  442. if (!unicode_data.compatibility_tags.contains_slow(mapping.tag))
  443. unicode_data.compatibility_tags.append(mapping.tag);
  444. }
  445. mapping.decomposition_index = unicode_data.decomposition_mappings.size();
  446. mapping.decomposition_size = parts.size();
  447. for (auto part : parts) {
  448. unicode_data.decomposition_mappings.append(AK::StringUtils::convert_to_uint_from_hex<u32>(part).value());
  449. }
  450. return mapping;
  451. }
  452. static ErrorOr<void> parse_block_display_names(Core::Stream::BufferedFile& file, UnicodeData& unicode_data)
  453. {
  454. Array<u8, 1024> buffer;
  455. while (TRY(file.can_read_line())) {
  456. auto line = TRY(file.read_line(buffer));
  457. if (line.is_empty() || line.starts_with('#'))
  458. continue;
  459. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  460. VERIFY(segments.size() == 2);
  461. auto code_point_range = parse_code_point_range(segments[0].trim_whitespace());
  462. auto display_name = segments[1].trim_whitespace();
  463. auto index = unicode_data.unique_strings.ensure(display_name);
  464. unicode_data.block_display_names.append({ code_point_range, index });
  465. }
  466. TRY(file.seek(0, Core::Stream::SeekMode::SetPosition));
  467. return {};
  468. }
  469. static ErrorOr<void> parse_unicode_data(Core::Stream::BufferedFile& file, UnicodeData& unicode_data)
  470. {
  471. Optional<u32> code_point_range_start;
  472. auto& assigned_code_points = unicode_data.prop_list.find("Assigned"sv)->value;
  473. Optional<u32> assigned_code_point_range_start = 0;
  474. u32 previous_code_point = 0;
  475. Array<u8, 1024> buffer;
  476. while (TRY(file.can_read_line())) {
  477. auto line = TRY(file.read_line(buffer));
  478. if (line.is_empty())
  479. continue;
  480. auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
  481. VERIFY(segments.size() == 15);
  482. CodePointData data {};
  483. data.code_point = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[0]).value();
  484. data.name = segments[1];
  485. data.canonical_combining_class = AK::StringUtils::convert_to_uint<u8>(segments[3]).value();
  486. data.bidi_class = segments[4];
  487. data.decomposition_mapping = parse_decomposition_mapping(segments[5], unicode_data);
  488. data.numeric_value_decimal = AK::StringUtils::convert_to_int<i8>(segments[6]);
  489. data.numeric_value_digit = AK::StringUtils::convert_to_int<i8>(segments[7]);
  490. data.numeric_value_numeric = AK::StringUtils::convert_to_int<i8>(segments[8]);
  491. data.bidi_mirrored = segments[9] == "Y"sv;
  492. data.unicode_1_name = segments[10];
  493. data.iso_comment = segments[11];
  494. data.simple_uppercase_mapping = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[12]);
  495. data.simple_lowercase_mapping = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[13]);
  496. data.simple_titlecase_mapping = AK::StringUtils::convert_to_uint_from_hex<u32>(segments[14]);
  497. if (auto abbreviation = unicode_data.code_point_abbreviations.get(data.code_point); abbreviation.has_value())
  498. data.abbreviation = *abbreviation;
  499. if (!assigned_code_point_range_start.has_value())
  500. assigned_code_point_range_start = data.code_point;
  501. if (data.name.starts_with("<"sv) && data.name.ends_with(", First>"sv)) {
  502. VERIFY(!code_point_range_start.has_value() && assigned_code_point_range_start.has_value());
  503. code_point_range_start = data.code_point;
  504. data.name = data.name.substring(1, data.name.length() - 9);
  505. assigned_code_points.append({ *assigned_code_point_range_start, previous_code_point });
  506. assigned_code_point_range_start.clear();
  507. } else if (data.name.starts_with("<"sv) && data.name.ends_with(", Last>"sv)) {
  508. VERIFY(code_point_range_start.has_value());
  509. CodePointRange code_point_range { *code_point_range_start, data.code_point };
  510. assigned_code_points.append(code_point_range);
  511. data.name = data.name.substring(1, data.name.length() - 8);
  512. code_point_range_start.clear();
  513. add_canonical_code_point_name(code_point_range, data.name, unicode_data);
  514. } else {
  515. add_canonical_code_point_name({ data.code_point, data.code_point }, data.name, unicode_data);
  516. if ((data.code_point > 0) && (data.code_point - previous_code_point) != 1) {
  517. VERIFY(assigned_code_point_range_start.has_value());
  518. assigned_code_points.append({ *assigned_code_point_range_start, previous_code_point });
  519. assigned_code_point_range_start = data.code_point;
  520. }
  521. }
  522. bool has_special_casing { false };
  523. for (auto const& casing : unicode_data.special_casing) {
  524. if (casing.code_point == data.code_point) {
  525. data.special_casing_indices.append(casing.index);
  526. has_special_casing = true;
  527. }
  528. }
  529. unicode_data.code_points_with_non_zero_combining_class += data.canonical_combining_class != 0;
  530. unicode_data.simple_uppercase_mapping_size += data.simple_uppercase_mapping.has_value();
  531. unicode_data.simple_lowercase_mapping_size += data.simple_lowercase_mapping.has_value();
  532. unicode_data.code_points_with_decomposition_mapping += data.decomposition_mapping.has_value();
  533. unicode_data.code_points_with_special_casing += has_special_casing;
  534. unicode_data.largest_special_casing_size = max(unicode_data.largest_special_casing_size, data.special_casing_indices.size());
  535. previous_code_point = data.code_point;
  536. unicode_data.code_point_data.append(move(data));
  537. }
  538. return {};
  539. }
  540. static ErrorOr<void> generate_unicode_data_header(Core::Stream::BufferedFile& file, UnicodeData& unicode_data)
  541. {
  542. StringBuilder builder;
  543. SourceGenerator generator { builder };
  544. generator.set("casing_transform_size", String::number(unicode_data.largest_casing_transform_size));
  545. auto generate_enum = [&](StringView name, StringView default_, Vector<String> values, Vector<Alias> aliases = {}) {
  546. quick_sort(values);
  547. quick_sort(aliases, [](auto& alias1, auto& alias2) { return alias1.alias < alias2.alias; });
  548. generator.set("name", name);
  549. generator.set("underlying", String::formatted("{}UnderlyingType", name));
  550. generator.set("type", ((values.size() + !default_.is_empty()) < 256) ? "u8"sv : "u16"sv);
  551. generator.append(R"~~~(
  552. using @underlying@ = @type@;
  553. enum class @name@ : @underlying@ {)~~~");
  554. if (!default_.is_empty()) {
  555. generator.set("default", default_);
  556. generator.append(R"~~~(
  557. @default@,)~~~");
  558. }
  559. for (auto const& value : values) {
  560. generator.set("value", value);
  561. generator.append(R"~~~(
  562. @value@,)~~~");
  563. }
  564. for (auto const& alias : aliases) {
  565. generator.set("alias", alias.alias);
  566. generator.set("value", alias.name);
  567. generator.append(R"~~~(
  568. @alias@ = @value@,)~~~");
  569. }
  570. generator.append(R"~~~(
  571. };
  572. )~~~");
  573. };
  574. generator.append(R"~~~(
  575. #pragma once
  576. #include <AK/Types.h>
  577. #include <LibUnicode/Forward.h>
  578. namespace Unicode {
  579. )~~~");
  580. generate_enum("Locale"sv, "None"sv, unicode_data.locales);
  581. generate_enum("Condition"sv, "None"sv, move(unicode_data.conditions));
  582. generate_enum("GeneralCategory"sv, {}, unicode_data.general_categories.keys(), unicode_data.general_category_aliases);
  583. generate_enum("Property"sv, {}, unicode_data.prop_list.keys(), unicode_data.prop_aliases);
  584. generate_enum("Script"sv, {}, unicode_data.script_list.keys(), unicode_data.script_aliases);
  585. generate_enum("Block"sv, {}, unicode_data.block_list.keys(), unicode_data.block_aliases);
  586. generate_enum("GraphemeBreakProperty"sv, {}, unicode_data.grapheme_break_props.keys());
  587. generate_enum("WordBreakProperty"sv, {}, unicode_data.word_break_props.keys());
  588. generate_enum("SentenceBreakProperty"sv, {}, unicode_data.sentence_break_props.keys());
  589. generate_enum("CompatibilityFormattingTag"sv, "Canonical"sv, unicode_data.compatibility_tags);
  590. generator.append(R"~~~(
  591. struct SpecialCasing {
  592. u32 code_point { 0 };
  593. u32 lowercase_mapping[@casing_transform_size@];
  594. u32 lowercase_mapping_size { 0 };
  595. u32 uppercase_mapping[@casing_transform_size@];
  596. u32 uppercase_mapping_size { 0 };
  597. u32 titlecase_mapping[@casing_transform_size@];
  598. u32 titlecase_mapping_size { 0 };
  599. Locale locale { Locale::None };
  600. Condition condition { Condition::None };
  601. };
  602. struct CodePointDecomposition {
  603. u32 code_point { 0 };
  604. CompatibilityFormattingTag tag { CompatibilityFormattingTag::Canonical };
  605. Span<u32 const> decomposition;
  606. };
  607. Optional<Locale> locale_from_string(StringView locale);
  608. }
  609. )~~~");
  610. TRY(file.write(generator.as_string_view().bytes()));
  611. return {};
  612. }
  613. static ErrorOr<void> generate_unicode_data_implementation(Core::Stream::BufferedFile& file, UnicodeData const& unicode_data)
  614. {
  615. StringBuilder builder;
  616. SourceGenerator generator { builder };
  617. generator.set("string_index_type"sv, s_string_index_type);
  618. generator.set("largest_special_casing_size", String::number(unicode_data.largest_special_casing_size));
  619. generator.set("special_casing_size", String::number(unicode_data.special_casing.size()));
  620. generator.append(R"~~~(
  621. #include <AK/Array.h>
  622. #include <AK/BinarySearch.h>
  623. #include <AK/CharacterTypes.h>
  624. #include <AK/Optional.h>
  625. #include <AK/Span.h>
  626. #include <AK/String.h>
  627. #include <AK/StringView.h>
  628. #include <LibUnicode/CharacterTypes.h>
  629. #include <LibUnicode/UnicodeData.h>
  630. #include <LibUnicode/Normalize.h>
  631. namespace Unicode {
  632. )~~~");
  633. unicode_data.unique_strings.generate(generator);
  634. auto append_list_and_size = [&](auto const& list, StringView format) {
  635. if (list.is_empty()) {
  636. generator.append(", {}, 0");
  637. return;
  638. }
  639. bool first = true;
  640. generator.append(", {");
  641. for (auto const& item : list) {
  642. generator.append(first ? " "sv : ", "sv);
  643. generator.append(String::formatted(format, item));
  644. first = false;
  645. }
  646. generator.append(String::formatted(" }}, {}", list.size()));
  647. };
  648. generator.append(R"~~~(
  649. static constexpr Array<SpecialCasing, @special_casing_size@> s_special_casing { {)~~~");
  650. for (auto const& casing : unicode_data.special_casing) {
  651. generator.set("code_point", String::formatted("{:#x}", casing.code_point));
  652. generator.append(R"~~~(
  653. { @code_point@)~~~");
  654. constexpr auto format = "0x{:x}"sv;
  655. append_list_and_size(casing.lowercase_mapping, format);
  656. append_list_and_size(casing.uppercase_mapping, format);
  657. append_list_and_size(casing.titlecase_mapping, format);
  658. generator.set("locale", casing.locale.is_empty() ? "None" : casing.locale);
  659. generator.append(", Locale::@locale@");
  660. generator.set("condition", casing.condition.is_empty() ? "None" : casing.condition);
  661. generator.append(", Condition::@condition@");
  662. generator.append(" },");
  663. }
  664. generator.append(R"~~~(
  665. } };
  666. struct CodePointMapping {
  667. u32 code_point { 0 };
  668. u32 mapping { 0 };
  669. };
  670. struct SpecialCaseMapping {
  671. u32 code_point { 0 };
  672. Array<SpecialCasing const*, @largest_special_casing_size@> special_casing {};
  673. u32 special_casing_size { 0 };
  674. };
  675. struct CodePointAbbreviation {
  676. u32 code_point { 0 };
  677. @string_index_type@ abbreviation { 0 };
  678. };
  679. template<typename MappingType>
  680. struct CodePointComparator {
  681. constexpr int operator()(u32 code_point, MappingType const& mapping)
  682. {
  683. return code_point - mapping.code_point;
  684. }
  685. };
  686. struct CodePointRangeComparator {
  687. constexpr int operator()(u32 code_point, CodePointRange const& range)
  688. {
  689. return (code_point > range.last) - (code_point < range.first);
  690. }
  691. };
  692. struct BlockNameData {
  693. CodePointRange code_point_range {};
  694. @string_index_type@ display_name { 0 };
  695. };
  696. struct BlockNameComparator : public CodePointRangeComparator {
  697. constexpr int operator()(u32 code_point, BlockNameData const& name)
  698. {
  699. return CodePointRangeComparator::operator()(code_point, name.code_point_range);
  700. }
  701. };
  702. struct CodePointName {
  703. CodePointRange code_point_range {};
  704. @string_index_type@ display_name { 0 };
  705. };
  706. struct CodePointNameComparator : public CodePointRangeComparator {
  707. constexpr int operator()(u32 code_point, CodePointName const& name)
  708. {
  709. return CodePointRangeComparator::operator()(code_point, name.code_point_range);
  710. }
  711. };
  712. )~~~");
  713. generator.set("decomposition_mappings_size", String::number(unicode_data.decomposition_mappings.size()));
  714. generator.append("\nstatic constexpr Array<u32, @decomposition_mappings_size@> s_decomposition_mappings_data { ");
  715. generator.append(String::join(", "sv, unicode_data.decomposition_mappings, "{:#x}"sv));
  716. generator.append(" };\n");
  717. auto append_code_point_mappings = [&](StringView name, StringView mapping_type, u32 size, auto mapping_getter) {
  718. generator.set("name", name);
  719. generator.set("mapping_type", mapping_type);
  720. generator.set("size", String::number(size));
  721. generator.append(R"~~~(
  722. static constexpr Array<@mapping_type@, @size@> s_@name@_mappings { {
  723. )~~~");
  724. constexpr size_t max_mappings_per_row = 20;
  725. size_t mappings_in_current_row = 0;
  726. for (auto const& data : unicode_data.code_point_data) {
  727. auto mapping = mapping_getter(data);
  728. if constexpr (requires { mapping.has_value(); }) {
  729. if (!mapping.has_value())
  730. continue;
  731. } else {
  732. if (mapping.is_empty())
  733. continue;
  734. }
  735. if (mappings_in_current_row++ > 0)
  736. generator.append(" ");
  737. generator.set("code_point", String::formatted("{:#x}", data.code_point));
  738. generator.append("{ @code_point@");
  739. if constexpr (IsSame<decltype(mapping), Optional<u32>> || IsSame<decltype(mapping), Optional<StringIndexType>>) {
  740. generator.set("mapping", String::formatted("{:#x}", *mapping));
  741. generator.append(", @mapping@ },");
  742. } else if constexpr (IsSame<decltype(mapping), Optional<CodePointDecomposition>>) {
  743. generator.set("tag", mapping->tag);
  744. generator.set("start", String::number(mapping->decomposition_index));
  745. generator.set("size", String::number(mapping->decomposition_size));
  746. generator.append(", CompatibilityFormattingTag::@tag@, Span<u32 const> { s_decomposition_mappings_data.data() + @start@, @size@ } },");
  747. } else {
  748. append_list_and_size(data.special_casing_indices, "&s_special_casing[{}]"sv);
  749. generator.append(" },");
  750. }
  751. if (mappings_in_current_row == max_mappings_per_row) {
  752. mappings_in_current_row = 0;
  753. generator.append("\n ");
  754. }
  755. }
  756. generator.append(R"~~~(
  757. } };
  758. )~~~");
  759. };
  760. append_code_point_mappings("combining_class"sv, "CodePointMapping"sv, unicode_data.code_points_with_non_zero_combining_class,
  761. [](auto const& data) -> Optional<u32> {
  762. if (data.canonical_combining_class == 0)
  763. return {};
  764. return data.canonical_combining_class;
  765. });
  766. append_code_point_mappings("uppercase"sv, "CodePointMapping"sv, unicode_data.simple_uppercase_mapping_size, [](auto const& data) { return data.simple_uppercase_mapping; });
  767. append_code_point_mappings("lowercase"sv, "CodePointMapping"sv, unicode_data.simple_lowercase_mapping_size, [](auto const& data) { return data.simple_lowercase_mapping; });
  768. append_code_point_mappings("special_case"sv, "SpecialCaseMapping"sv, unicode_data.code_points_with_special_casing, [](auto const& data) { return data.special_casing_indices; });
  769. append_code_point_mappings("abbreviation"sv, "CodePointAbbreviation"sv, unicode_data.code_point_abbreviations.size(), [](auto const& data) { return data.abbreviation; });
  770. append_code_point_mappings("decomposition"sv, "CodePointDecomposition"sv, unicode_data.code_points_with_decomposition_mapping,
  771. [](auto const& data) {
  772. return data.decomposition_mapping;
  773. });
  774. auto append_code_point_range_list = [&](String name, Vector<CodePointRange> const& ranges) {
  775. generator.set("name", name);
  776. generator.set("size", String::number(ranges.size()));
  777. generator.append(R"~~~(
  778. static constexpr Array<CodePointRange, @size@> @name@ { {
  779. )~~~");
  780. constexpr size_t max_ranges_per_row = 20;
  781. size_t ranges_in_current_row = 0;
  782. for (auto const& range : ranges) {
  783. if (ranges_in_current_row++ > 0)
  784. generator.append(" ");
  785. generator.set("first", String::formatted("{:#x}", range.first));
  786. generator.set("last", String::formatted("{:#x}", range.last));
  787. generator.append("{ @first@, @last@ },");
  788. if (ranges_in_current_row == max_ranges_per_row) {
  789. ranges_in_current_row = 0;
  790. generator.append("\n ");
  791. }
  792. }
  793. generator.append(R"~~~(
  794. } };
  795. )~~~");
  796. };
  797. auto append_prop_list = [&](StringView collection_name, StringView property_format, PropList const& property_list) {
  798. for (auto const& property : property_list) {
  799. auto name = String::formatted(property_format, property.key);
  800. append_code_point_range_list(move(name), property.value);
  801. }
  802. auto property_names = property_list.keys();
  803. quick_sort(property_names);
  804. generator.set("name", collection_name);
  805. generator.set("size", String::number(property_names.size()));
  806. generator.append(R"~~~(
  807. static constexpr Array<Span<CodePointRange const>, @size@> @name@ { {)~~~");
  808. for (auto const& property_name : property_names) {
  809. generator.set("name", String::formatted(property_format, property_name));
  810. generator.append(R"~~~(
  811. @name@.span(),)~~~");
  812. }
  813. generator.append(R"~~~(
  814. } };
  815. )~~~");
  816. };
  817. append_prop_list("s_general_categories"sv, "s_general_category_{}"sv, unicode_data.general_categories);
  818. append_prop_list("s_properties"sv, "s_property_{}"sv, unicode_data.prop_list);
  819. append_prop_list("s_scripts"sv, "s_script_{}"sv, unicode_data.script_list);
  820. append_prop_list("s_script_extensions"sv, "s_script_extension_{}"sv, unicode_data.script_extensions);
  821. append_prop_list("s_blocks"sv, "s_block_{}"sv, unicode_data.block_list);
  822. append_prop_list("s_grapheme_break_properties"sv, "s_grapheme_break_property_{}"sv, unicode_data.grapheme_break_props);
  823. append_prop_list("s_word_break_properties"sv, "s_word_break_property_{}"sv, unicode_data.word_break_props);
  824. append_prop_list("s_sentence_break_properties"sv, "s_sentence_break_property_{}"sv, unicode_data.sentence_break_props);
  825. auto append_code_point_display_names = [&](StringView type, StringView name, auto const& display_names) {
  826. constexpr size_t max_values_per_row = 30;
  827. size_t values_in_current_row = 0;
  828. generator.set("type", type);
  829. generator.set("name", name);
  830. generator.set("size", String::number(display_names.size()));
  831. generator.append(R"~~~(
  832. static constexpr Array<@type@, @size@> @name@ { {
  833. )~~~");
  834. for (auto const& display_name : display_names) {
  835. if (values_in_current_row++ > 0)
  836. generator.append(", ");
  837. generator.set("first", String::formatted("{:#x}", display_name.code_point_range.first));
  838. generator.set("last", String::formatted("{:#x}", display_name.code_point_range.last));
  839. generator.set("name", String::number(display_name.name));
  840. generator.append("{ { @first@, @last@ }, @name@ }");
  841. if (values_in_current_row == max_values_per_row) {
  842. values_in_current_row = 0;
  843. generator.append(",\n ");
  844. }
  845. }
  846. generator.append(R"~~~(
  847. } };
  848. )~~~");
  849. };
  850. append_code_point_display_names("BlockNameData"sv, "s_block_display_names"sv, unicode_data.block_display_names);
  851. append_code_point_display_names("CodePointName"sv, "s_code_point_display_names"sv, unicode_data.code_point_display_names);
  852. generator.append(R"~~~(
  853. Optional<StringView> code_point_block_display_name(u32 code_point)
  854. {
  855. if (auto const* entry = binary_search(s_block_display_names, code_point, nullptr, BlockNameComparator {}))
  856. return decode_string(entry->display_name);
  857. return {};
  858. }
  859. Span<BlockName const> block_display_names()
  860. {
  861. static auto display_names = []() {
  862. Array<BlockName, s_block_display_names.size()> display_names;
  863. for (size_t i = 0; i < s_block_display_names.size(); ++i) {
  864. auto const& display_name = s_block_display_names[i];
  865. display_names[i] = { display_name.code_point_range, decode_string(display_name.display_name) };
  866. }
  867. return display_names;
  868. }();
  869. return display_names.span();
  870. }
  871. Optional<String> code_point_display_name(u32 code_point)
  872. {
  873. if (auto const* entry = binary_search(s_code_point_display_names, code_point, nullptr, CodePointNameComparator {})) {
  874. auto display_name = decode_string(entry->display_name);
  875. if (display_name.ends_with("{:X}"sv))
  876. return String::formatted(display_name, code_point);
  877. return display_name;
  878. }
  879. return {};
  880. }
  881. )~~~");
  882. auto append_code_point_mapping_search = [&](StringView method, StringView mappings, StringView fallback) {
  883. generator.set("method", method);
  884. generator.set("mappings", mappings);
  885. generator.set("fallback", fallback);
  886. generator.append(R"~~~(
  887. u32 @method@(u32 code_point)
  888. {
  889. auto const* mapping = binary_search(@mappings@, code_point, nullptr, CodePointComparator<CodePointMapping> {});
  890. return mapping ? mapping->mapping : @fallback@;
  891. }
  892. )~~~");
  893. };
  894. append_code_point_mapping_search("canonical_combining_class"sv, "s_combining_class_mappings"sv, "0"sv);
  895. append_code_point_mapping_search("to_unicode_uppercase"sv, "s_uppercase_mappings"sv, "code_point"sv);
  896. append_code_point_mapping_search("to_unicode_lowercase"sv, "s_lowercase_mappings"sv, "code_point"sv);
  897. generator.append(R"~~~(
  898. Span<SpecialCasing const* const> special_case_mapping(u32 code_point)
  899. {
  900. auto const* mapping = binary_search(s_special_case_mappings, code_point, nullptr, CodePointComparator<SpecialCaseMapping> {});
  901. if (mapping == nullptr)
  902. return {};
  903. return mapping->special_casing.span().slice(0, mapping->special_casing_size);
  904. }
  905. Optional<StringView> code_point_abbreviation(u32 code_point)
  906. {
  907. auto const* mapping = binary_search(s_abbreviation_mappings, code_point, nullptr, CodePointComparator<CodePointAbbreviation> {});
  908. if (mapping == nullptr)
  909. return {};
  910. if (mapping->abbreviation == 0)
  911. return {};
  912. return decode_string(mapping->abbreviation);
  913. }
  914. Optional<CodePointDecomposition const&> code_point_decomposition(u32 code_point)
  915. {
  916. auto const* mapping = binary_search(s_decomposition_mappings, code_point, nullptr, CodePointComparator<CodePointDecomposition> {});
  917. if (mapping == nullptr)
  918. return {};
  919. return *mapping;
  920. }
  921. Span<CodePointDecomposition const> code_point_decompositions()
  922. {
  923. return s_decomposition_mappings;
  924. }
  925. )~~~");
  926. auto append_prop_search = [&](StringView enum_title, StringView enum_snake, StringView collection_name) {
  927. generator.set("enum_title", enum_title);
  928. generator.set("enum_snake", enum_snake);
  929. generator.set("collection_name", collection_name);
  930. generator.append(R"~~~(
  931. bool code_point_has_@enum_snake@(u32 code_point, @enum_title@ @enum_snake@)
  932. {
  933. auto index = static_cast<@enum_title@UnderlyingType>(@enum_snake@);
  934. auto const& ranges = @collection_name@.at(index);
  935. auto const* range = binary_search(ranges, code_point, nullptr, CodePointRangeComparator {});
  936. return range != nullptr;
  937. }
  938. )~~~");
  939. };
  940. auto append_from_string = [&](StringView enum_title, StringView enum_snake, auto const& prop_list, Vector<Alias> const& aliases) {
  941. HashValueMap<StringView> hashes;
  942. hashes.ensure_capacity(prop_list.size() + aliases.size());
  943. ValueFromStringOptions options {};
  944. for (auto const& prop : prop_list) {
  945. if constexpr (IsSame<RemoveCVReference<decltype(prop)>, String>) {
  946. hashes.set(CaseInsensitiveStringViewTraits::hash(prop), prop);
  947. options.sensitivity = CaseSensitivity::CaseInsensitive;
  948. } else {
  949. hashes.set(prop.key.hash(), prop.key);
  950. }
  951. }
  952. for (auto const& alias : aliases)
  953. hashes.set(alias.alias.hash(), alias.alias);
  954. generate_value_from_string(generator, "{}_from_string"sv, enum_title, enum_snake, move(hashes), options);
  955. };
  956. append_from_string("Locale"sv, "locale"sv, unicode_data.locales, {});
  957. append_prop_search("GeneralCategory"sv, "general_category"sv, "s_general_categories"sv);
  958. append_from_string("GeneralCategory"sv, "general_category"sv, unicode_data.general_categories, unicode_data.general_category_aliases);
  959. append_prop_search("Property"sv, "property"sv, "s_properties"sv);
  960. append_from_string("Property"sv, "property"sv, unicode_data.prop_list, unicode_data.prop_aliases);
  961. append_prop_search("Script"sv, "script"sv, "s_scripts"sv);
  962. append_prop_search("Script"sv, "script_extension"sv, "s_script_extensions"sv);
  963. append_from_string("Script"sv, "script"sv, unicode_data.script_list, unicode_data.script_aliases);
  964. append_prop_search("Block"sv, "block"sv, "s_blocks"sv);
  965. append_from_string("Block"sv, "block"sv, unicode_data.block_list, unicode_data.block_aliases);
  966. append_prop_search("GraphemeBreakProperty"sv, "grapheme_break_property"sv, "s_grapheme_break_properties"sv);
  967. append_prop_search("WordBreakProperty"sv, "word_break_property"sv, "s_word_break_properties"sv);
  968. append_prop_search("SentenceBreakProperty"sv, "sentence_break_property"sv, "s_sentence_break_properties"sv);
  969. generator.append(R"~~~(
  970. }
  971. )~~~");
  972. TRY(file.write(generator.as_string_view().bytes()));
  973. return {};
  974. }
  975. static Vector<u32> flatten_code_point_ranges(Vector<CodePointRange> const& code_points)
  976. {
  977. Vector<u32> flattened;
  978. for (auto const& range : code_points) {
  979. flattened.grow_capacity(range.last - range.first);
  980. for (u32 code_point = range.first; code_point <= range.last; ++code_point)
  981. flattened.append(code_point);
  982. }
  983. return flattened;
  984. }
  985. static Vector<CodePointRange> form_code_point_ranges(Vector<u32> code_points)
  986. {
  987. Vector<CodePointRange> ranges;
  988. u32 range_start = code_points[0];
  989. u32 range_end = range_start;
  990. for (size_t i = 1; i < code_points.size(); ++i) {
  991. u32 code_point = code_points[i];
  992. if ((code_point - range_end) == 1) {
  993. range_end = code_point;
  994. } else {
  995. ranges.append({ range_start, range_end });
  996. range_start = code_point;
  997. range_end = code_point;
  998. }
  999. }
  1000. ranges.append({ range_start, range_end });
  1001. return ranges;
  1002. }
  1003. static void sort_and_merge_code_point_ranges(Vector<CodePointRange>& code_points)
  1004. {
  1005. quick_sort(code_points, [](auto const& range1, auto const& range2) {
  1006. return range1.first < range2.first;
  1007. });
  1008. for (size_t i = 0; i < code_points.size() - 1;) {
  1009. if (code_points[i].last >= code_points[i + 1].first) {
  1010. code_points[i].last = max(code_points[i].last, code_points[i + 1].last);
  1011. code_points.remove(i + 1);
  1012. } else {
  1013. ++i;
  1014. }
  1015. }
  1016. auto all_code_points = flatten_code_point_ranges(code_points);
  1017. code_points = form_code_point_ranges(all_code_points);
  1018. }
  1019. static void populate_general_category_unions(PropList& general_categories)
  1020. {
  1021. // The Unicode standard defines General Category values which are not in any UCD file. These
  1022. // values are simply unions of other values.
  1023. // https://www.unicode.org/reports/tr44/#GC_Values_Table
  1024. auto populate_union = [&](auto alias, auto categories) {
  1025. auto& code_points = general_categories.ensure(alias);
  1026. for (auto const& category : categories)
  1027. code_points.extend(general_categories.find(category)->value);
  1028. sort_and_merge_code_point_ranges(code_points);
  1029. };
  1030. populate_union("LC"sv, Array { "Ll"sv, "Lu"sv, "Lt"sv });
  1031. populate_union("L"sv, Array { "Lu"sv, "Ll"sv, "Lt"sv, "Lm"sv, "Lo"sv });
  1032. populate_union("M"sv, Array { "Mn"sv, "Mc"sv, "Me"sv });
  1033. populate_union("N"sv, Array { "Nd"sv, "Nl"sv, "No"sv });
  1034. populate_union("P"sv, Array { "Pc"sv, "Pd"sv, "Ps"sv, "Pe"sv, "Pi"sv, "Pf"sv, "Po"sv });
  1035. populate_union("S"sv, Array { "Sm"sv, "Sc"sv, "Sk"sv, "So"sv });
  1036. populate_union("Z"sv, Array { "Zs"sv, "Zl"sv, "Zp"sv });
  1037. populate_union("C"sv, Array { "Cc"sv, "Cf"sv, "Cs"sv, "Co"sv, "Cn"sv });
  1038. }
  1039. static void normalize_script_extensions(PropList& script_extensions, PropList const& script_list, Vector<Alias> const& script_aliases)
  1040. {
  1041. // The ScriptExtensions UCD file lays out its code point ranges rather uniquely compared to
  1042. // other files. The Script listed on each line may either be a full Script string or an aliased
  1043. // abbreviation. Further, the extensions may or may not include the base Script list. Normalize
  1044. // the extensions here to be keyed by the full Script name and always include the base list.
  1045. auto extensions = move(script_extensions);
  1046. script_extensions = script_list;
  1047. for (auto const& extension : extensions) {
  1048. auto it = find_if(script_aliases.begin(), script_aliases.end(), [&](auto const& alias) { return extension.key == alias.alias; });
  1049. auto const& key = (it == script_aliases.end()) ? extension.key : it->name;
  1050. auto& code_points = script_extensions.find(key)->value;
  1051. code_points.extend(extension.value);
  1052. sort_and_merge_code_point_ranges(code_points);
  1053. }
  1054. // Lastly, the Common and Inherited script extensions are special. They must not contain any
  1055. // code points which appear in other script extensions. The ScriptExtensions UCD file does not
  1056. // list these extensions, therefore this peculiarity must be handled programmatically.
  1057. // https://www.unicode.org/reports/tr24/#Assignment_ScriptX_Values
  1058. auto code_point_has_other_extension = [&](StringView key, u32 code_point) {
  1059. for (auto const& extension : extensions) {
  1060. if (extension.key == key)
  1061. continue;
  1062. if (any_of(extension.value, [&](auto const& r) { return (r.first <= code_point) && (code_point <= r.last); }))
  1063. return true;
  1064. }
  1065. return false;
  1066. };
  1067. auto get_code_points_without_other_extensions = [&](StringView key) {
  1068. auto code_points = flatten_code_point_ranges(script_list.find(key)->value);
  1069. code_points.remove_all_matching([&](u32 c) { return code_point_has_other_extension(key, c); });
  1070. return code_points;
  1071. };
  1072. auto common_code_points = get_code_points_without_other_extensions("Common"sv);
  1073. script_extensions.set("Common"sv, form_code_point_ranges(common_code_points));
  1074. auto inherited_code_points = get_code_points_without_other_extensions("Inherited"sv);
  1075. script_extensions.set("Inherited"sv, form_code_point_ranges(inherited_code_points));
  1076. }
  1077. ErrorOr<int> serenity_main(Main::Arguments arguments)
  1078. {
  1079. StringView generated_header_path;
  1080. StringView generated_implementation_path;
  1081. StringView unicode_data_path;
  1082. StringView special_casing_path;
  1083. StringView derived_general_category_path;
  1084. StringView prop_list_path;
  1085. StringView derived_core_prop_path;
  1086. StringView derived_binary_prop_path;
  1087. StringView prop_alias_path;
  1088. StringView prop_value_alias_path;
  1089. StringView name_alias_path;
  1090. StringView scripts_path;
  1091. StringView script_extensions_path;
  1092. StringView blocks_path;
  1093. StringView emoji_data_path;
  1094. StringView normalization_path;
  1095. StringView grapheme_break_path;
  1096. StringView word_break_path;
  1097. StringView sentence_break_path;
  1098. Core::ArgsParser args_parser;
  1099. args_parser.add_option(generated_header_path, "Path to the Unicode Data header file to generate", "generated-header-path", 'h', "generated-header-path");
  1100. args_parser.add_option(generated_implementation_path, "Path to the Unicode Data implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
  1101. args_parser.add_option(unicode_data_path, "Path to UnicodeData.txt file", "unicode-data-path", 'u', "unicode-data-path");
  1102. args_parser.add_option(special_casing_path, "Path to SpecialCasing.txt file", "special-casing-path", 's', "special-casing-path");
  1103. args_parser.add_option(derived_general_category_path, "Path to DerivedGeneralCategory.txt file", "derived-general-category-path", 'g', "derived-general-category-path");
  1104. args_parser.add_option(prop_list_path, "Path to PropList.txt file", "prop-list-path", 'p', "prop-list-path");
  1105. args_parser.add_option(derived_core_prop_path, "Path to DerivedCoreProperties.txt file", "derived-core-prop-path", 'd', "derived-core-prop-path");
  1106. args_parser.add_option(derived_binary_prop_path, "Path to DerivedBinaryProperties.txt file", "derived-binary-prop-path", 'b', "derived-binary-prop-path");
  1107. args_parser.add_option(prop_alias_path, "Path to PropertyAliases.txt file", "prop-alias-path", 'a', "prop-alias-path");
  1108. args_parser.add_option(prop_value_alias_path, "Path to PropertyValueAliases.txt file", "prop-value-alias-path", 'v', "prop-value-alias-path");
  1109. args_parser.add_option(name_alias_path, "Path to NameAliases.txt file", "name-alias-path", 'm', "name-alias-path");
  1110. args_parser.add_option(scripts_path, "Path to Scripts.txt file", "scripts-path", 'r', "scripts-path");
  1111. args_parser.add_option(script_extensions_path, "Path to ScriptExtensions.txt file", "script-extensions-path", 'x', "script-extensions-path");
  1112. args_parser.add_option(blocks_path, "Path to Blocks.txt file", "blocks-path", 'k', "blocks-path");
  1113. args_parser.add_option(emoji_data_path, "Path to emoji-data.txt file", "emoji-data-path", 'e', "emoji-data-path");
  1114. args_parser.add_option(normalization_path, "Path to DerivedNormalizationProps.txt file", "normalization-path", 'n', "normalization-path");
  1115. args_parser.add_option(grapheme_break_path, "Path to GraphemeBreakProperty.txt file", "grapheme-break-path", 'f', "grapheme-break-path");
  1116. args_parser.add_option(word_break_path, "Path to WordBreakProperty.txt file", "word-break-path", 'w', "word-break-path");
  1117. args_parser.add_option(sentence_break_path, "Path to SentenceBreakProperty.txt file", "sentence-break-path", 'i', "sentence-break-path");
  1118. args_parser.parse(arguments);
  1119. auto generated_header_file = TRY(open_file(generated_header_path, Core::Stream::OpenMode::Write));
  1120. auto generated_implementation_file = TRY(open_file(generated_implementation_path, Core::Stream::OpenMode::Write));
  1121. auto unicode_data_file = TRY(open_file(unicode_data_path, Core::Stream::OpenMode::Read));
  1122. auto derived_general_category_file = TRY(open_file(derived_general_category_path, Core::Stream::OpenMode::Read));
  1123. auto special_casing_file = TRY(open_file(special_casing_path, Core::Stream::OpenMode::Read));
  1124. auto prop_list_file = TRY(open_file(prop_list_path, Core::Stream::OpenMode::Read));
  1125. auto derived_core_prop_file = TRY(open_file(derived_core_prop_path, Core::Stream::OpenMode::Read));
  1126. auto derived_binary_prop_file = TRY(open_file(derived_binary_prop_path, Core::Stream::OpenMode::Read));
  1127. auto prop_alias_file = TRY(open_file(prop_alias_path, Core::Stream::OpenMode::Read));
  1128. auto prop_value_alias_file = TRY(open_file(prop_value_alias_path, Core::Stream::OpenMode::Read));
  1129. auto name_alias_file = TRY(open_file(name_alias_path, Core::Stream::OpenMode::Read));
  1130. auto scripts_file = TRY(open_file(scripts_path, Core::Stream::OpenMode::Read));
  1131. auto script_extensions_file = TRY(open_file(script_extensions_path, Core::Stream::OpenMode::Read));
  1132. auto blocks_file = TRY(open_file(blocks_path, Core::Stream::OpenMode::Read));
  1133. auto emoji_data_file = TRY(open_file(emoji_data_path, Core::Stream::OpenMode::Read));
  1134. auto normalization_file = TRY(open_file(normalization_path, Core::Stream::OpenMode::Read));
  1135. auto grapheme_break_file = TRY(open_file(grapheme_break_path, Core::Stream::OpenMode::Read));
  1136. auto word_break_file = TRY(open_file(word_break_path, Core::Stream::OpenMode::Read));
  1137. auto sentence_break_file = TRY(open_file(sentence_break_path, Core::Stream::OpenMode::Read));
  1138. UnicodeData unicode_data {};
  1139. TRY(parse_special_casing(*special_casing_file, unicode_data));
  1140. TRY(parse_prop_list(*derived_general_category_file, unicode_data.general_categories));
  1141. TRY(parse_prop_list(*prop_list_file, unicode_data.prop_list));
  1142. TRY(parse_prop_list(*derived_core_prop_file, unicode_data.prop_list));
  1143. TRY(parse_prop_list(*derived_binary_prop_file, unicode_data.prop_list));
  1144. TRY(parse_prop_list(*emoji_data_file, unicode_data.prop_list));
  1145. TRY(parse_normalization_props(*normalization_file, unicode_data));
  1146. TRY(parse_alias_list(*prop_alias_file, unicode_data.prop_list, unicode_data.prop_aliases));
  1147. TRY(parse_prop_list(*scripts_file, unicode_data.script_list));
  1148. TRY(parse_prop_list(*script_extensions_file, unicode_data.script_extensions, true));
  1149. TRY(parse_block_display_names(*blocks_file, unicode_data));
  1150. TRY(parse_prop_list(*blocks_file, unicode_data.block_list, false, true));
  1151. TRY(parse_name_aliases(*name_alias_file, unicode_data));
  1152. TRY(parse_prop_list(*grapheme_break_file, unicode_data.grapheme_break_props));
  1153. TRY(parse_prop_list(*word_break_file, unicode_data.word_break_props));
  1154. TRY(parse_prop_list(*sentence_break_file, unicode_data.sentence_break_props));
  1155. populate_general_category_unions(unicode_data.general_categories);
  1156. TRY(parse_unicode_data(*unicode_data_file, unicode_data));
  1157. TRY(parse_value_alias_list(*prop_value_alias_file, "gc"sv, unicode_data.general_categories.keys(), unicode_data.general_category_aliases));
  1158. TRY(parse_value_alias_list(*prop_value_alias_file, "sc"sv, unicode_data.script_list.keys(), unicode_data.script_aliases, false));
  1159. TRY(parse_value_alias_list(*prop_value_alias_file, "blk"sv, unicode_data.block_list.keys(), unicode_data.block_aliases, false, true));
  1160. normalize_script_extensions(unicode_data.script_extensions, unicode_data.script_list, unicode_data.script_aliases);
  1161. TRY(generate_unicode_data_header(*generated_header_file, unicode_data));
  1162. TRY(generate_unicode_data_implementation(*generated_implementation_file, unicode_data));
  1163. return 0;
  1164. }