GenerateUnicodeLocale.cpp 46 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084
  1. /*
  2. * Copyright (c) 2021, Tim Flynn <trflynn89@pm.me>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include "GeneratorUtil.h"
  7. #include <AK/AllOf.h>
  8. #include <AK/CharacterTypes.h>
  9. #include <AK/Format.h>
  10. #include <AK/HashMap.h>
  11. #include <AK/JsonObject.h>
  12. #include <AK/JsonParser.h>
  13. #include <AK/JsonValue.h>
  14. #include <AK/LexicalPath.h>
  15. #include <AK/QuickSort.h>
  16. #include <AK/SourceGenerator.h>
  17. #include <AK/String.h>
  18. #include <AK/StringBuilder.h>
  19. #include <LibCore/ArgsParser.h>
  20. #include <LibCore/DirIterator.h>
  21. #include <LibCore/File.h>
  22. using StringIndexType = u16;
  23. constexpr auto s_string_index_type = "u16"sv;
  24. struct ListPatterns {
  25. String type;
  26. String style;
  27. StringIndexType start { 0 };
  28. StringIndexType middle { 0 };
  29. StringIndexType end { 0 };
  30. StringIndexType pair { 0 };
  31. };
  32. struct Locale {
  33. String language;
  34. Optional<String> territory;
  35. Optional<String> variant;
  36. HashMap<String, StringIndexType> languages;
  37. HashMap<String, StringIndexType> territories;
  38. HashMap<String, StringIndexType> scripts;
  39. HashMap<String, StringIndexType> long_currencies;
  40. HashMap<String, StringIndexType> short_currencies;
  41. HashMap<String, StringIndexType> narrow_currencies;
  42. HashMap<String, StringIndexType> numeric_currencies;
  43. HashMap<String, StringIndexType> keywords;
  44. Vector<ListPatterns> list_patterns;
  45. };
  46. struct LanguageMapping {
  47. CanonicalLanguageID<StringIndexType> key {};
  48. CanonicalLanguageID<StringIndexType> alias {};
  49. };
  50. struct UnicodeLocaleData {
  51. UniqueStringStorage<StringIndexType> unique_strings;
  52. HashMap<String, Locale> locales;
  53. Vector<String> languages;
  54. Vector<String> territories;
  55. Vector<String> scripts;
  56. Vector<String> variants;
  57. Vector<String> currencies;
  58. Vector<String> keywords;
  59. Vector<String> list_pattern_types;
  60. Vector<String> list_pattern_styles;
  61. HashMap<String, StringIndexType> language_aliases;
  62. HashMap<String, StringIndexType> territory_aliases;
  63. HashMap<String, StringIndexType> script_aliases;
  64. HashMap<String, StringIndexType> variant_aliases;
  65. HashMap<String, StringIndexType> subdivision_aliases;
  66. Vector<LanguageMapping> complex_mappings;
  67. Vector<LanguageMapping> likely_subtags;
  68. size_t max_variant_size { 0 };
  69. };
  70. static Optional<LanguageMapping> parse_language_mapping(UnicodeLocaleData& locale_data, StringView key, StringView alias)
  71. {
  72. auto parsed_key = CanonicalLanguageID<StringIndexType>::parse(locale_data.unique_strings, key);
  73. if (!parsed_key.has_value())
  74. return {};
  75. auto parsed_alias = CanonicalLanguageID<StringIndexType>::parse(locale_data.unique_strings, alias);
  76. if (!parsed_alias.has_value())
  77. return {};
  78. return LanguageMapping { parsed_key.release_value(), parsed_alias.release_value() };
  79. }
  80. static void parse_core_aliases(String core_supplemental_path, UnicodeLocaleData& locale_data)
  81. {
  82. LexicalPath core_aliases_path(move(core_supplemental_path));
  83. core_aliases_path = core_aliases_path.append("aliases.json"sv);
  84. VERIFY(Core::File::exists(core_aliases_path.string()));
  85. auto core_aliases_file_or_error = Core::File::open(core_aliases_path.string(), Core::OpenMode::ReadOnly);
  86. VERIFY(!core_aliases_file_or_error.is_error());
  87. auto core_aliases = JsonValue::from_string(core_aliases_file_or_error.value()->read_all()).release_value_but_fixme_should_propagate_errors();
  88. auto const& supplemental_object = core_aliases.as_object().get("supplemental"sv);
  89. auto const& metadata_object = supplemental_object.as_object().get("metadata"sv);
  90. auto const& alias_object = metadata_object.as_object().get("alias"sv);
  91. auto append_aliases = [&](auto& alias_object, auto& alias_map) {
  92. alias_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  93. auto alias = value.as_object().get("_replacement"sv).as_string();
  94. if (key.contains('-')) {
  95. auto mapping = parse_language_mapping(locale_data, key, alias);
  96. if (!mapping.has_value())
  97. return;
  98. locale_data.max_variant_size = max(mapping->key.variants.size(), locale_data.max_variant_size);
  99. locale_data.max_variant_size = max(mapping->alias.variants.size(), locale_data.max_variant_size);
  100. locale_data.complex_mappings.append(mapping.release_value());
  101. } else {
  102. alias_map.set(key, locale_data.unique_strings.ensure(alias));
  103. }
  104. });
  105. };
  106. append_aliases(alias_object.as_object().get("languageAlias"sv), locale_data.language_aliases);
  107. append_aliases(alias_object.as_object().get("territoryAlias"sv), locale_data.territory_aliases);
  108. append_aliases(alias_object.as_object().get("scriptAlias"sv), locale_data.script_aliases);
  109. append_aliases(alias_object.as_object().get("variantAlias"sv), locale_data.variant_aliases);
  110. append_aliases(alias_object.as_object().get("subdivisionAlias"sv), locale_data.subdivision_aliases);
  111. }
  112. static void parse_likely_subtags(String core_supplemental_path, UnicodeLocaleData& locale_data)
  113. {
  114. LexicalPath likely_subtags_path(move(core_supplemental_path));
  115. likely_subtags_path = likely_subtags_path.append("likelySubtags.json"sv);
  116. VERIFY(Core::File::exists(likely_subtags_path.string()));
  117. auto likely_subtags_file_or_error = Core::File::open(likely_subtags_path.string(), Core::OpenMode::ReadOnly);
  118. VERIFY(!likely_subtags_file_or_error.is_error());
  119. auto likely_subtags = JsonValue::from_string(likely_subtags_file_or_error.value()->read_all()).release_value_but_fixme_should_propagate_errors();
  120. auto const& supplemental_object = likely_subtags.as_object().get("supplemental"sv);
  121. auto const& likely_subtags_object = supplemental_object.as_object().get("likelySubtags"sv);
  122. likely_subtags_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  123. auto mapping = parse_language_mapping(locale_data, key, value.as_string());
  124. if (!mapping.has_value())
  125. return;
  126. locale_data.max_variant_size = max(mapping->key.variants.size(), locale_data.max_variant_size);
  127. locale_data.max_variant_size = max(mapping->alias.variants.size(), locale_data.max_variant_size);
  128. locale_data.likely_subtags.append(mapping.release_value());
  129. });
  130. }
  131. static void parse_identity(String locale_path, UnicodeLocaleData& locale_data, Locale& locale)
  132. {
  133. LexicalPath languages_path(move(locale_path)); // Note: Every JSON file defines identity data, so we can use any of them.
  134. languages_path = languages_path.append("languages.json"sv);
  135. VERIFY(Core::File::exists(languages_path.string()));
  136. auto languages_file_or_error = Core::File::open(languages_path.string(), Core::OpenMode::ReadOnly);
  137. VERIFY(!languages_file_or_error.is_error());
  138. auto languages = JsonValue::from_string(languages_file_or_error.value()->read_all()).release_value_but_fixme_should_propagate_errors();
  139. auto const& main_object = languages.as_object().get("main"sv);
  140. auto const& locale_object = main_object.as_object().get(languages_path.parent().basename());
  141. auto const& identity_object = locale_object.as_object().get("identity"sv);
  142. auto const& language_string = identity_object.as_object().get("language"sv);
  143. auto const& territory_string = identity_object.as_object().get("territory"sv);
  144. auto const& variant_string = identity_object.as_object().get("variant"sv);
  145. locale.language = language_string.as_string();
  146. if (!locale_data.languages.contains_slow(locale.language))
  147. locale_data.languages.append(locale.language);
  148. if (territory_string.is_string()) {
  149. locale.territory = territory_string.as_string();
  150. if (!locale_data.territories.contains_slow(*locale.territory))
  151. locale_data.territories.append(*locale.territory);
  152. }
  153. if (variant_string.is_string()) {
  154. locale.variant = variant_string.as_string();
  155. if (!locale_data.variants.contains_slow(*locale.variant))
  156. locale_data.variants.append(*locale.variant);
  157. }
  158. }
  159. static void parse_locale_languages(String locale_path, UnicodeLocaleData& locale_data, Locale& locale)
  160. {
  161. LexicalPath languages_path(move(locale_path));
  162. languages_path = languages_path.append("languages.json"sv);
  163. VERIFY(Core::File::exists(languages_path.string()));
  164. auto languages_file_or_error = Core::File::open(languages_path.string(), Core::OpenMode::ReadOnly);
  165. VERIFY(!languages_file_or_error.is_error());
  166. auto languages = JsonValue::from_string(languages_file_or_error.value()->read_all()).release_value_but_fixme_should_propagate_errors();
  167. auto const& main_object = languages.as_object().get("main"sv);
  168. auto const& locale_object = main_object.as_object().get(languages_path.parent().basename());
  169. auto const& locale_display_names_object = locale_object.as_object().get("localeDisplayNames"sv);
  170. auto const& languages_object = locale_display_names_object.as_object().get("languages"sv);
  171. languages_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  172. if (!locale_data.languages.contains_slow(key))
  173. return;
  174. auto index = locale_data.unique_strings.ensure(value.as_string());
  175. locale.languages.set(key, index);
  176. });
  177. }
  178. static void parse_locale_territories(String locale_path, UnicodeLocaleData& locale_data, Locale& locale)
  179. {
  180. LexicalPath territories_path(move(locale_path));
  181. territories_path = territories_path.append("territories.json"sv);
  182. VERIFY(Core::File::exists(territories_path.string()));
  183. auto territories_file_or_error = Core::File::open(territories_path.string(), Core::OpenMode::ReadOnly);
  184. VERIFY(!territories_file_or_error.is_error());
  185. auto territories = JsonValue::from_string(territories_file_or_error.value()->read_all()).release_value_but_fixme_should_propagate_errors();
  186. auto const& main_object = territories.as_object().get("main"sv);
  187. auto const& locale_object = main_object.as_object().get(territories_path.parent().basename());
  188. auto const& locale_display_names_object = locale_object.as_object().get("localeDisplayNames"sv);
  189. auto const& territories_object = locale_display_names_object.as_object().get("territories"sv);
  190. territories_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  191. if (!locale_data.territories.contains_slow(key))
  192. return;
  193. auto index = locale_data.unique_strings.ensure(value.as_string());
  194. locale.territories.set(key, index);
  195. });
  196. }
  197. static void parse_locale_scripts(String locale_path, UnicodeLocaleData& locale_data, Locale& locale)
  198. {
  199. LexicalPath scripts_path(move(locale_path));
  200. scripts_path = scripts_path.append("scripts.json"sv);
  201. VERIFY(Core::File::exists(scripts_path.string()));
  202. auto scripts_file_or_error = Core::File::open(scripts_path.string(), Core::OpenMode::ReadOnly);
  203. VERIFY(!scripts_file_or_error.is_error());
  204. auto scripts = JsonValue::from_string(scripts_file_or_error.value()->read_all()).release_value_but_fixme_should_propagate_errors();
  205. auto const& main_object = scripts.as_object().get("main"sv);
  206. auto const& locale_object = main_object.as_object().get(scripts_path.parent().basename());
  207. auto const& locale_display_names_object = locale_object.as_object().get("localeDisplayNames"sv);
  208. auto const& scripts_object = locale_display_names_object.as_object().get("scripts"sv);
  209. scripts_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  210. auto index = locale_data.unique_strings.ensure(value.as_string());
  211. locale.scripts.set(key, index);
  212. if (!locale_data.scripts.contains_slow(key))
  213. locale_data.scripts.append(key);
  214. });
  215. }
  216. static void parse_locale_list_patterns(String misc_path, UnicodeLocaleData& locale_data, Locale& locale)
  217. {
  218. LexicalPath list_patterns_path(move(misc_path));
  219. list_patterns_path = list_patterns_path.append("listPatterns.json"sv);
  220. VERIFY(Core::File::exists(list_patterns_path.string()));
  221. auto list_patterns_file_or_error = Core::File::open(list_patterns_path.string(), Core::OpenMode::ReadOnly);
  222. VERIFY(!list_patterns_file_or_error.is_error());
  223. auto list_patterns = JsonValue::from_string(list_patterns_file_or_error.value()->read_all()).release_value_but_fixme_should_propagate_errors();
  224. auto const& main_object = list_patterns.as_object().get("main"sv);
  225. auto const& locale_object = main_object.as_object().get(list_patterns_path.parent().basename());
  226. auto const& list_patterns_object = locale_object.as_object().get("listPatterns"sv);
  227. auto list_pattern_type = [](StringView key) {
  228. if (key.contains("type-standard"sv))
  229. return "conjunction"sv;
  230. if (key.contains("type-or"sv))
  231. return "disjunction"sv;
  232. if (key.contains("type-unit"sv))
  233. return "unit"sv;
  234. VERIFY_NOT_REACHED();
  235. };
  236. auto list_pattern_style = [](StringView key) {
  237. if (key.contains("short"sv))
  238. return "short"sv;
  239. if (key.contains("narrow"sv))
  240. return "narrow"sv;
  241. return "long"sv;
  242. };
  243. list_patterns_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  244. auto type = list_pattern_type(key);
  245. auto style = list_pattern_style(key);
  246. auto start = locale_data.unique_strings.ensure(value.as_object().get("start"sv).as_string());
  247. auto middle = locale_data.unique_strings.ensure(value.as_object().get("middle"sv).as_string());
  248. auto end = locale_data.unique_strings.ensure(value.as_object().get("end"sv).as_string());
  249. auto pair = locale_data.unique_strings.ensure(value.as_object().get("2"sv).as_string());
  250. if (!locale_data.list_pattern_types.contains_slow(type))
  251. locale_data.list_pattern_types.append(type);
  252. if (!locale_data.list_pattern_styles.contains_slow(style))
  253. locale_data.list_pattern_styles.append(style);
  254. locale.list_patterns.append({ move(type), move(style), move(start), move(middle), move(end), move(pair) });
  255. });
  256. }
  257. static void parse_locale_currencies(String numbers_path, UnicodeLocaleData& locale_data, Locale& locale)
  258. {
  259. LexicalPath currencies_path(move(numbers_path));
  260. currencies_path = currencies_path.append("currencies.json"sv);
  261. VERIFY(Core::File::exists(currencies_path.string()));
  262. auto currencies_file_or_error = Core::File::open(currencies_path.string(), Core::OpenMode::ReadOnly);
  263. VERIFY(!currencies_file_or_error.is_error());
  264. auto currencies = JsonValue::from_string(currencies_file_or_error.value()->read_all()).release_value_but_fixme_should_propagate_errors();
  265. auto const& main_object = currencies.as_object().get("main"sv);
  266. auto const& locale_object = main_object.as_object().get(currencies_path.parent().basename());
  267. auto const& locale_numbers_object = locale_object.as_object().get("numbers"sv);
  268. auto const& currencies_object = locale_numbers_object.as_object().get("currencies"sv);
  269. currencies_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  270. auto const& long_name = value.as_object().get("displayName"sv);
  271. auto const& short_name = value.as_object().get("symbol"sv);
  272. auto const& narrow_name = value.as_object().get("symbol-alt-narrow"sv);
  273. auto const& numeric_name = value.as_object().get("displayName-count-other"sv);
  274. locale.long_currencies.set(key, locale_data.unique_strings.ensure(long_name.as_string()));
  275. locale.short_currencies.set(key, locale_data.unique_strings.ensure(short_name.as_string()));
  276. locale.narrow_currencies.set(key, narrow_name.is_null() ? 0 : locale_data.unique_strings.ensure(narrow_name.as_string()));
  277. locale.numeric_currencies.set(key, locale_data.unique_strings.ensure(numeric_name.is_null() ? long_name.as_string() : numeric_name.as_string()));
  278. if (!locale_data.currencies.contains_slow(key))
  279. locale_data.currencies.append(key);
  280. });
  281. }
  282. static void parse_numeric_keywords(String locale_numbers_path, UnicodeLocaleData& locale_data, Locale& locale)
  283. {
  284. static constexpr StringView key = "nu"sv;
  285. LexicalPath numbers_path(move(locale_numbers_path));
  286. numbers_path = numbers_path.append("numbers.json"sv);
  287. VERIFY(Core::File::exists(numbers_path.string()));
  288. auto numbers_file_or_error = Core::File::open(numbers_path.string(), Core::OpenMode::ReadOnly);
  289. VERIFY(!numbers_file_or_error.is_error());
  290. auto numbers = JsonValue::from_string(numbers_file_or_error.value()->read_all()).release_value_but_fixme_should_propagate_errors();
  291. auto const& main_object = numbers.as_object().get("main"sv);
  292. auto const& locale_object = main_object.as_object().get(numbers_path.parent().basename());
  293. auto const& locale_numbers_object = locale_object.as_object().get("numbers"sv);
  294. auto const& default_numbering_system_object = locale_numbers_object.as_object().get("defaultNumberingSystem"sv);
  295. auto const& other_numbering_systems_object = locale_numbers_object.as_object().get("otherNumberingSystems"sv);
  296. Vector<String> keyword_values {};
  297. keyword_values.append(default_numbering_system_object.as_string());
  298. other_numbering_systems_object.as_object().for_each_member([&](auto const&, JsonValue const& value) {
  299. auto keyword_value = value.as_string();
  300. if (!keyword_values.contains_slow(keyword_value))
  301. keyword_values.append(move(keyword_value));
  302. });
  303. locale_numbers_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  304. if (!key.starts_with("defaultNumberingSystem-alt-"sv))
  305. return;
  306. auto keyword_value = value.as_string();
  307. if (!keyword_values.contains_slow(keyword_value))
  308. keyword_values.append(move(keyword_value));
  309. });
  310. StringBuilder builder;
  311. builder.join(',', keyword_values);
  312. auto index = locale_data.unique_strings.ensure(builder.build());
  313. locale.keywords.set(key, index);
  314. if (!locale_data.keywords.contains_slow(key))
  315. locale_data.keywords.append(key);
  316. }
  317. static void parse_all_locales(String core_path, String locale_names_path, String misc_path, String numbers_path, UnicodeLocaleData& locale_data)
  318. {
  319. auto identity_iterator = path_to_dir_iterator(locale_names_path);
  320. auto locale_names_iterator = path_to_dir_iterator(move(locale_names_path));
  321. auto misc_iterator = path_to_dir_iterator(move(misc_path));
  322. auto numbers_iterator = path_to_dir_iterator(move(numbers_path));
  323. LexicalPath core_supplemental_path(core_path);
  324. core_supplemental_path = core_supplemental_path.append("supplemental"sv);
  325. VERIFY(Core::File::is_directory(core_supplemental_path.string()));
  326. parse_core_aliases(core_supplemental_path.string(), locale_data);
  327. parse_likely_subtags(core_supplemental_path.string(), locale_data);
  328. auto remove_variants_from_path = [&](String path) -> Optional<String> {
  329. auto parsed_locale = CanonicalLanguageID<StringIndexType>::parse(locale_data.unique_strings, LexicalPath::basename(path));
  330. if (!parsed_locale.has_value())
  331. return {};
  332. StringBuilder builder;
  333. builder.append(locale_data.unique_strings.get(parsed_locale->language));
  334. if (auto script = locale_data.unique_strings.get(parsed_locale->script); !script.is_empty())
  335. builder.appendff("-{}", script);
  336. if (auto region = locale_data.unique_strings.get(parsed_locale->region); !region.is_empty())
  337. builder.appendff("-{}", region);
  338. return builder.build();
  339. };
  340. while (identity_iterator.has_next()) {
  341. auto locale_path = identity_iterator.next_full_path();
  342. VERIFY(Core::File::is_directory(locale_path));
  343. auto language = remove_variants_from_path(locale_path);
  344. if (!language.has_value())
  345. continue;
  346. auto& locale = locale_data.locales.ensure(*language);
  347. parse_identity(locale_path, locale_data, locale);
  348. }
  349. while (locale_names_iterator.has_next()) {
  350. auto locale_path = locale_names_iterator.next_full_path();
  351. VERIFY(Core::File::is_directory(locale_path));
  352. auto language = remove_variants_from_path(locale_path);
  353. if (!language.has_value())
  354. continue;
  355. auto& locale = locale_data.locales.ensure(*language);
  356. parse_locale_languages(locale_path, locale_data, locale);
  357. parse_locale_territories(locale_path, locale_data, locale);
  358. parse_locale_scripts(locale_path, locale_data, locale);
  359. }
  360. while (misc_iterator.has_next()) {
  361. auto misc_path = misc_iterator.next_full_path();
  362. VERIFY(Core::File::is_directory(misc_path));
  363. auto language = remove_variants_from_path(misc_path);
  364. if (!language.has_value())
  365. continue;
  366. auto& locale = locale_data.locales.ensure(*language);
  367. parse_locale_list_patterns(misc_path, locale_data, locale);
  368. }
  369. while (numbers_iterator.has_next()) {
  370. auto numbers_path = numbers_iterator.next_full_path();
  371. VERIFY(Core::File::is_directory(numbers_path));
  372. auto language = remove_variants_from_path(numbers_path);
  373. if (!language.has_value())
  374. continue;
  375. auto& locale = locale_data.locales.ensure(*language);
  376. parse_locale_currencies(numbers_path, locale_data, locale);
  377. parse_numeric_keywords(numbers_path, locale_data, locale);
  378. }
  379. parse_default_content_locales(move(core_path), locale_data);
  380. }
  381. static String format_identifier(StringView owner, String identifier)
  382. {
  383. identifier = identifier.replace("-"sv, "_"sv, true);
  384. if (all_of(identifier, is_ascii_digit))
  385. return String::formatted("{}_{}", owner[0], identifier);
  386. if (is_ascii_lower_alpha(identifier[0]))
  387. return String::formatted("{:c}{}", to_ascii_uppercase(identifier[0]), identifier.substring_view(1));
  388. return identifier;
  389. }
  390. static void generate_unicode_locale_header(Core::File& file, UnicodeLocaleData& locale_data)
  391. {
  392. StringBuilder builder;
  393. SourceGenerator generator { builder };
  394. generator.append(R"~~~(
  395. #pragma once
  396. #include <AK/Optional.h>
  397. #include <AK/StringView.h>
  398. #include <AK/Types.h>
  399. #include <AK/Vector.h>
  400. #include <LibUnicode/Forward.h>
  401. namespace Unicode {
  402. )~~~");
  403. auto locales = locale_data.locales.keys();
  404. generate_enum(generator, format_identifier, "Locale"sv, "None"sv, locales);
  405. generate_enum(generator, format_identifier, "Language"sv, {}, locale_data.languages);
  406. generate_enum(generator, format_identifier, "Territory"sv, {}, locale_data.territories);
  407. generate_enum(generator, format_identifier, "ScriptTag"sv, {}, locale_data.scripts);
  408. generate_enum(generator, format_identifier, "Currency"sv, {}, locale_data.currencies);
  409. generate_enum(generator, format_identifier, "Key"sv, {}, locale_data.keywords);
  410. generate_enum(generator, format_identifier, "Variant"sv, {}, locale_data.variants);
  411. generate_enum(generator, format_identifier, "ListPatternType"sv, {}, locale_data.list_pattern_types);
  412. generate_enum(generator, format_identifier, "ListPatternStyle"sv, {}, locale_data.list_pattern_styles);
  413. generator.append(R"~~~(
  414. namespace Detail {
  415. Optional<Locale> locale_from_string(StringView locale);
  416. Optional<StringView> get_locale_language_mapping(StringView locale, StringView language);
  417. Optional<Language> language_from_string(StringView language);
  418. Optional<StringView> resolve_language_alias(StringView language);
  419. Optional<StringView> get_locale_territory_mapping(StringView locale, StringView territory);
  420. Optional<Territory> territory_from_string(StringView territory);
  421. Optional<StringView> resolve_territory_alias(StringView territory);
  422. Optional<StringView> get_locale_script_tag_mapping(StringView locale, StringView script_tag);
  423. Optional<ScriptTag> script_tag_from_string(StringView script_tag);
  424. Optional<StringView> resolve_script_tag_alias(StringView script_tag);
  425. Optional<StringView> get_locale_long_currency_mapping(StringView locale, StringView currency);
  426. Optional<StringView> get_locale_short_currency_mapping(StringView locale, StringView currency);
  427. Optional<StringView> get_locale_narrow_currency_mapping(StringView locale, StringView currency);
  428. Optional<StringView> get_locale_numeric_currency_mapping(StringView locale, StringView currency);
  429. Optional<Currency> currency_from_string(StringView currency);
  430. Optional<StringView> get_locale_key_mapping(StringView locale, StringView key);
  431. Optional<Key> key_from_string(StringView key);
  432. Optional<ListPatterns> get_locale_list_pattern_mapping(StringView locale, StringView list_pattern_type, StringView list_pattern_style);
  433. Optional<ListPatternType> list_pattern_type_from_string(StringView list_pattern_type);
  434. Optional<ListPatternStyle> list_pattern_style_from_string(StringView list_pattern_style);
  435. Optional<StringView> resolve_variant_alias(StringView variant);
  436. Optional<StringView> resolve_subdivision_alias(StringView subdivision);
  437. void resolve_complex_language_aliases(Unicode::LanguageID& language_id);
  438. Optional<Unicode::LanguageID> add_likely_subtags(Unicode::LanguageID const& language_id);
  439. Optional<String> resolve_most_likely_territory(Unicode::LanguageID const& language_id);
  440. }
  441. }
  442. )~~~");
  443. file.write(generator.as_string_view());
  444. }
  445. static void generate_unicode_locale_implementation(Core::File& file, UnicodeLocaleData& locale_data)
  446. {
  447. StringBuilder builder;
  448. SourceGenerator generator { builder };
  449. generator.set("string_index_type"sv, s_string_index_type);
  450. generator.set("locales_size"sv, String::number(locale_data.locales.size()));
  451. generator.set("territories_size", String::number(locale_data.territories.size()));
  452. generator.set("variants_size", String::number(locale_data.max_variant_size));
  453. generator.append(R"~~~(
  454. #include <AK/Array.h>
  455. #include <AK/BinarySearch.h>
  456. #include <AK/Span.h>
  457. #include <LibUnicode/Locale.h>
  458. #include <LibUnicode/UnicodeLocale.h>
  459. namespace Unicode::Detail {
  460. struct Patterns {
  461. ListPatternType type;
  462. ListPatternStyle style;
  463. @string_index_type@ start { 0 };
  464. @string_index_type@ middle { 0 };
  465. @string_index_type@ end { 0 };
  466. @string_index_type@ pair { 0 };
  467. };
  468. )~~~");
  469. locale_data.unique_strings.generate(generator);
  470. auto append_index = [&](auto index) {
  471. generator.append(String::formatted(", {}", index));
  472. };
  473. auto append_list_and_size = [&](auto const& list) {
  474. if (list.is_empty()) {
  475. generator.append(", {}, 0");
  476. return;
  477. }
  478. bool first = true;
  479. generator.append(", {");
  480. for (auto const& item : list) {
  481. generator.append(first ? " " : ", ");
  482. generator.append(String::number(item));
  483. first = false;
  484. }
  485. generator.append(String::formatted(" }}, {}", list.size()));
  486. };
  487. auto append_string_index_list = [&](String name, auto const& keys, auto const& mappings) {
  488. generator.set("name", name);
  489. generator.set("size", String::number(keys.size()));
  490. generator.append(R"~~~(
  491. static constexpr Array<@string_index_type@, @size@> @name@ { {
  492. )~~~");
  493. constexpr size_t max_values_per_row = 30;
  494. size_t values_in_current_row = 0;
  495. for (auto const& key : keys) {
  496. if (values_in_current_row++ > 0)
  497. generator.append(" ");
  498. if (auto it = mappings.find(key); it != mappings.end())
  499. generator.set("mapping"sv, String::number(it->value));
  500. else
  501. generator.set("mapping"sv, "0"sv);
  502. generator.append("@mapping@,");
  503. if (values_in_current_row == max_values_per_row) {
  504. values_in_current_row = 0;
  505. generator.append("\n ");
  506. }
  507. }
  508. generator.append(R"~~~(
  509. } };
  510. )~~~");
  511. };
  512. auto append_list_patterns = [&](StringView name, Vector<ListPatterns> const& list_patterns) {
  513. generator.set("name", name);
  514. generator.set("size", String::number(list_patterns.size()));
  515. generator.append(R"~~~(
  516. static constexpr Array<Patterns, @size@> @name@ { {)~~~");
  517. for (auto const& list_pattern : list_patterns) {
  518. generator.set("type"sv, String::formatted("ListPatternType::{}", format_identifier({}, list_pattern.type)));
  519. generator.set("style"sv, String::formatted("ListPatternStyle::{}", format_identifier({}, list_pattern.style)));
  520. generator.set("start"sv, String::number(list_pattern.start));
  521. generator.set("middle"sv, String::number(list_pattern.middle));
  522. generator.set("end"sv, String::number(list_pattern.end));
  523. generator.set("pair"sv, String::number(list_pattern.pair));
  524. generator.append(R"~~~(
  525. { @type@, @style@, @start@, @middle@, @end@, @pair@ },)~~~");
  526. }
  527. generator.append(R"~~~(
  528. } };
  529. )~~~");
  530. };
  531. generate_mapping(generator, locale_data.locales, s_string_index_type, "s_languages"sv, "s_languages_{}", [&](auto const& name, auto const& value) { append_string_index_list(name, locale_data.languages, value.languages); });
  532. generate_mapping(generator, locale_data.locales, s_string_index_type, "s_territories"sv, "s_territories_{}", [&](auto const& name, auto const& value) { append_string_index_list(name, locale_data.territories, value.territories); });
  533. generate_mapping(generator, locale_data.locales, s_string_index_type, "s_scripts"sv, "s_scripts_{}", [&](auto const& name, auto const& value) { append_string_index_list(name, locale_data.scripts, value.scripts); });
  534. generate_mapping(generator, locale_data.locales, s_string_index_type, "s_long_currencies"sv, "s_long_currencies_{}", [&](auto const& name, auto const& value) { append_string_index_list(name, locale_data.currencies, value.long_currencies); });
  535. generate_mapping(generator, locale_data.locales, s_string_index_type, "s_short_currencies"sv, "s_short_currencies_{}", [&](auto const& name, auto const& value) { append_string_index_list(name, locale_data.currencies, value.short_currencies); });
  536. generate_mapping(generator, locale_data.locales, s_string_index_type, "s_narrow_currencies"sv, "s_narrow_currencies_{}", [&](auto const& name, auto const& value) { append_string_index_list(name, locale_data.currencies, value.narrow_currencies); });
  537. generate_mapping(generator, locale_data.locales, s_string_index_type, "s_numeric_currencies"sv, "s_numeric_currencies_{}", [&](auto const& name, auto const& value) { append_string_index_list(name, locale_data.currencies, value.numeric_currencies); });
  538. generate_mapping(generator, locale_data.locales, s_string_index_type, "s_keywords"sv, "s_keywords_{}", [&](auto const& name, auto const& value) { append_string_index_list(name, locale_data.keywords, value.keywords); });
  539. generate_mapping(generator, locale_data.locales, "Patterns"sv, "s_list_patterns"sv, "s_list_patterns_{}", [&](auto const& name, auto const& value) { append_list_patterns(name, value.list_patterns); });
  540. generator.append(R"~~~(
  541. struct CanonicalLanguageID {
  542. Unicode::LanguageID to_unicode_language_id() const
  543. {
  544. Unicode::LanguageID language_id {};
  545. language_id.variants.ensure_capacity(variants_size);
  546. language_id.language = s_string_list[language];
  547. if (script != 0)
  548. language_id.script = s_string_list[script];
  549. if (region != 0)
  550. language_id.region = s_string_list[region];
  551. for (size_t i = 0; i < variants_size; ++i)
  552. language_id.variants.append(s_string_list[variants[i]]);
  553. return language_id;
  554. }
  555. bool matches_variants(Vector<String> const& other_variants) const {
  556. if (variants_size == 0)
  557. return true;
  558. if (other_variants.size() != variants_size)
  559. return false;
  560. for (size_t i = 0; i < variants_size; ++i) {
  561. if (s_string_list[variants[i]] != other_variants[i])
  562. return false;
  563. }
  564. return true;
  565. };
  566. @string_index_type@ language { 0 };
  567. @string_index_type@ script { 0 };
  568. @string_index_type@ region { 0 };
  569. Array<@string_index_type@, @variants_size@> variants {};
  570. size_t variants_size { 0 };
  571. };
  572. struct LanguageMapping {
  573. CanonicalLanguageID key;
  574. CanonicalLanguageID alias;
  575. };
  576. )~~~");
  577. auto append_complex_mapping = [&](StringView name, auto& mappings) {
  578. generator.set("size", String::number(mappings.size()));
  579. generator.set("name"sv, name);
  580. generator.append(R"~~~(
  581. static constexpr Array<LanguageMapping, @size@> s_@name@ { {
  582. )~~~");
  583. quick_sort(mappings, [&](auto const& lhs, auto const& rhs) {
  584. auto const& lhs_language = locale_data.unique_strings.get(lhs.key.language);
  585. auto const& rhs_language = locale_data.unique_strings.get(rhs.key.language);
  586. // Sort the keys such that "und" language tags are at the end, as those are less specific.
  587. if (lhs_language.starts_with("und"sv) && !rhs_language.starts_with("und"sv))
  588. return false;
  589. if (!lhs_language.starts_with("und"sv) && rhs_language.starts_with("und"sv))
  590. return true;
  591. return lhs_language < rhs_language;
  592. });
  593. for (auto const& mapping : mappings) {
  594. generator.set("language"sv, String::number(mapping.key.language));
  595. generator.append(" { { @language@");
  596. append_index(mapping.key.script);
  597. append_index(mapping.key.region);
  598. append_list_and_size(mapping.key.variants);
  599. generator.set("language"sv, String::number(mapping.alias.language));
  600. generator.append(" }, { @language@");
  601. append_index(mapping.alias.script);
  602. append_index(mapping.alias.region);
  603. append_list_and_size(mapping.alias.variants);
  604. generator.append(" } },\n");
  605. }
  606. generator.append("} };\n");
  607. };
  608. append_complex_mapping("complex_alias"sv, locale_data.complex_mappings);
  609. append_complex_mapping("likely_subtags"sv, locale_data.likely_subtags);
  610. generator.append(R"~~~(
  611. static LanguageMapping const* resolve_likely_subtag(Unicode::LanguageID const& language_id)
  612. {
  613. // https://unicode.org/reports/tr35/#Likely_Subtags
  614. enum class State {
  615. LanguageScriptRegion,
  616. LanguageRegion,
  617. LanguageScript,
  618. Language,
  619. UndScript,
  620. Done,
  621. };
  622. auto state = State::LanguageScriptRegion;
  623. while (state != State::Done) {
  624. Unicode::LanguageID search_key;
  625. switch (state) {
  626. case State::LanguageScriptRegion:
  627. state = State::LanguageRegion;
  628. if (!language_id.script.has_value() || !language_id.region.has_value())
  629. continue;
  630. search_key.language = *language_id.language;
  631. search_key.script = *language_id.script;
  632. search_key.region = *language_id.region;
  633. break;
  634. case State::LanguageRegion:
  635. state = State::LanguageScript;
  636. if (!language_id.region.has_value())
  637. continue;
  638. search_key.language = *language_id.language;
  639. search_key.region = *language_id.region;
  640. break;
  641. case State::LanguageScript:
  642. state = State::Language;
  643. if (!language_id.script.has_value())
  644. continue;
  645. search_key.language = *language_id.language;
  646. search_key.script = *language_id.script;
  647. break;
  648. case State::Language:
  649. state = State::UndScript;
  650. search_key.language = *language_id.language;
  651. break;
  652. case State::UndScript:
  653. state = State::Done;
  654. if (!language_id.script.has_value())
  655. continue;
  656. search_key.language = "und"sv;
  657. search_key.script = *language_id.script;
  658. break;
  659. default:
  660. VERIFY_NOT_REACHED();
  661. }
  662. for (auto const& map : s_likely_subtags) {
  663. auto const& key_language = s_string_list[map.key.language];
  664. auto const& key_script = s_string_list[map.key.script];
  665. auto const& key_region = s_string_list[map.key.region];
  666. if (key_language != search_key.language)
  667. continue;
  668. if (!key_script.is_empty() || search_key.script.has_value()) {
  669. if (key_script != search_key.script)
  670. continue;
  671. }
  672. if (!key_region.is_empty() || search_key.region.has_value()) {
  673. if (key_region != search_key.region)
  674. continue;
  675. }
  676. return &map;
  677. }
  678. }
  679. return nullptr;
  680. }
  681. )~~~");
  682. auto append_mapping_search = [&](StringView enum_snake, StringView from_string_name, StringView collection_name) {
  683. generator.set("enum_snake", enum_snake);
  684. generator.set("from_string_name", from_string_name);
  685. generator.set("collection_name", collection_name);
  686. generator.append(R"~~~(
  687. Optional<StringView> get_locale_@enum_snake@_mapping(StringView locale, StringView @enum_snake@)
  688. {
  689. auto locale_value = locale_from_string(locale);
  690. if (!locale_value.has_value())
  691. return {};
  692. auto @enum_snake@_value = @from_string_name@_from_string(@enum_snake@);
  693. if (!@enum_snake@_value.has_value())
  694. return {};
  695. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  696. auto @enum_snake@_index = to_underlying(*@enum_snake@_value);
  697. auto const& mappings = @collection_name@.at(locale_index);
  698. auto @enum_snake@_string_index = mappings.at(@enum_snake@_index);
  699. auto @enum_snake@_mapping = s_string_list.at(@enum_snake@_string_index);
  700. if (@enum_snake@_mapping.is_empty())
  701. return {};
  702. return @enum_snake@_mapping;
  703. }
  704. )~~~");
  705. };
  706. auto append_from_string = [&](StringView enum_title, StringView enum_snake, auto const& values) {
  707. HashValueMap<String> hashes;
  708. hashes.ensure_capacity(values.size());
  709. for (auto const& value : values)
  710. hashes.set(value.hash(), format_identifier(enum_title, value));
  711. generate_value_from_string(generator, "{}_from_string"sv, enum_title, enum_snake, move(hashes));
  712. };
  713. auto append_alias_search = [&](StringView enum_snake, auto const& aliases) {
  714. HashValueMap<StringIndexType> hashes;
  715. hashes.ensure_capacity(aliases.size());
  716. for (auto const& alias : aliases)
  717. hashes.set(alias.key.hash(), alias.value);
  718. generate_value_from_string(generator, "resolve_{}_alias"sv, s_string_index_type, enum_snake, move(hashes), "StringView"sv, "s_string_list[{}]"sv);
  719. };
  720. append_from_string("Locale"sv, "locale"sv, locale_data.locales.keys());
  721. append_mapping_search("language"sv, "language"sv, "s_languages"sv);
  722. append_from_string("Language"sv, "language"sv, locale_data.languages);
  723. append_alias_search("language"sv, locale_data.language_aliases);
  724. append_mapping_search("territory"sv, "territory"sv, "s_territories"sv);
  725. append_from_string("Territory"sv, "territory"sv, locale_data.territories);
  726. append_alias_search("territory"sv, locale_data.territory_aliases);
  727. append_mapping_search("script_tag"sv, "script_tag"sv, "s_scripts"sv);
  728. append_from_string("ScriptTag"sv, "script_tag"sv, locale_data.scripts);
  729. append_alias_search("script_tag"sv, locale_data.script_aliases);
  730. append_mapping_search("long_currency"sv, "currency"sv, "s_long_currencies"sv);
  731. append_mapping_search("short_currency"sv, "currency"sv, "s_short_currencies"sv);
  732. append_mapping_search("narrow_currency"sv, "currency"sv, "s_narrow_currencies"sv);
  733. append_mapping_search("numeric_currency"sv, "currency"sv, "s_numeric_currencies"sv);
  734. append_from_string("Currency"sv, "currency"sv, locale_data.currencies);
  735. append_mapping_search("key"sv, "key"sv, "s_keywords"sv);
  736. append_from_string("Key"sv, "key"sv, locale_data.keywords);
  737. append_alias_search("variant"sv, locale_data.variant_aliases);
  738. append_alias_search("subdivision"sv, locale_data.subdivision_aliases);
  739. append_from_string("ListPatternType"sv, "list_pattern_type"sv, locale_data.list_pattern_types);
  740. append_from_string("ListPatternStyle"sv, "list_pattern_style"sv, locale_data.list_pattern_styles);
  741. generator.append(R"~~~(
  742. Optional<ListPatterns> get_locale_list_pattern_mapping(StringView locale, StringView list_pattern_type, StringView list_pattern_style)
  743. {
  744. auto locale_value = locale_from_string(locale);
  745. if (!locale_value.has_value())
  746. return {};
  747. auto type_value = list_pattern_type_from_string(list_pattern_type);
  748. if (!type_value.has_value())
  749. return {};
  750. auto style_value = list_pattern_style_from_string(list_pattern_style);
  751. if (!style_value.has_value())
  752. return {};
  753. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  754. auto const& locale_list_patterns = s_list_patterns.at(locale_index);
  755. for (auto const& list_patterns : locale_list_patterns) {
  756. if ((list_patterns.type == type_value) && (list_patterns.style == style_value)) {
  757. auto const& start = s_string_list[list_patterns.start];
  758. auto const& middle = s_string_list[list_patterns.middle];
  759. auto const& end = s_string_list[list_patterns.end];
  760. auto const& pair = s_string_list[list_patterns.pair];
  761. return ListPatterns { start, middle, end, pair };
  762. }
  763. }
  764. return {};
  765. }
  766. void resolve_complex_language_aliases(Unicode::LanguageID& language_id)
  767. {
  768. for (auto const& map : s_complex_alias) {
  769. auto const& key_language = s_string_list[map.key.language];
  770. auto const& key_script = s_string_list[map.key.script];
  771. auto const& key_region = s_string_list[map.key.region];
  772. if ((key_language != language_id.language) && (key_language != "und"sv))
  773. continue;
  774. if (!key_script.is_empty() && (key_script != language_id.script))
  775. continue;
  776. if (!key_region.is_empty() && (key_region != language_id.region))
  777. continue;
  778. if (!map.key.matches_variants(language_id.variants))
  779. continue;
  780. auto alias = map.alias.to_unicode_language_id();
  781. if (alias.language == "und"sv)
  782. alias.language = move(language_id.language);
  783. if (key_script.is_empty() && !alias.script.has_value())
  784. alias.script = move(language_id.script);
  785. if (key_region.is_empty() && !alias.region.has_value())
  786. alias.region = move(language_id.region);
  787. if (map.key.variants_size == 0 && alias.variants.is_empty())
  788. alias.variants = move(language_id.variants);
  789. language_id = move(alias);
  790. break;
  791. }
  792. }
  793. Optional<Unicode::LanguageID> add_likely_subtags(Unicode::LanguageID const& language_id)
  794. {
  795. // https://www.unicode.org/reports/tr35/#Likely_Subtags
  796. auto const* likely_subtag = resolve_likely_subtag(language_id);
  797. if (likely_subtag == nullptr)
  798. return {};
  799. auto maximized = language_id;
  800. auto const& key_script = s_string_list[likely_subtag->key.script];
  801. auto const& key_region = s_string_list[likely_subtag->key.region];
  802. auto const& alias_language = s_string_list[likely_subtag->alias.language];
  803. auto const& alias_script = s_string_list[likely_subtag->alias.script];
  804. auto const& alias_region = s_string_list[likely_subtag->alias.region];
  805. if (maximized.language == "und"sv)
  806. maximized.language = alias_language;
  807. if (!maximized.script.has_value() || (!key_script.is_empty() && !alias_script.is_empty()))
  808. maximized.script = alias_script;
  809. if (!maximized.region.has_value() || (!key_region.is_empty() && !alias_region.is_empty()))
  810. maximized.region = alias_region;
  811. return maximized;
  812. }
  813. Optional<String> resolve_most_likely_territory(Unicode::LanguageID const& language_id)
  814. {
  815. if (auto const* likely_subtag = resolve_likely_subtag(language_id); likely_subtag != nullptr)
  816. return s_string_list[likely_subtag->alias.region];
  817. return {};
  818. }
  819. }
  820. )~~~");
  821. file.write(generator.as_string_view());
  822. }
  823. int main(int argc, char** argv)
  824. {
  825. char const* generated_header_path = nullptr;
  826. char const* generated_implementation_path = nullptr;
  827. char const* core_path = nullptr;
  828. char const* locale_names_path = nullptr;
  829. char const* misc_path = nullptr;
  830. char const* numbers_path = nullptr;
  831. Core::ArgsParser args_parser;
  832. args_parser.add_option(generated_header_path, "Path to the Unicode locale header file to generate", "generated-header-path", 'h', "generated-header-path");
  833. args_parser.add_option(generated_implementation_path, "Path to the Unicode locale implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
  834. args_parser.add_option(core_path, "Path to cldr-core directory", "core-path", 'r', "core-path");
  835. args_parser.add_option(locale_names_path, "Path to cldr-localenames directory", "locale-names-path", 'l', "locale-names-path");
  836. args_parser.add_option(misc_path, "Path to cldr-misc directory", "misc-path", 'm', "misc-path");
  837. args_parser.add_option(numbers_path, "Path to cldr-numbers directory", "numbers-path", 'n', "numbers-path");
  838. args_parser.parse(argc, argv);
  839. auto open_file = [&](StringView path, StringView flags, Core::OpenMode mode = Core::OpenMode::ReadOnly) {
  840. if (path.is_empty()) {
  841. warnln("{} is required", flags);
  842. args_parser.print_usage(stderr, argv[0]);
  843. exit(1);
  844. }
  845. auto file_or_error = Core::File::open(path, mode);
  846. if (file_or_error.is_error()) {
  847. warnln("Failed to open {}: {}", path, file_or_error.release_error());
  848. exit(1);
  849. }
  850. return file_or_error.release_value();
  851. };
  852. auto generated_header_file = open_file(generated_header_path, "-h/--generated-header-path", Core::OpenMode::ReadWrite);
  853. auto generated_implementation_file = open_file(generated_implementation_path, "-c/--generated-implementation-path", Core::OpenMode::ReadWrite);
  854. UnicodeLocaleData locale_data;
  855. parse_all_locales(core_path, locale_names_path, misc_path, numbers_path, locale_data);
  856. generate_unicode_locale_header(generated_header_file, locale_data);
  857. generate_unicode_locale_implementation(generated_implementation_file, locale_data);
  858. return 0;
  859. }