GenerateUnicodeLocale.cpp 72 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721
  1. /*
  2. * Copyright (c) 2021-2022, Tim Flynn <trflynn89@serenityos.org>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include "GeneratorUtil.h"
  7. #include <AK/AllOf.h>
  8. #include <AK/CharacterTypes.h>
  9. #include <AK/Format.h>
  10. #include <AK/HashMap.h>
  11. #include <AK/JsonObject.h>
  12. #include <AK/JsonParser.h>
  13. #include <AK/JsonValue.h>
  14. #include <AK/LexicalPath.h>
  15. #include <AK/QuickSort.h>
  16. #include <AK/SourceGenerator.h>
  17. #include <AK/String.h>
  18. #include <AK/StringBuilder.h>
  19. #include <LibCore/ArgsParser.h>
  20. #include <LibCore/DirIterator.h>
  21. #include <LibCore/File.h>
  22. #include <LibCore/Stream.h>
  23. using StringIndexType = u32;
  24. constexpr auto s_string_index_type = "u32"sv;
  25. using DisplayPatternIndexType = u8;
  26. constexpr auto s_display_pattern_index_type = "u8"sv;
  27. using LanguageListIndexType = u8;
  28. constexpr auto s_language_list_index_type = "u8"sv;
  29. using TerritoryListIndexType = u8;
  30. constexpr auto s_territory_list_index_type = "u8"sv;
  31. using ScriptListIndexType = u8;
  32. constexpr auto s_script_list_index_type = "u8"sv;
  33. using CurrencyListIndexType = u16;
  34. constexpr auto s_currency_list_index_type = "u16"sv;
  35. using CalendarListIndexType = u8;
  36. constexpr auto s_calendar_list_index_type = "u8"sv;
  37. using DateFieldListIndexType = u8;
  38. constexpr auto s_date_field_list_index_type = "u8"sv;
  39. using KeywordListIndexType = u8;
  40. constexpr auto s_keyword_list_index_type = "u8"sv;
  41. using ListPatternIndexType = u16;
  42. constexpr auto s_list_pattern_index_type = "u16"sv;
  43. using ListPatternListIndexType = u8;
  44. constexpr auto s_list_pattern_list_index_type = "u8"sv;
  45. using TextLayoutIndexType = u8;
  46. constexpr auto s_text_layout_index_type = "u8"sv;
  47. static String format_identifier(StringView owner, String identifier)
  48. {
  49. identifier = identifier.replace("-"sv, "_"sv, ReplaceMode::All);
  50. if (all_of(identifier, is_ascii_digit))
  51. return String::formatted("{}_{}", owner[0], identifier);
  52. if (is_ascii_lower_alpha(identifier[0]))
  53. return String::formatted("{:c}{}", to_ascii_uppercase(identifier[0]), identifier.substring_view(1));
  54. return identifier;
  55. }
  56. struct DisplayPattern {
  57. unsigned hash() const
  58. {
  59. return pair_int_hash(locale_pattern, locale_separator);
  60. }
  61. bool operator==(DisplayPattern const& other) const
  62. {
  63. return (locale_pattern == other.locale_pattern)
  64. && (locale_separator == other.locale_separator);
  65. }
  66. StringIndexType locale_pattern { 0 };
  67. StringIndexType locale_separator { 0 };
  68. };
  69. template<>
  70. struct AK::Formatter<DisplayPattern> : Formatter<FormatString> {
  71. ErrorOr<void> format(FormatBuilder& builder, DisplayPattern const& patterns)
  72. {
  73. return Formatter<FormatString>::format(builder,
  74. "{{ {}, {} }}",
  75. patterns.locale_pattern,
  76. patterns.locale_separator);
  77. }
  78. };
  79. template<>
  80. struct AK::Traits<DisplayPattern> : public GenericTraits<DisplayPattern> {
  81. static unsigned hash(DisplayPattern const& p) { return p.hash(); }
  82. };
  83. struct ListPatterns {
  84. unsigned hash() const
  85. {
  86. auto hash = pair_int_hash(type.hash(), style.hash());
  87. hash = pair_int_hash(hash, start);
  88. hash = pair_int_hash(hash, middle);
  89. hash = pair_int_hash(hash, end);
  90. hash = pair_int_hash(hash, pair);
  91. return hash;
  92. }
  93. bool operator==(ListPatterns const& other) const
  94. {
  95. return (type == other.type)
  96. && (style == other.style)
  97. && (start == other.start)
  98. && (middle == other.middle)
  99. && (end == other.end)
  100. && (pair == other.pair);
  101. }
  102. StringView type;
  103. StringView style;
  104. StringIndexType start { 0 };
  105. StringIndexType middle { 0 };
  106. StringIndexType end { 0 };
  107. StringIndexType pair { 0 };
  108. };
  109. template<>
  110. struct AK::Formatter<ListPatterns> : Formatter<FormatString> {
  111. ErrorOr<void> format(FormatBuilder& builder, ListPatterns const& patterns)
  112. {
  113. return Formatter<FormatString>::format(builder,
  114. "{{ ListPatternType::{}, Style::{}, {}, {}, {}, {} }}",
  115. format_identifier({}, patterns.type),
  116. format_identifier({}, patterns.style),
  117. patterns.start,
  118. patterns.middle,
  119. patterns.end,
  120. patterns.pair);
  121. }
  122. };
  123. template<>
  124. struct AK::Traits<ListPatterns> : public GenericTraits<ListPatterns> {
  125. static unsigned hash(ListPatterns const& p) { return p.hash(); }
  126. };
  127. struct TextLayout {
  128. unsigned hash() const
  129. {
  130. return character_order.hash();
  131. }
  132. bool operator==(TextLayout const& other) const
  133. {
  134. return character_order == other.character_order;
  135. }
  136. StringView character_order;
  137. };
  138. template<>
  139. struct AK::Formatter<TextLayout> : Formatter<FormatString> {
  140. ErrorOr<void> format(FormatBuilder& builder, TextLayout const& patterns)
  141. {
  142. return Formatter<FormatString>::format(builder,
  143. "{{ CharacterOrder::{} }}",
  144. format_identifier({}, patterns.character_order));
  145. }
  146. };
  147. template<>
  148. struct AK::Traits<TextLayout> : public GenericTraits<TextLayout> {
  149. static unsigned hash(TextLayout const& t) { return t.hash(); }
  150. };
  151. using LanguageList = Vector<StringIndexType>;
  152. using TerritoryList = Vector<StringIndexType>;
  153. using ScriptList = Vector<StringIndexType>;
  154. using CurrencyList = Vector<StringIndexType>;
  155. using CalendarList = Vector<StringIndexType>;
  156. using DateFieldList = Vector<StringIndexType>;
  157. using KeywordList = Vector<StringIndexType>;
  158. using ListPatternList = Vector<ListPatternIndexType>;
  159. struct Locale {
  160. String language;
  161. Optional<String> territory;
  162. Optional<String> variant;
  163. DisplayPatternIndexType display_patterns { 0 };
  164. LanguageListIndexType languages { 0 };
  165. TerritoryListIndexType territories { 0 };
  166. ScriptListIndexType scripts { 0 };
  167. CurrencyListIndexType long_currencies { 0 };
  168. CurrencyListIndexType short_currencies { 0 };
  169. CurrencyListIndexType narrow_currencies { 0 };
  170. CurrencyListIndexType numeric_currencies { 0 };
  171. CalendarListIndexType calendars { 0 };
  172. DateFieldListIndexType long_date_fields { 0 };
  173. DateFieldListIndexType short_date_fields { 0 };
  174. DateFieldListIndexType narrow_date_fields { 0 };
  175. KeywordListIndexType calendar_keywords { 0 };
  176. KeywordListIndexType collation_case_keywords { 0 };
  177. KeywordListIndexType collation_numeric_keywords { 0 };
  178. KeywordListIndexType number_system_keywords { 0 };
  179. ListPatternListIndexType list_patterns { 0 };
  180. TextLayoutIndexType text_layout { 0 };
  181. };
  182. struct LanguageMapping {
  183. CanonicalLanguageID<StringIndexType> key {};
  184. CanonicalLanguageID<StringIndexType> alias {};
  185. };
  186. struct UnicodeLocaleData {
  187. UniqueStringStorage<StringIndexType> unique_strings;
  188. UniqueStorage<DisplayPattern, DisplayPatternIndexType> unique_display_patterns;
  189. UniqueStorage<LanguageList, LanguageListIndexType> unique_language_lists;
  190. UniqueStorage<TerritoryList, TerritoryListIndexType> unique_territory_lists;
  191. UniqueStorage<ScriptList, ScriptListIndexType> unique_script_lists;
  192. UniqueStorage<CurrencyList, CurrencyListIndexType> unique_currency_lists;
  193. UniqueStorage<CalendarList, CalendarListIndexType> unique_calendar_lists;
  194. UniqueStorage<DateFieldList, DateFieldListIndexType> unique_date_field_lists;
  195. UniqueStorage<KeywordList, KeywordListIndexType> unique_keyword_lists;
  196. UniqueStorage<ListPatterns, ListPatternIndexType> unique_list_patterns;
  197. UniqueStorage<ListPatternList, ListPatternListIndexType> unique_list_pattern_lists;
  198. UniqueStorage<TextLayout, TextLayoutIndexType> unique_text_layouts;
  199. HashMap<String, Locale> locales;
  200. Vector<Alias> locale_aliases;
  201. Vector<String> languages;
  202. Vector<String> territories;
  203. Vector<String> scripts;
  204. Vector<String> variants;
  205. Vector<String> currencies;
  206. Vector<String> date_fields;
  207. Vector<Alias> date_field_aliases {
  208. // ECMA-402 and the CLDR refer to some date fields with different names. Defining these aliases
  209. // means we can remain agnostic about the naming differences elsewhere.
  210. { "dayperiod"sv, "dayPeriod"sv },
  211. { "week"sv, "weekOfYear"sv },
  212. { "zone"sv, "timeZoneName"sv },
  213. };
  214. HashMap<String, Vector<String>> keywords;
  215. HashMap<String, Vector<Alias>> keyword_aliases;
  216. HashMap<String, String> keyword_names;
  217. Vector<String> list_pattern_types;
  218. Vector<String> character_orders;
  219. HashMap<String, StringIndexType> language_aliases;
  220. HashMap<String, StringIndexType> territory_aliases;
  221. HashMap<String, StringIndexType> script_aliases;
  222. HashMap<String, StringIndexType> variant_aliases;
  223. HashMap<String, StringIndexType> subdivision_aliases;
  224. Vector<LanguageMapping> complex_mappings;
  225. Vector<LanguageMapping> likely_subtags;
  226. size_t max_variant_size { 0 };
  227. };
  228. // Some parsing is expected to fail. For example, the CLDR contains language mappings
  229. // with locales such as "en-GB-oed" that are canonically invalid locale IDs.
  230. #define TRY_OR_DISCARD(expression) \
  231. ({ \
  232. auto _temporary_result = (expression); \
  233. if (_temporary_result.is_error()) \
  234. return; \
  235. _temporary_result.release_value(); \
  236. })
  237. static ErrorOr<LanguageMapping> parse_language_mapping(UnicodeLocaleData& locale_data, StringView key, StringView alias)
  238. {
  239. auto parsed_key = TRY(CanonicalLanguageID<StringIndexType>::parse(locale_data.unique_strings, key));
  240. auto parsed_alias = TRY(CanonicalLanguageID<StringIndexType>::parse(locale_data.unique_strings, alias));
  241. return LanguageMapping { move(parsed_key), move(parsed_alias) };
  242. }
  243. static ErrorOr<void> parse_core_aliases(String core_supplemental_path, UnicodeLocaleData& locale_data)
  244. {
  245. LexicalPath core_aliases_path(move(core_supplemental_path));
  246. core_aliases_path = core_aliases_path.append("aliases.json"sv);
  247. auto core_aliases = TRY(read_json_file(core_aliases_path.string()));
  248. auto const& supplemental_object = core_aliases.as_object().get("supplemental"sv);
  249. auto const& metadata_object = supplemental_object.as_object().get("metadata"sv);
  250. auto const& alias_object = metadata_object.as_object().get("alias"sv);
  251. auto append_aliases = [&](auto& alias_object, auto& alias_map) {
  252. alias_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  253. auto alias = value.as_object().get("_replacement"sv).as_string();
  254. if (key.contains('-')) {
  255. auto mapping = TRY_OR_DISCARD(parse_language_mapping(locale_data, key, alias));
  256. locale_data.max_variant_size = max(mapping.key.variants.size(), locale_data.max_variant_size);
  257. locale_data.max_variant_size = max(mapping.alias.variants.size(), locale_data.max_variant_size);
  258. locale_data.complex_mappings.append(move(mapping));
  259. } else {
  260. alias_map.set(key, locale_data.unique_strings.ensure(alias));
  261. }
  262. });
  263. };
  264. append_aliases(alias_object.as_object().get("languageAlias"sv), locale_data.language_aliases);
  265. append_aliases(alias_object.as_object().get("territoryAlias"sv), locale_data.territory_aliases);
  266. append_aliases(alias_object.as_object().get("scriptAlias"sv), locale_data.script_aliases);
  267. append_aliases(alias_object.as_object().get("variantAlias"sv), locale_data.variant_aliases);
  268. append_aliases(alias_object.as_object().get("subdivisionAlias"sv), locale_data.subdivision_aliases);
  269. return {};
  270. }
  271. static ErrorOr<void> parse_likely_subtags(String core_supplemental_path, UnicodeLocaleData& locale_data)
  272. {
  273. LexicalPath likely_subtags_path(move(core_supplemental_path));
  274. likely_subtags_path = likely_subtags_path.append("likelySubtags.json"sv);
  275. auto likely_subtags = TRY(read_json_file(likely_subtags_path.string()));
  276. auto const& supplemental_object = likely_subtags.as_object().get("supplemental"sv);
  277. auto const& likely_subtags_object = supplemental_object.as_object().get("likelySubtags"sv);
  278. likely_subtags_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  279. auto mapping = TRY_OR_DISCARD(parse_language_mapping(locale_data, key, value.as_string()));
  280. locale_data.max_variant_size = max(mapping.key.variants.size(), locale_data.max_variant_size);
  281. locale_data.max_variant_size = max(mapping.alias.variants.size(), locale_data.max_variant_size);
  282. locale_data.likely_subtags.append(move(mapping));
  283. });
  284. return {};
  285. }
  286. static ErrorOr<void> parse_identity(String locale_path, UnicodeLocaleData& locale_data, Locale& locale)
  287. {
  288. LexicalPath languages_path(move(locale_path)); // Note: Every JSON file defines identity data, so we can use any of them.
  289. languages_path = languages_path.append("languages.json"sv);
  290. auto languages = TRY(read_json_file(languages_path.string()));
  291. auto const& main_object = languages.as_object().get("main"sv);
  292. auto const& locale_object = main_object.as_object().get(languages_path.parent().basename());
  293. auto const& identity_object = locale_object.as_object().get("identity"sv);
  294. auto const& language_string = identity_object.as_object().get("language"sv);
  295. auto const& territory_string = identity_object.as_object().get("territory"sv);
  296. auto const& script_string = identity_object.as_object().get("script"sv);
  297. auto const& variant_string = identity_object.as_object().get("variant"sv);
  298. locale.language = language_string.as_string();
  299. if (territory_string.is_string()) {
  300. locale.territory = territory_string.as_string();
  301. if (!locale_data.territories.contains_slow(*locale.territory))
  302. locale_data.territories.append(*locale.territory);
  303. }
  304. if (script_string.is_string()) {
  305. auto script = script_string.as_string();
  306. if (!locale_data.scripts.contains_slow(script))
  307. locale_data.scripts.append(script);
  308. }
  309. if (variant_string.is_string()) {
  310. locale.variant = variant_string.as_string();
  311. if (!locale_data.variants.contains_slow(*locale.variant))
  312. locale_data.variants.append(*locale.variant);
  313. }
  314. return {};
  315. }
  316. static ErrorOr<void> parse_locale_display_patterns(String locale_path, UnicodeLocaleData& locale_data, Locale& locale)
  317. {
  318. LexicalPath locale_display_names_path(move(locale_path));
  319. locale_display_names_path = locale_display_names_path.append("localeDisplayNames.json"sv);
  320. auto locale_display_names = TRY(read_json_file(locale_display_names_path.string()));
  321. auto const& main_object = locale_display_names.as_object().get("main"sv);
  322. auto const& locale_object = main_object.as_object().get(locale_display_names_path.parent().basename());
  323. auto const& locale_display_names_object = locale_object.as_object().get("localeDisplayNames"sv);
  324. auto const& locale_display_patterns_object = locale_display_names_object.as_object().get("localeDisplayPattern"sv);
  325. auto const& locale_pattern = locale_display_patterns_object.as_object().get("localePattern"sv);
  326. auto const& locale_separator = locale_display_patterns_object.as_object().get("localeSeparator"sv);
  327. DisplayPattern patterns {};
  328. patterns.locale_pattern = locale_data.unique_strings.ensure(locale_pattern.as_string());
  329. patterns.locale_separator = locale_data.unique_strings.ensure(locale_separator.as_string());
  330. locale.display_patterns = locale_data.unique_display_patterns.ensure(move(patterns));
  331. return {};
  332. }
  333. static ErrorOr<void> preprocess_languages(String locale_path, UnicodeLocaleData& locale_data)
  334. {
  335. LexicalPath languages_path(move(locale_path));
  336. languages_path = languages_path.append("languages.json"sv);
  337. auto locale_languages = TRY(read_json_file(languages_path.string()));
  338. auto const& main_object = locale_languages.as_object().get("main"sv);
  339. auto const& locale_object = main_object.as_object().get(languages_path.parent().basename());
  340. auto const& locale_display_names_object = locale_object.as_object().get("localeDisplayNames"sv);
  341. auto const& languages_object = locale_display_names_object.as_object().get("languages"sv);
  342. languages_object.as_object().for_each_member([&](auto const& key, auto const&) {
  343. if (!key.contains("-alt-"sv) && !locale_data.languages.contains_slow(key))
  344. locale_data.languages.append(key);
  345. });
  346. return {};
  347. }
  348. static ErrorOr<void> parse_unicode_extension_keywords(String bcp47_path, UnicodeLocaleData& locale_data)
  349. {
  350. constexpr auto desired_keywords = Array { "ca"sv, "kf"sv, "kn"sv, "nu"sv };
  351. auto keywords = TRY(read_json_file(bcp47_path));
  352. auto const& keyword_object = keywords.as_object().get("keyword"sv);
  353. auto const& unicode_object = keyword_object.as_object().get("u"sv);
  354. if (unicode_object.is_null())
  355. return {};
  356. unicode_object.as_object().for_each_member([&](auto const& key, auto const& value) {
  357. if (!desired_keywords.span().contains_slow(key))
  358. return;
  359. auto const& name = value.as_object().get("_alias");
  360. locale_data.keyword_names.set(key, name.as_string());
  361. auto& keywords = locale_data.keywords.ensure(key);
  362. value.as_object().for_each_member([&](auto const& keyword, auto const& properties) {
  363. if (!properties.is_object())
  364. return;
  365. if (auto const& preferred = properties.as_object().get("_preferred"sv); preferred.is_string()) {
  366. locale_data.keyword_aliases.ensure(key).append({ preferred.as_string(), keyword });
  367. return;
  368. }
  369. if (auto const& alias = properties.as_object().get("_alias"sv); alias.is_string())
  370. locale_data.keyword_aliases.ensure(key).append({ keyword, alias.as_string() });
  371. keywords.append(keyword);
  372. });
  373. });
  374. return {};
  375. }
  376. static Optional<String> find_keyword_alias(StringView key, StringView calendar, UnicodeLocaleData& locale_data)
  377. {
  378. auto it = locale_data.keyword_aliases.find(key);
  379. if (it == locale_data.keyword_aliases.end())
  380. return {};
  381. auto alias = it->value.find_if([&](auto const& alias) { return calendar == alias.alias; });
  382. if (alias == it->value.end())
  383. return {};
  384. return alias->name;
  385. }
  386. static ErrorOr<void> parse_locale_languages(String locale_path, UnicodeLocaleData& locale_data, Locale& locale)
  387. {
  388. LexicalPath languages_path(move(locale_path));
  389. languages_path = languages_path.append("languages.json"sv);
  390. auto locale_languages = TRY(read_json_file(languages_path.string()));
  391. auto const& main_object = locale_languages.as_object().get("main"sv);
  392. auto const& locale_object = main_object.as_object().get(languages_path.parent().basename());
  393. auto const& locale_display_names_object = locale_object.as_object().get("localeDisplayNames"sv);
  394. auto const& languages_object = locale_display_names_object.as_object().get("languages"sv);
  395. LanguageList languages;
  396. languages.resize(locale_data.languages.size());
  397. languages_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  398. if (key.contains("-alt-"sv))
  399. return;
  400. auto index = locale_data.languages.find_first_index(key).value();
  401. languages[index] = locale_data.unique_strings.ensure(value.as_string());
  402. });
  403. locale.languages = locale_data.unique_language_lists.ensure(move(languages));
  404. return {};
  405. }
  406. static ErrorOr<void> parse_locale_territories(String locale_path, UnicodeLocaleData& locale_data, Locale& locale)
  407. {
  408. LexicalPath territories_path(move(locale_path));
  409. territories_path = territories_path.append("territories.json"sv);
  410. auto locale_territories = TRY(read_json_file(territories_path.string()));
  411. auto const& main_object = locale_territories.as_object().get("main"sv);
  412. auto const& locale_object = main_object.as_object().get(territories_path.parent().basename());
  413. auto const& locale_display_names_object = locale_object.as_object().get("localeDisplayNames"sv);
  414. auto const& territories_object = locale_display_names_object.as_object().get("territories"sv);
  415. TerritoryList territories;
  416. territories.resize(locale_data.territories.size());
  417. territories_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  418. if (auto index = locale_data.territories.find_first_index(key); index.has_value())
  419. territories[*index] = locale_data.unique_strings.ensure(value.as_string());
  420. });
  421. locale.territories = locale_data.unique_territory_lists.ensure(move(territories));
  422. return {};
  423. }
  424. static ErrorOr<void> parse_locale_scripts(String locale_path, UnicodeLocaleData& locale_data, Locale& locale)
  425. {
  426. LexicalPath scripts_path(move(locale_path));
  427. scripts_path = scripts_path.append("scripts.json"sv);
  428. auto locale_scripts = TRY(read_json_file(scripts_path.string()));
  429. auto const& main_object = locale_scripts.as_object().get("main"sv);
  430. auto const& locale_object = main_object.as_object().get(scripts_path.parent().basename());
  431. auto const& locale_display_names_object = locale_object.as_object().get("localeDisplayNames"sv);
  432. auto const& scripts_object = locale_display_names_object.as_object().get("scripts"sv);
  433. ScriptList scripts;
  434. scripts.resize(locale_data.scripts.size());
  435. scripts_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  436. if (auto index = locale_data.scripts.find_first_index(key); index.has_value())
  437. scripts[*index] = locale_data.unique_strings.ensure(value.as_string());
  438. });
  439. locale.scripts = locale_data.unique_script_lists.ensure(move(scripts));
  440. return {};
  441. }
  442. static ErrorOr<void> parse_locale_list_patterns(String misc_path, UnicodeLocaleData& locale_data, Locale& locale)
  443. {
  444. LexicalPath list_patterns_path(move(misc_path));
  445. list_patterns_path = list_patterns_path.append("listPatterns.json"sv);
  446. auto locale_list_patterns = TRY(read_json_file(list_patterns_path.string()));
  447. auto const& main_object = locale_list_patterns.as_object().get("main"sv);
  448. auto const& locale_object = main_object.as_object().get(list_patterns_path.parent().basename());
  449. auto const& list_patterns_object = locale_object.as_object().get("listPatterns"sv);
  450. auto list_pattern_type = [](StringView key) {
  451. if (key.contains("type-standard"sv))
  452. return "conjunction"sv;
  453. if (key.contains("type-or"sv))
  454. return "disjunction"sv;
  455. if (key.contains("type-unit"sv))
  456. return "unit"sv;
  457. VERIFY_NOT_REACHED();
  458. };
  459. auto list_pattern_style = [](StringView key) {
  460. if (key.contains("short"sv))
  461. return "short"sv;
  462. if (key.contains("narrow"sv))
  463. return "narrow"sv;
  464. return "long"sv;
  465. };
  466. ListPatternList list_patterns;
  467. list_patterns.ensure_capacity(list_patterns_object.as_object().size());
  468. list_patterns_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  469. auto type = list_pattern_type(key);
  470. auto style = list_pattern_style(key);
  471. auto start = locale_data.unique_strings.ensure(value.as_object().get("start"sv).as_string());
  472. auto middle = locale_data.unique_strings.ensure(value.as_object().get("middle"sv).as_string());
  473. auto end = locale_data.unique_strings.ensure(value.as_object().get("end"sv).as_string());
  474. auto pair = locale_data.unique_strings.ensure(value.as_object().get("2"sv).as_string());
  475. if (!locale_data.list_pattern_types.contains_slow(type))
  476. locale_data.list_pattern_types.append(type);
  477. ListPatterns list_pattern { type, style, start, middle, end, pair };
  478. list_patterns.append(locale_data.unique_list_patterns.ensure(move(list_pattern)));
  479. });
  480. locale.list_patterns = locale_data.unique_list_pattern_lists.ensure(move(list_patterns));
  481. return {};
  482. }
  483. static ErrorOr<void> parse_locale_layout(String misc_path, UnicodeLocaleData& locale_data, Locale& locale)
  484. {
  485. LexicalPath layout_path(move(misc_path));
  486. layout_path = layout_path.append("layout.json"sv);
  487. auto locale_layout = TRY(read_json_file(layout_path.string()));
  488. auto const& main_object = locale_layout.as_object().get("main"sv);
  489. auto const& locale_object = main_object.as_object().get(layout_path.parent().basename());
  490. auto const& layout_object = locale_object.as_object().get("layout"sv);
  491. auto const& orientation_object = layout_object.as_object().get("orientation"sv);
  492. auto text_layout_character_order = [](StringView key) {
  493. if (key == "left-to-right"sv)
  494. return "ltr"sv;
  495. if (key == "right-to-left"sv)
  496. return "rtl"sv;
  497. VERIFY_NOT_REACHED();
  498. };
  499. auto const& character_order_string = orientation_object.as_object().get("characterOrder"sv);
  500. auto const& character_order = character_order_string.as_string();
  501. TextLayout layout {};
  502. layout.character_order = text_layout_character_order(character_order);
  503. if (!locale_data.character_orders.contains_slow(layout.character_order))
  504. locale_data.character_orders.append(layout.character_order);
  505. locale.text_layout = locale_data.unique_text_layouts.ensure(move(layout));
  506. return {};
  507. }
  508. static ErrorOr<void> parse_locale_currencies(String numbers_path, UnicodeLocaleData& locale_data, Locale& locale)
  509. {
  510. LexicalPath currencies_path(move(numbers_path));
  511. currencies_path = currencies_path.append("currencies.json"sv);
  512. auto locale_currencies = TRY(read_json_file(currencies_path.string()));
  513. auto const& main_object = locale_currencies.as_object().get("main"sv);
  514. auto const& locale_object = main_object.as_object().get(currencies_path.parent().basename());
  515. auto const& locale_numbers_object = locale_object.as_object().get("numbers"sv);
  516. auto const& currencies_object = locale_numbers_object.as_object().get("currencies"sv);
  517. currencies_object.as_object().for_each_member([&](auto const& key, JsonValue const&) {
  518. if (!locale_data.currencies.contains_slow(key))
  519. locale_data.currencies.append(key);
  520. });
  521. CurrencyList long_currencies {};
  522. long_currencies.resize(locale_data.currencies.size());
  523. CurrencyList short_currencies {};
  524. short_currencies.resize(locale_data.currencies.size());
  525. CurrencyList narrow_currencies {};
  526. narrow_currencies.resize(locale_data.currencies.size());
  527. CurrencyList numeric_currencies {};
  528. numeric_currencies.resize(locale_data.currencies.size());
  529. currencies_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  530. auto const& long_name = value.as_object().get("displayName"sv);
  531. auto const& short_name = value.as_object().get("symbol"sv);
  532. auto const& narrow_name = value.as_object().get("symbol-alt-narrow"sv);
  533. auto const& numeric_name = value.as_object().get("displayName-count-other"sv);
  534. auto index = locale_data.currencies.find_first_index(key).value();
  535. long_currencies[index] = locale_data.unique_strings.ensure(long_name.as_string());
  536. short_currencies[index] = locale_data.unique_strings.ensure(short_name.as_string());
  537. narrow_currencies[index] = narrow_name.is_null() ? 0 : locale_data.unique_strings.ensure(narrow_name.as_string());
  538. numeric_currencies[index] = locale_data.unique_strings.ensure(numeric_name.is_null() ? long_name.as_string() : numeric_name.as_string());
  539. });
  540. locale.long_currencies = locale_data.unique_currency_lists.ensure(move(long_currencies));
  541. locale.short_currencies = locale_data.unique_currency_lists.ensure(move(short_currencies));
  542. locale.narrow_currencies = locale_data.unique_currency_lists.ensure(move(narrow_currencies));
  543. locale.numeric_currencies = locale_data.unique_currency_lists.ensure(move(numeric_currencies));
  544. return {};
  545. }
  546. static ErrorOr<void> parse_locale_calendars(String locale_path, UnicodeLocaleData& locale_data, Locale& locale)
  547. {
  548. LexicalPath locale_display_names_path(move(locale_path));
  549. locale_display_names_path = locale_display_names_path.append("localeDisplayNames.json"sv);
  550. auto locale_display_names = TRY(read_json_file(locale_display_names_path.string()));
  551. auto const& main_object = locale_display_names.as_object().get("main"sv);
  552. auto const& locale_object = main_object.as_object().get(locale_display_names_path.parent().basename());
  553. auto const& locale_display_names_object = locale_object.as_object().get("localeDisplayNames"sv);
  554. auto const& types_object = locale_display_names_object.as_object().get("types"sv);
  555. auto const& calendar_object = types_object.as_object().get("calendar"sv);
  556. auto const& supported_calendars = locale_data.keywords.find("ca"sv)->value;
  557. CalendarList calendars;
  558. calendars.resize(supported_calendars.size());
  559. calendar_object.as_object().for_each_member([&](auto const& key, auto const& calendar) {
  560. auto index = supported_calendars.find_first_index(key);
  561. if (!index.has_value()) {
  562. auto alias = find_keyword_alias("ca"sv, key, locale_data);
  563. index = supported_calendars.find_first_index(*alias);
  564. }
  565. calendars[*index] = locale_data.unique_strings.ensure(calendar.as_string());
  566. });
  567. locale.calendars = locale_data.unique_calendar_lists.ensure(move(calendars));
  568. return {};
  569. }
  570. static ErrorOr<void> parse_locale_date_fields(String dates_path, UnicodeLocaleData& locale_data, Locale& locale)
  571. {
  572. LexicalPath date_fields_path(move(dates_path));
  573. date_fields_path = date_fields_path.append("dateFields.json"sv);
  574. auto locale_date_fields = TRY(read_json_file(date_fields_path.string()));
  575. auto const& main_object = locale_date_fields.as_object().get("main"sv);
  576. auto const& locale_object = main_object.as_object().get(date_fields_path.parent().basename());
  577. auto const& dates_object = locale_object.as_object().get("dates"sv);
  578. auto const& fields_object = dates_object.as_object().get("fields"sv);
  579. auto is_sanctioned_field = [](StringView field) {
  580. // This is a copy of the units sanctioned for use within ECMA-402, with names adjusted for the names used by the CLDR.
  581. // https://tc39.es/ecma402/#table-validcodeforDateField
  582. return field.is_one_of("era"sv, "year"sv, "quarter"sv, "month"sv, "week"sv, "weekday"sv, "day"sv, "dayperiod"sv, "hour"sv, "minute"sv, "second"sv, "zone"sv);
  583. };
  584. fields_object.as_object().for_each_member([&](auto const& key, JsonValue const&) {
  585. if (!is_sanctioned_field(key))
  586. return;
  587. if (!locale_data.date_fields.contains_slow(key))
  588. locale_data.date_fields.append(key);
  589. });
  590. quick_sort(locale_data.date_fields);
  591. DateFieldList long_date_fields {};
  592. long_date_fields.resize(locale_data.date_fields.size());
  593. DateFieldList short_date_fields {};
  594. short_date_fields.resize(locale_data.date_fields.size());
  595. DateFieldList narrow_date_fields {};
  596. narrow_date_fields.resize(locale_data.date_fields.size());
  597. fields_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  598. if (!is_sanctioned_field(key))
  599. return;
  600. auto const& long_name = value.as_object().get("displayName"sv);
  601. auto const& short_name = fields_object.as_object().get(String::formatted("{}-short", key)).as_object().get("displayName"sv);
  602. auto const& narrow_name = fields_object.as_object().get(String::formatted("{}-narrow", key)).as_object().get("displayName"sv);
  603. auto index = locale_data.date_fields.find_first_index(key).value();
  604. long_date_fields[index] = locale_data.unique_strings.ensure(long_name.as_string());
  605. short_date_fields[index] = locale_data.unique_strings.ensure(short_name.as_string());
  606. narrow_date_fields[index] = locale_data.unique_strings.ensure(narrow_name.as_string());
  607. });
  608. locale.long_date_fields = locale_data.unique_date_field_lists.ensure(move(long_date_fields));
  609. locale.short_date_fields = locale_data.unique_date_field_lists.ensure(move(short_date_fields));
  610. locale.narrow_date_fields = locale_data.unique_date_field_lists.ensure(move(narrow_date_fields));
  611. return {};
  612. }
  613. static ErrorOr<void> parse_number_system_keywords(String locale_numbers_path, UnicodeLocaleData& locale_data, Locale& locale)
  614. {
  615. LexicalPath numbers_path(move(locale_numbers_path));
  616. numbers_path = numbers_path.append("numbers.json"sv);
  617. auto numbers = TRY(read_json_file(numbers_path.string()));
  618. auto const& main_object = numbers.as_object().get("main"sv);
  619. auto const& locale_object = main_object.as_object().get(numbers_path.parent().basename());
  620. auto const& locale_numbers_object = locale_object.as_object().get("numbers"sv);
  621. auto const& default_numbering_system_object = locale_numbers_object.as_object().get("defaultNumberingSystem"sv);
  622. auto const& other_numbering_systems_object = locale_numbers_object.as_object().get("otherNumberingSystems"sv);
  623. KeywordList keywords {};
  624. auto append_numbering_system = [&](String system_name) {
  625. if (auto system_alias = find_keyword_alias("nu"sv, system_name, locale_data); system_alias.has_value())
  626. system_name = system_alias.release_value();
  627. auto index = locale_data.unique_strings.ensure(move(system_name));
  628. if (!keywords.contains_slow(index))
  629. keywords.append(move(index));
  630. };
  631. append_numbering_system(default_numbering_system_object.as_string());
  632. other_numbering_systems_object.as_object().for_each_member([&](auto const&, JsonValue const& value) {
  633. append_numbering_system(value.as_string());
  634. });
  635. locale_numbers_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  636. if (!key.starts_with("defaultNumberingSystem-alt-"sv))
  637. return;
  638. append_numbering_system(value.as_string());
  639. });
  640. locale.number_system_keywords = locale_data.unique_keyword_lists.ensure(move(keywords));
  641. return {};
  642. }
  643. static ErrorOr<void> parse_calendar_keywords(String locale_dates_path, UnicodeLocaleData& locale_data, Locale& locale)
  644. {
  645. auto calendars_iterator = TRY(path_to_dir_iterator(locale_dates_path, {}));
  646. KeywordList keywords {};
  647. while (calendars_iterator.has_next()) {
  648. auto locale_calendars_path = TRY(next_path_from_dir_iterator(calendars_iterator));
  649. LexicalPath calendars_path(move(locale_calendars_path));
  650. if (!calendars_path.basename().starts_with("ca-"sv))
  651. continue;
  652. auto calendars = TRY(read_json_file(calendars_path.string()));
  653. auto const& main_object = calendars.as_object().get("main"sv);
  654. auto const& locale_object = main_object.as_object().get(calendars_path.parent().basename());
  655. auto const& dates_object = locale_object.as_object().get("dates"sv);
  656. auto const& calendars_object = dates_object.as_object().get("calendars"sv);
  657. calendars_object.as_object().for_each_member([&](auto calendar_name, JsonValue const&) {
  658. // The generic calendar is not a supported Unicode calendar key, so skip it:
  659. // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/Locale/calendar#unicode_calendar_keys
  660. if (calendar_name == "generic"sv)
  661. return;
  662. if (auto calendar_alias = find_keyword_alias("ca"sv, calendar_name, locale_data); calendar_alias.has_value())
  663. calendar_name = calendar_alias.release_value();
  664. keywords.append(locale_data.unique_strings.ensure(calendar_name));
  665. });
  666. }
  667. locale.calendar_keywords = locale_data.unique_keyword_lists.ensure(move(keywords));
  668. return {};
  669. }
  670. static void fill_in_collation_keywords(UnicodeLocaleData& locale_data, Locale& locale)
  671. {
  672. // FIXME: If collation data becomes available in the CLDR, parse per-locale ordering from there.
  673. auto create_list_with_default_first = [&](auto key, auto default_value) {
  674. auto& values = locale_data.keywords.find(key)->value;
  675. quick_sort(values, [&](auto const& lhs, auto const& rhs) {
  676. if (lhs == default_value)
  677. return true;
  678. if (rhs == default_value)
  679. return false;
  680. return lhs < rhs;
  681. });
  682. KeywordList keywords;
  683. keywords.ensure_capacity(values.size());
  684. for (auto const& value : values)
  685. keywords.append(locale_data.unique_strings.ensure(value));
  686. return locale_data.unique_keyword_lists.ensure(move(keywords));
  687. };
  688. static auto kf_index = create_list_with_default_first("kf"sv, "upper"sv);
  689. static auto kn_index = create_list_with_default_first("kn"sv, "true"sv);
  690. locale.collation_case_keywords = kf_index;
  691. locale.collation_numeric_keywords = kn_index;
  692. }
  693. static ErrorOr<void> parse_default_content_locales(String core_path, UnicodeLocaleData& locale_data)
  694. {
  695. LexicalPath default_content_path(move(core_path));
  696. default_content_path = default_content_path.append("defaultContent.json"sv);
  697. auto default_content = TRY(read_json_file(default_content_path.string()));
  698. auto const& default_content_array = default_content.as_object().get("defaultContent"sv);
  699. default_content_array.as_array().for_each([&](JsonValue const& value) {
  700. auto locale = value.as_string();
  701. StringView default_locale = locale;
  702. while (true) {
  703. if (locale_data.locales.contains(default_locale))
  704. break;
  705. auto pos = default_locale.find_last('-');
  706. if (!pos.has_value())
  707. return;
  708. default_locale = default_locale.substring_view(0, *pos);
  709. }
  710. if (default_locale != locale)
  711. locale_data.locale_aliases.append({ default_locale, move(locale) });
  712. });
  713. return {};
  714. }
  715. static ErrorOr<void> define_aliases_without_scripts(UnicodeLocaleData& locale_data)
  716. {
  717. // From ECMA-402: https://tc39.es/ecma402/#sec-internal-slots
  718. //
  719. // For locales that include a script subtag in addition to language and region, the
  720. // corresponding locale without a script subtag must also be supported.
  721. //
  722. // So we define aliases for locales that contain all three subtags, but we must also take
  723. // care to handle when the locale itself or the locale without a script subtag are an alias
  724. // by way of default-content locales.
  725. auto find_alias = [&](auto const& locale) {
  726. return locale_data.locale_aliases.find_if([&](auto const& alias) { return locale == alias.alias; });
  727. };
  728. auto append_alias_without_script = [&](auto const& locale) -> ErrorOr<void> {
  729. auto parsed_locale = TRY(CanonicalLanguageID<StringIndexType>::parse(locale_data.unique_strings, locale));
  730. if ((parsed_locale.language == 0) || (parsed_locale.script == 0) || (parsed_locale.region == 0))
  731. return {};
  732. auto locale_without_script = String::formatted("{}-{}",
  733. locale_data.unique_strings.get(parsed_locale.language),
  734. locale_data.unique_strings.get(parsed_locale.region));
  735. if (locale_data.locales.contains(locale_without_script))
  736. return {};
  737. if (find_alias(locale_without_script) != locale_data.locale_aliases.end())
  738. return {};
  739. if (auto it = find_alias(locale); it != locale_data.locale_aliases.end())
  740. locale_data.locale_aliases.append({ it->name, locale_without_script });
  741. else
  742. locale_data.locale_aliases.append({ locale, locale_without_script });
  743. return {};
  744. };
  745. for (auto const& locale : locale_data.locales)
  746. TRY(append_alias_without_script(locale.key));
  747. for (auto const& locale : locale_data.locale_aliases)
  748. TRY(append_alias_without_script(locale.alias));
  749. return {};
  750. }
  751. static ErrorOr<void> parse_all_locales(String bcp47_path, String core_path, String locale_names_path, String misc_path, String numbers_path, String dates_path, UnicodeLocaleData& locale_data)
  752. {
  753. auto bcp47_iterator = TRY(path_to_dir_iterator(move(bcp47_path), "bcp47"sv));
  754. auto identity_iterator = TRY(path_to_dir_iterator(locale_names_path));
  755. auto preprocess_iterator = TRY(path_to_dir_iterator(locale_names_path));
  756. auto locale_names_iterator = TRY(path_to_dir_iterator(move(locale_names_path)));
  757. auto misc_iterator = TRY(path_to_dir_iterator(move(misc_path)));
  758. auto numbers_iterator = TRY(path_to_dir_iterator(move(numbers_path)));
  759. auto dates_iterator = TRY(path_to_dir_iterator(move(dates_path)));
  760. LexicalPath core_supplemental_path(core_path);
  761. core_supplemental_path = core_supplemental_path.append("supplemental"sv);
  762. VERIFY(Core::File::is_directory(core_supplemental_path.string()));
  763. TRY(parse_core_aliases(core_supplemental_path.string(), locale_data));
  764. TRY(parse_likely_subtags(core_supplemental_path.string(), locale_data));
  765. auto remove_variants_from_path = [&](String path) -> ErrorOr<String> {
  766. auto parsed_locale = TRY(CanonicalLanguageID<StringIndexType>::parse(locale_data.unique_strings, LexicalPath::basename(path)));
  767. StringBuilder builder;
  768. builder.append(locale_data.unique_strings.get(parsed_locale.language));
  769. if (auto script = locale_data.unique_strings.get(parsed_locale.script); !script.is_empty())
  770. builder.appendff("-{}", script);
  771. if (auto region = locale_data.unique_strings.get(parsed_locale.region); !region.is_empty())
  772. builder.appendff("-{}", region);
  773. return builder.build();
  774. };
  775. while (identity_iterator.has_next()) {
  776. auto locale_path = TRY(next_path_from_dir_iterator(identity_iterator));
  777. auto language = TRY(remove_variants_from_path(locale_path));
  778. auto& locale = locale_data.locales.ensure(language);
  779. TRY(parse_identity(locale_path, locale_data, locale));
  780. }
  781. while (preprocess_iterator.has_next()) {
  782. auto locale_path = TRY(next_path_from_dir_iterator(preprocess_iterator));
  783. TRY(preprocess_languages(locale_path, locale_data));
  784. }
  785. quick_sort(locale_data.languages);
  786. quick_sort(locale_data.territories);
  787. quick_sort(locale_data.scripts);
  788. while (bcp47_iterator.has_next()) {
  789. auto bcp47_path = TRY(next_path_from_dir_iterator(bcp47_iterator));
  790. TRY(parse_unicode_extension_keywords(move(bcp47_path), locale_data));
  791. }
  792. while (locale_names_iterator.has_next()) {
  793. auto locale_path = TRY(next_path_from_dir_iterator(locale_names_iterator));
  794. auto language = TRY(remove_variants_from_path(locale_path));
  795. auto& locale = locale_data.locales.ensure(language);
  796. TRY(parse_locale_display_patterns(locale_path, locale_data, locale));
  797. TRY(parse_locale_languages(locale_path, locale_data, locale));
  798. TRY(parse_locale_territories(locale_path, locale_data, locale));
  799. TRY(parse_locale_scripts(locale_path, locale_data, locale));
  800. TRY(parse_locale_calendars(locale_path, locale_data, locale));
  801. }
  802. while (misc_iterator.has_next()) {
  803. auto misc_path = TRY(next_path_from_dir_iterator(misc_iterator));
  804. auto language = TRY(remove_variants_from_path(misc_path));
  805. auto& locale = locale_data.locales.ensure(language);
  806. TRY(parse_locale_list_patterns(misc_path, locale_data, locale));
  807. TRY(parse_locale_layout(misc_path, locale_data, locale));
  808. }
  809. while (numbers_iterator.has_next()) {
  810. auto numbers_path = TRY(next_path_from_dir_iterator(numbers_iterator));
  811. auto language = TRY(remove_variants_from_path(numbers_path));
  812. auto& locale = locale_data.locales.ensure(language);
  813. TRY(parse_locale_currencies(numbers_path, locale_data, locale));
  814. TRY(parse_number_system_keywords(numbers_path, locale_data, locale));
  815. fill_in_collation_keywords(locale_data, locale);
  816. }
  817. while (dates_iterator.has_next()) {
  818. auto dates_path = TRY(next_path_from_dir_iterator(dates_iterator));
  819. auto language = TRY(remove_variants_from_path(dates_path));
  820. auto& locale = locale_data.locales.ensure(language);
  821. TRY(parse_locale_date_fields(dates_path, locale_data, locale));
  822. TRY(parse_calendar_keywords(dates_path, locale_data, locale));
  823. }
  824. TRY(parse_default_content_locales(move(core_path), locale_data));
  825. TRY(define_aliases_without_scripts(locale_data));
  826. return {};
  827. }
  828. static ErrorOr<void> generate_unicode_locale_header(Core::Stream::BufferedFile& file, UnicodeLocaleData& locale_data)
  829. {
  830. StringBuilder builder;
  831. SourceGenerator generator { builder };
  832. generator.append(R"~~~(
  833. #pragma once
  834. #include <AK/Types.h>
  835. namespace Unicode {
  836. )~~~");
  837. auto locales = locale_data.locales.keys();
  838. auto keywords = locale_data.keywords.keys();
  839. generate_enum(generator, format_identifier, "Locale"sv, "None"sv, locales, locale_data.locale_aliases);
  840. generate_enum(generator, format_identifier, "Language"sv, {}, locale_data.languages);
  841. generate_enum(generator, format_identifier, "Territory"sv, {}, locale_data.territories);
  842. generate_enum(generator, format_identifier, "ScriptTag"sv, {}, locale_data.scripts);
  843. generate_enum(generator, format_identifier, "Currency"sv, {}, locale_data.currencies);
  844. generate_enum(generator, format_identifier, "DateField"sv, {}, locale_data.date_fields, locale_data.date_field_aliases);
  845. generate_enum(generator, format_identifier, "Variant"sv, {}, locale_data.variants);
  846. generate_enum(generator, format_identifier, "ListPatternType"sv, {}, locale_data.list_pattern_types);
  847. generate_enum(generator, format_identifier, "CharacterOrder"sv, {}, locale_data.character_orders);
  848. generate_enum(generator, format_identifier, "Key"sv, {}, keywords);
  849. for (auto& keyword : locale_data.keywords) {
  850. auto const& keyword_name = locale_data.keyword_names.find(keyword.key)->value;
  851. auto enum_name = String::formatted("Keyword{}", format_identifier({}, keyword_name));
  852. if (auto aliases = locale_data.keyword_aliases.find(keyword.key); aliases != locale_data.keyword_aliases.end())
  853. generate_enum(generator, format_identifier, enum_name, {}, keyword.value, aliases->value);
  854. else
  855. generate_enum(generator, format_identifier, enum_name, {}, keyword.value);
  856. }
  857. generator.append(R"~~~(
  858. }
  859. )~~~");
  860. TRY(file.write(generator.as_string_view().bytes()));
  861. return {};
  862. }
  863. static ErrorOr<void> generate_unicode_locale_implementation(Core::Stream::BufferedFile& file, UnicodeLocaleData& locale_data)
  864. {
  865. StringBuilder builder;
  866. SourceGenerator generator { builder };
  867. generator.set("string_index_type"sv, s_string_index_type);
  868. generator.set("locales_size"sv, String::number(locale_data.locales.size()));
  869. generator.set("territories_size", String::number(locale_data.territories.size()));
  870. generator.set("variants_size", String::number(locale_data.max_variant_size));
  871. generator.append(R"~~~(
  872. #include <AK/Array.h>
  873. #include <AK/BinarySearch.h>
  874. #include <AK/Optional.h>
  875. #include <AK/Span.h>
  876. #include <AK/StringView.h>
  877. #include <AK/Vector.h>
  878. #include <LibUnicode/CurrencyCode.h>
  879. #include <LibUnicode/DateTimeFormat.h>
  880. #include <LibUnicode/Locale.h>
  881. #include <LibUnicode/UnicodeLocale.h>
  882. namespace Unicode {
  883. )~~~");
  884. locale_data.unique_strings.generate(generator);
  885. generator.append(R"~~~(
  886. struct DisplayPatternImpl {
  887. DisplayPattern to_display_pattern() const
  888. {
  889. DisplayPattern display_patterns {};
  890. display_patterns.locale_pattern = s_string_list[locale_pattern];
  891. display_patterns.locale_separator = s_string_list[locale_separator];
  892. return display_patterns;
  893. }
  894. @string_index_type@ locale_pattern { 0 };
  895. @string_index_type@ locale_separator { 0 };
  896. };
  897. struct Patterns {
  898. ListPatternType type;
  899. Style style;
  900. @string_index_type@ start { 0 };
  901. @string_index_type@ middle { 0 };
  902. @string_index_type@ end { 0 };
  903. @string_index_type@ pair { 0 };
  904. };
  905. struct TextLayout {
  906. CharacterOrder character_order;
  907. };
  908. )~~~");
  909. generate_available_values(generator, "get_available_calendars"sv, locale_data.keywords.find("ca"sv)->value, locale_data.keyword_aliases.find("ca"sv)->value);
  910. generate_available_values(generator, "get_available_number_systems"sv, locale_data.keywords.find("nu"sv)->value, locale_data.keyword_aliases.find("nu"sv)->value);
  911. generate_available_values(generator, "get_available_currencies"sv, locale_data.currencies);
  912. locale_data.unique_display_patterns.generate(generator, "DisplayPatternImpl"sv, "s_display_patterns"sv, 30);
  913. locale_data.unique_language_lists.generate(generator, s_string_index_type, "s_language_lists"sv);
  914. locale_data.unique_territory_lists.generate(generator, s_string_index_type, "s_territory_lists"sv);
  915. locale_data.unique_script_lists.generate(generator, s_string_index_type, "s_script_lists"sv);
  916. locale_data.unique_currency_lists.generate(generator, s_string_index_type, "s_currency_lists"sv);
  917. locale_data.unique_calendar_lists.generate(generator, s_string_index_type, "s_calendar_lists"sv);
  918. locale_data.unique_date_field_lists.generate(generator, s_string_index_type, "s_date_field_lists"sv);
  919. locale_data.unique_keyword_lists.generate(generator, s_string_index_type, "s_keyword_lists"sv);
  920. locale_data.unique_list_patterns.generate(generator, "Patterns"sv, "s_list_patterns"sv, 10);
  921. locale_data.unique_list_pattern_lists.generate(generator, s_list_pattern_index_type, "s_list_pattern_lists"sv);
  922. locale_data.unique_text_layouts.generate(generator, "TextLayout"sv, "s_text_layouts"sv, 30);
  923. auto append_index = [&](auto index) {
  924. generator.append(String::formatted(", {}", index));
  925. };
  926. auto append_list_and_size = [&](auto const& list) {
  927. if (list.is_empty()) {
  928. generator.append(", {}, 0");
  929. return;
  930. }
  931. bool first = true;
  932. generator.append(", {");
  933. for (auto const& item : list) {
  934. generator.append(first ? " " : ", ");
  935. generator.append(String::number(item));
  936. first = false;
  937. }
  938. generator.append(String::formatted(" }}, {}", list.size()));
  939. };
  940. auto append_mapping = [&](auto const& keys, auto const& map, auto type, auto name, auto mapping_getter) {
  941. generator.set("type", type);
  942. generator.set("name", name);
  943. generator.set("size", String::number(keys.size()));
  944. generator.append(R"~~~(
  945. static constexpr Array<@type@, @size@> @name@ { {)~~~");
  946. bool first = true;
  947. for (auto const& key : keys) {
  948. auto const& value = map.find(key)->value;
  949. auto mapping = mapping_getter(value);
  950. generator.append(first ? " " : ", ");
  951. generator.append(String::number(mapping));
  952. first = false;
  953. }
  954. generator.append(" } };");
  955. };
  956. auto locales = locale_data.locales.keys();
  957. quick_sort(locales);
  958. append_mapping(locales, locale_data.locales, s_display_pattern_index_type, "s_locale_display_patterns"sv, [&](auto const& locale) { return locale.display_patterns; });
  959. append_mapping(locales, locale_data.locales, s_language_list_index_type, "s_languages"sv, [&](auto const& locale) { return locale.languages; });
  960. append_mapping(locales, locale_data.locales, s_territory_list_index_type, "s_territories"sv, [&](auto const& locale) { return locale.territories; });
  961. append_mapping(locales, locale_data.locales, s_script_list_index_type, "s_scripts"sv, [&](auto const& locale) { return locale.scripts; });
  962. append_mapping(locales, locale_data.locales, s_currency_list_index_type, "s_long_currencies"sv, [&](auto const& locale) { return locale.long_currencies; });
  963. append_mapping(locales, locale_data.locales, s_currency_list_index_type, "s_short_currencies"sv, [&](auto const& locale) { return locale.short_currencies; });
  964. append_mapping(locales, locale_data.locales, s_currency_list_index_type, "s_narrow_currencies"sv, [&](auto const& locale) { return locale.narrow_currencies; });
  965. append_mapping(locales, locale_data.locales, s_currency_list_index_type, "s_numeric_currencies"sv, [&](auto const& locale) { return locale.numeric_currencies; });
  966. append_mapping(locales, locale_data.locales, s_calendar_list_index_type, "s_calendars"sv, [&](auto const& locale) { return locale.calendars; });
  967. append_mapping(locales, locale_data.locales, s_date_field_list_index_type, "s_long_date_fields"sv, [&](auto const& locale) { return locale.long_date_fields; });
  968. append_mapping(locales, locale_data.locales, s_date_field_list_index_type, "s_short_date_fields"sv, [&](auto const& locale) { return locale.short_date_fields; });
  969. append_mapping(locales, locale_data.locales, s_date_field_list_index_type, "s_narrow_date_fields"sv, [&](auto const& locale) { return locale.narrow_date_fields; });
  970. append_mapping(locales, locale_data.locales, s_keyword_list_index_type, "s_calendar_keywords"sv, [&](auto const& locale) { return locale.calendar_keywords; });
  971. append_mapping(locales, locale_data.locales, s_keyword_list_index_type, "s_collation_case_keywords"sv, [&](auto const& locale) { return locale.collation_case_keywords; });
  972. append_mapping(locales, locale_data.locales, s_keyword_list_index_type, "s_collation_numeric_keywords"sv, [&](auto const& locale) { return locale.collation_numeric_keywords; });
  973. append_mapping(locales, locale_data.locales, s_keyword_list_index_type, "s_number_system_keywords"sv, [&](auto const& locale) { return locale.number_system_keywords; });
  974. append_mapping(locales, locale_data.locales, s_list_pattern_list_index_type, "s_locale_list_patterns"sv, [&](auto const& locale) { return locale.list_patterns; });
  975. append_mapping(locales, locale_data.locales, s_text_layout_index_type, "s_locale_text_layouts"sv, [&](auto const& locale) { return locale.text_layout; });
  976. generator.append(R"~~~(
  977. struct CanonicalLanguageID {
  978. LanguageID to_unicode_language_id() const
  979. {
  980. LanguageID language_id {};
  981. language_id.variants.ensure_capacity(variants_size);
  982. language_id.language = s_string_list[language];
  983. if (script != 0)
  984. language_id.script = s_string_list[script];
  985. if (region != 0)
  986. language_id.region = s_string_list[region];
  987. for (size_t i = 0; i < variants_size; ++i)
  988. language_id.variants.append(s_string_list[variants[i]]);
  989. return language_id;
  990. }
  991. bool matches_variants(Vector<String> const& other_variants) const {
  992. if (variants_size == 0)
  993. return true;
  994. if (other_variants.size() != variants_size)
  995. return false;
  996. for (size_t i = 0; i < variants_size; ++i) {
  997. if (s_string_list[variants[i]] != other_variants[i])
  998. return false;
  999. }
  1000. return true;
  1001. };
  1002. @string_index_type@ language { 0 };
  1003. @string_index_type@ script { 0 };
  1004. @string_index_type@ region { 0 };
  1005. Array<@string_index_type@, @variants_size@> variants {};
  1006. size_t variants_size { 0 };
  1007. };
  1008. struct LanguageMapping {
  1009. CanonicalLanguageID key;
  1010. CanonicalLanguageID alias;
  1011. };
  1012. )~~~");
  1013. auto append_complex_mapping = [&](StringView name, auto& mappings) {
  1014. generator.set("size", String::number(mappings.size()));
  1015. generator.set("name"sv, name);
  1016. generator.append(R"~~~(
  1017. static constexpr Array<LanguageMapping, @size@> s_@name@ { {
  1018. )~~~");
  1019. quick_sort(mappings, [&](auto const& lhs, auto const& rhs) {
  1020. auto const& lhs_language = locale_data.unique_strings.get(lhs.key.language);
  1021. auto const& rhs_language = locale_data.unique_strings.get(rhs.key.language);
  1022. // Sort the keys such that "und" language tags are at the end, as those are less specific.
  1023. if (lhs_language.starts_with("und"sv) && !rhs_language.starts_with("und"sv))
  1024. return false;
  1025. if (!lhs_language.starts_with("und"sv) && rhs_language.starts_with("und"sv))
  1026. return true;
  1027. return lhs_language < rhs_language;
  1028. });
  1029. for (auto const& mapping : mappings) {
  1030. generator.set("language"sv, String::number(mapping.key.language));
  1031. generator.append(" { { @language@");
  1032. append_index(mapping.key.script);
  1033. append_index(mapping.key.region);
  1034. append_list_and_size(mapping.key.variants);
  1035. generator.set("language"sv, String::number(mapping.alias.language));
  1036. generator.append(" }, { @language@");
  1037. append_index(mapping.alias.script);
  1038. append_index(mapping.alias.region);
  1039. append_list_and_size(mapping.alias.variants);
  1040. generator.append(" } },\n");
  1041. }
  1042. generator.append("} };\n");
  1043. };
  1044. append_complex_mapping("complex_alias"sv, locale_data.complex_mappings);
  1045. append_complex_mapping("likely_subtags"sv, locale_data.likely_subtags);
  1046. generator.append(R"~~~(
  1047. static LanguageMapping const* resolve_likely_subtag(LanguageID const& language_id)
  1048. {
  1049. // https://unicode.org/reports/tr35/#Likely_Subtags
  1050. enum class State {
  1051. LanguageScriptRegion,
  1052. LanguageRegion,
  1053. LanguageScript,
  1054. Language,
  1055. UndScript,
  1056. Done,
  1057. };
  1058. auto state = State::LanguageScriptRegion;
  1059. while (state != State::Done) {
  1060. LanguageID search_key;
  1061. switch (state) {
  1062. case State::LanguageScriptRegion:
  1063. state = State::LanguageRegion;
  1064. if (!language_id.script.has_value() || !language_id.region.has_value())
  1065. continue;
  1066. search_key.language = *language_id.language;
  1067. search_key.script = *language_id.script;
  1068. search_key.region = *language_id.region;
  1069. break;
  1070. case State::LanguageRegion:
  1071. state = State::LanguageScript;
  1072. if (!language_id.region.has_value())
  1073. continue;
  1074. search_key.language = *language_id.language;
  1075. search_key.region = *language_id.region;
  1076. break;
  1077. case State::LanguageScript:
  1078. state = State::Language;
  1079. if (!language_id.script.has_value())
  1080. continue;
  1081. search_key.language = *language_id.language;
  1082. search_key.script = *language_id.script;
  1083. break;
  1084. case State::Language:
  1085. state = State::UndScript;
  1086. search_key.language = *language_id.language;
  1087. break;
  1088. case State::UndScript:
  1089. state = State::Done;
  1090. if (!language_id.script.has_value())
  1091. continue;
  1092. search_key.language = "und"sv;
  1093. search_key.script = *language_id.script;
  1094. break;
  1095. default:
  1096. VERIFY_NOT_REACHED();
  1097. }
  1098. for (auto const& map : s_likely_subtags) {
  1099. auto const& key_language = s_string_list[map.key.language];
  1100. auto const& key_script = s_string_list[map.key.script];
  1101. auto const& key_region = s_string_list[map.key.region];
  1102. if (key_language != search_key.language)
  1103. continue;
  1104. if (!key_script.is_empty() || search_key.script.has_value()) {
  1105. if (key_script != search_key.script)
  1106. continue;
  1107. }
  1108. if (!key_region.is_empty() || search_key.region.has_value()) {
  1109. if (key_region != search_key.region)
  1110. continue;
  1111. }
  1112. return &map;
  1113. }
  1114. }
  1115. return nullptr;
  1116. }
  1117. )~~~");
  1118. auto append_mapping_search = [&](StringView enum_snake, StringView from_string_name, StringView collection_name, StringView unique_list) {
  1119. generator.set("enum_snake", enum_snake);
  1120. generator.set("from_string_name", from_string_name);
  1121. generator.set("collection_name", collection_name);
  1122. generator.set("unique_list", unique_list);
  1123. generator.append(R"~~~(
  1124. Optional<StringView> get_locale_@enum_snake@_mapping(StringView locale, StringView @enum_snake@)
  1125. {
  1126. auto locale_value = locale_from_string(locale);
  1127. if (!locale_value.has_value())
  1128. return {};
  1129. auto @enum_snake@_value = @from_string_name@_from_string(@enum_snake@);
  1130. if (!@enum_snake@_value.has_value())
  1131. return {};
  1132. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1133. auto @enum_snake@_index = to_underlying(*@enum_snake@_value);
  1134. auto mapping_index = @collection_name@.at(locale_index);
  1135. auto const& mappings = @unique_list@.at(mapping_index);
  1136. auto @enum_snake@_string_index = mappings.at(@enum_snake@_index);
  1137. auto @enum_snake@_mapping = s_string_list.at(@enum_snake@_string_index);
  1138. if (@enum_snake@_mapping.is_empty())
  1139. return {};
  1140. return @enum_snake@_mapping;
  1141. }
  1142. )~~~");
  1143. };
  1144. auto append_from_string = [&](StringView enum_title, StringView enum_snake, auto const& values, Vector<Alias> const& aliases = {}) {
  1145. HashValueMap<String> hashes;
  1146. hashes.ensure_capacity(values.size());
  1147. for (auto const& value : values)
  1148. hashes.set(value.hash(), format_identifier(enum_title, value));
  1149. for (auto const& alias : aliases)
  1150. hashes.set(alias.alias.hash(), format_identifier(enum_title, alias.alias));
  1151. generate_value_from_string(generator, "{}_from_string"sv, enum_title, enum_snake, move(hashes));
  1152. };
  1153. auto append_alias_search = [&](StringView enum_snake, auto const& aliases) {
  1154. HashValueMap<StringIndexType> hashes;
  1155. hashes.ensure_capacity(aliases.size());
  1156. for (auto const& alias : aliases)
  1157. hashes.set(alias.key.hash(), alias.value);
  1158. ValueFromStringOptions options {};
  1159. options.return_type = "StringView"sv;
  1160. options.return_format = "s_string_list[{}]"sv;
  1161. generate_value_from_string(generator, "resolve_{}_alias"sv, s_string_index_type, enum_snake, move(hashes), options);
  1162. };
  1163. append_from_string("Locale"sv, "locale"sv, locale_data.locales.keys(), locale_data.locale_aliases);
  1164. append_from_string("Language"sv, "language"sv, locale_data.languages);
  1165. append_mapping_search("language"sv, "language"sv, "s_languages"sv, "s_language_lists"sv);
  1166. append_alias_search("language"sv, locale_data.language_aliases);
  1167. append_from_string("Territory"sv, "territory"sv, locale_data.territories);
  1168. append_mapping_search("territory"sv, "territory"sv, "s_territories"sv, "s_territory_lists"sv);
  1169. append_alias_search("territory"sv, locale_data.territory_aliases);
  1170. append_from_string("ScriptTag"sv, "script_tag"sv, locale_data.scripts);
  1171. append_mapping_search("script"sv, "script_tag"sv, "s_scripts"sv, "s_script_lists"sv);
  1172. append_alias_search("script_tag"sv, locale_data.script_aliases);
  1173. append_from_string("Currency"sv, "currency"sv, locale_data.currencies);
  1174. append_mapping_search("long_currency"sv, "currency"sv, "s_long_currencies"sv, "s_currency_lists"sv);
  1175. append_mapping_search("short_currency"sv, "currency"sv, "s_short_currencies"sv, "s_currency_lists"sv);
  1176. append_mapping_search("narrow_currency"sv, "currency"sv, "s_narrow_currencies"sv, "s_currency_lists"sv);
  1177. append_mapping_search("numeric_currency"sv, "currency"sv, "s_numeric_currencies"sv, "s_currency_lists"sv);
  1178. append_from_string("DateField"sv, "date_field"sv, locale_data.date_fields, locale_data.date_field_aliases);
  1179. append_mapping_search("long_date_field"sv, "date_field"sv, "s_long_date_fields"sv, "s_date_field_lists"sv);
  1180. append_mapping_search("short_date_field"sv, "date_field"sv, "s_short_date_fields"sv, "s_date_field_lists"sv);
  1181. append_mapping_search("narrow_date_field"sv, "date_field"sv, "s_narrow_date_fields"sv, "s_date_field_lists"sv);
  1182. append_from_string("Key"sv, "key"sv, locale_data.keywords.keys());
  1183. for (auto const& keyword : locale_data.keywords) {
  1184. auto const& keyword_name = locale_data.keyword_names.find(keyword.key)->value;
  1185. auto enum_name = String::formatted("Keyword{}", format_identifier({}, keyword_name));
  1186. auto enum_snake = String::formatted("keyword_{}", keyword.key);
  1187. if (auto aliases = locale_data.keyword_aliases.find(keyword.key); aliases != locale_data.keyword_aliases.end())
  1188. append_from_string(enum_name, enum_snake, keyword.value, aliases->value);
  1189. else
  1190. append_from_string(enum_name, enum_snake, keyword.value);
  1191. }
  1192. append_mapping_search("calendar"sv, "keyword_ca"sv, "s_calendars"sv, "s_calendar_lists"sv);
  1193. append_alias_search("variant"sv, locale_data.variant_aliases);
  1194. append_alias_search("subdivision"sv, locale_data.subdivision_aliases);
  1195. append_from_string("ListPatternType"sv, "list_pattern_type"sv, locale_data.list_pattern_types);
  1196. append_from_string("CharacterOrder"sv, "character_order"sv, locale_data.character_orders);
  1197. generate_value_to_string(generator, "{}_to_string"sv, "CharacterOrder"sv, "character_order"sv, format_identifier, locale_data.character_orders);
  1198. generator.append(R"~~~(
  1199. Vector<StringView> get_keywords_for_locale(StringView locale, StringView key)
  1200. {
  1201. // Hour cycle keywords are region-based rather than locale-based, so they need to be handled specially.
  1202. // FIXME: Calendar keywords are also region-based, and will need to be handled here when we support non-Gregorian calendars:
  1203. // https://github.com/unicode-org/cldr-json/blob/main/cldr-json/cldr-core/supplemental/calendarPreferenceData.json
  1204. if (key == "hc"sv) {
  1205. auto hour_cycles = get_locale_hour_cycles(locale);
  1206. Vector<StringView> values;
  1207. values.ensure_capacity(hour_cycles.size());
  1208. for (auto hour_cycle : hour_cycles)
  1209. values.unchecked_append(hour_cycle_to_string(hour_cycle));
  1210. return values;
  1211. }
  1212. auto locale_value = locale_from_string(locale);
  1213. if (!locale_value.has_value())
  1214. return {};
  1215. auto key_value = key_from_string(key);
  1216. if (!key_value.has_value())
  1217. return {};
  1218. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1219. size_t keywords_index = 0;
  1220. switch (*key_value) {
  1221. case Key::Ca:
  1222. keywords_index = s_calendar_keywords.at(locale_index);
  1223. break;
  1224. case Key::Kf:
  1225. keywords_index = s_collation_case_keywords.at(locale_index);
  1226. break;
  1227. case Key::Kn:
  1228. keywords_index = s_collation_numeric_keywords.at(locale_index);
  1229. break;
  1230. case Key::Nu:
  1231. keywords_index = s_number_system_keywords.at(locale_index);
  1232. break;
  1233. default:
  1234. VERIFY_NOT_REACHED();
  1235. }
  1236. auto keyword_indices = s_keyword_lists.at(keywords_index);
  1237. Vector<StringView> keywords;
  1238. keywords.ensure_capacity(keyword_indices.size());
  1239. for (auto keyword : keyword_indices)
  1240. keywords.unchecked_append(s_string_list[keyword]);
  1241. return keywords;
  1242. }
  1243. Optional<DisplayPattern> get_locale_display_patterns(StringView locale)
  1244. {
  1245. auto locale_value = locale_from_string(locale);
  1246. if (!locale_value.has_value())
  1247. return {};
  1248. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1249. auto display_patterns_index = s_locale_display_patterns.at(locale_index);
  1250. auto const& display_patterns = s_display_patterns.at(display_patterns_index);
  1251. return display_patterns.to_display_pattern();
  1252. }
  1253. Optional<ListPatterns> get_locale_list_patterns(StringView locale, StringView list_pattern_type, Style list_pattern_style)
  1254. {
  1255. auto locale_value = locale_from_string(locale);
  1256. if (!locale_value.has_value())
  1257. return {};
  1258. auto type_value = list_pattern_type_from_string(list_pattern_type);
  1259. if (!type_value.has_value())
  1260. return {};
  1261. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1262. auto list_patterns_list_index = s_locale_list_patterns.at(locale_index);
  1263. auto const& locale_list_patterns = s_list_pattern_lists.at(list_patterns_list_index);
  1264. for (auto list_patterns_index : locale_list_patterns) {
  1265. auto const& list_patterns = s_list_patterns.at(list_patterns_index);
  1266. if ((list_patterns.type == type_value) && (list_patterns.style == list_pattern_style)) {
  1267. auto const& start = s_string_list[list_patterns.start];
  1268. auto const& middle = s_string_list[list_patterns.middle];
  1269. auto const& end = s_string_list[list_patterns.end];
  1270. auto const& pair = s_string_list[list_patterns.pair];
  1271. return ListPatterns { start, middle, end, pair };
  1272. }
  1273. }
  1274. return {};
  1275. }
  1276. static Optional<TextLayout> text_layout_for_locale(StringView locale)
  1277. {
  1278. auto locale_value = locale_from_string(locale);
  1279. if (!locale_value.has_value())
  1280. return {};
  1281. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1282. auto text_layouts_index = s_locale_text_layouts.at(locale_index);
  1283. return s_text_layouts.at(text_layouts_index);
  1284. }
  1285. Optional<CharacterOrder> character_order_for_locale(StringView locale)
  1286. {
  1287. if (auto text_layout = text_layout_for_locale(locale); text_layout.has_value())
  1288. return text_layout->character_order;
  1289. return {};
  1290. }
  1291. void resolve_complex_language_aliases(LanguageID& language_id)
  1292. {
  1293. for (auto const& map : s_complex_alias) {
  1294. auto const& key_language = s_string_list[map.key.language];
  1295. auto const& key_script = s_string_list[map.key.script];
  1296. auto const& key_region = s_string_list[map.key.region];
  1297. if ((key_language != language_id.language) && (key_language != "und"sv))
  1298. continue;
  1299. if (!key_script.is_empty() && (key_script != language_id.script))
  1300. continue;
  1301. if (!key_region.is_empty() && (key_region != language_id.region))
  1302. continue;
  1303. if (!map.key.matches_variants(language_id.variants))
  1304. continue;
  1305. auto alias = map.alias.to_unicode_language_id();
  1306. if (alias.language == "und"sv)
  1307. alias.language = move(language_id.language);
  1308. if (key_script.is_empty() && !alias.script.has_value())
  1309. alias.script = move(language_id.script);
  1310. if (key_region.is_empty() && !alias.region.has_value())
  1311. alias.region = move(language_id.region);
  1312. if (map.key.variants_size == 0 && alias.variants.is_empty())
  1313. alias.variants = move(language_id.variants);
  1314. language_id = move(alias);
  1315. break;
  1316. }
  1317. }
  1318. Optional<LanguageID> add_likely_subtags(LanguageID const& language_id)
  1319. {
  1320. // https://www.unicode.org/reports/tr35/#Likely_Subtags
  1321. auto const* likely_subtag = resolve_likely_subtag(language_id);
  1322. if (likely_subtag == nullptr)
  1323. return {};
  1324. auto maximized = language_id;
  1325. auto const& key_script = s_string_list[likely_subtag->key.script];
  1326. auto const& key_region = s_string_list[likely_subtag->key.region];
  1327. auto const& alias_language = s_string_list[likely_subtag->alias.language];
  1328. auto const& alias_script = s_string_list[likely_subtag->alias.script];
  1329. auto const& alias_region = s_string_list[likely_subtag->alias.region];
  1330. if (maximized.language == "und"sv)
  1331. maximized.language = alias_language;
  1332. if (!maximized.script.has_value() || (!key_script.is_empty() && !alias_script.is_empty()))
  1333. maximized.script = alias_script;
  1334. if (!maximized.region.has_value() || (!key_region.is_empty() && !alias_region.is_empty()))
  1335. maximized.region = alias_region;
  1336. return maximized;
  1337. }
  1338. Optional<String> resolve_most_likely_territory(LanguageID const& language_id)
  1339. {
  1340. if (auto const* likely_subtag = resolve_likely_subtag(language_id); likely_subtag != nullptr)
  1341. return s_string_list[likely_subtag->alias.region];
  1342. return {};
  1343. }
  1344. }
  1345. )~~~");
  1346. TRY(file.write(generator.as_string_view().bytes()));
  1347. return {};
  1348. }
  1349. ErrorOr<int> serenity_main(Main::Arguments arguments)
  1350. {
  1351. StringView generated_header_path;
  1352. StringView generated_implementation_path;
  1353. StringView bcp47_path;
  1354. StringView core_path;
  1355. StringView locale_names_path;
  1356. StringView misc_path;
  1357. StringView numbers_path;
  1358. StringView dates_path;
  1359. Core::ArgsParser args_parser;
  1360. args_parser.add_option(generated_header_path, "Path to the Unicode locale header file to generate", "generated-header-path", 'h', "generated-header-path");
  1361. args_parser.add_option(generated_implementation_path, "Path to the Unicode locale implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
  1362. args_parser.add_option(bcp47_path, "Path to cldr-bcp47 directory", "bcp47-path", 'b', "bcp47-path");
  1363. args_parser.add_option(core_path, "Path to cldr-core directory", "core-path", 'r', "core-path");
  1364. args_parser.add_option(locale_names_path, "Path to cldr-localenames directory", "locale-names-path", 'l', "locale-names-path");
  1365. args_parser.add_option(misc_path, "Path to cldr-misc directory", "misc-path", 'm', "misc-path");
  1366. args_parser.add_option(numbers_path, "Path to cldr-numbers directory", "numbers-path", 'n', "numbers-path");
  1367. args_parser.add_option(dates_path, "Path to cldr-dates directory", "dates-path", 'd', "dates-path");
  1368. args_parser.parse(arguments);
  1369. auto generated_header_file = TRY(open_file(generated_header_path, Core::Stream::OpenMode::Write));
  1370. auto generated_implementation_file = TRY(open_file(generated_implementation_path, Core::Stream::OpenMode::Write));
  1371. UnicodeLocaleData locale_data;
  1372. TRY(parse_all_locales(bcp47_path, core_path, locale_names_path, misc_path, numbers_path, dates_path, locale_data));
  1373. TRY(generate_unicode_locale_header(*generated_header_file, locale_data));
  1374. TRY(generate_unicode_locale_implementation(*generated_implementation_file, locale_data));
  1375. return 0;
  1376. }