Locale.cpp 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878
  1. /*
  2. * Copyright (c) 2021, Tim Flynn <trflynn89@pm.me>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include <AK/AllOf.h>
  7. #include <AK/CharacterTypes.h>
  8. #include <AK/GenericLexer.h>
  9. #include <AK/QuickSort.h>
  10. #include <AK/StringBuilder.h>
  11. #include <LibUnicode/Locale.h>
  12. #if ENABLE_UNICODE_DATA
  13. # include <LibUnicode/UnicodeLocale.h>
  14. #endif
  15. namespace Unicode {
  16. bool is_unicode_language_subtag(StringView subtag)
  17. {
  18. // unicode_language_subtag = alpha{2,3} | alpha{5,8}
  19. if ((subtag.length() < 2) || (subtag.length() == 4) || (subtag.length() > 8))
  20. return false;
  21. return all_of(subtag, is_ascii_alpha);
  22. }
  23. bool is_unicode_script_subtag(StringView subtag)
  24. {
  25. // unicode_script_subtag = alpha{4}
  26. if (subtag.length() != 4)
  27. return false;
  28. return all_of(subtag, is_ascii_alpha);
  29. }
  30. bool is_unicode_region_subtag(StringView subtag)
  31. {
  32. // unicode_region_subtag = (alpha{2} | digit{3})
  33. if (subtag.length() == 2)
  34. return all_of(subtag, is_ascii_alpha);
  35. if (subtag.length() == 3)
  36. return all_of(subtag, is_ascii_digit);
  37. return false;
  38. }
  39. bool is_unicode_variant_subtag(StringView subtag)
  40. {
  41. // unicode_variant_subtag = (alphanum{5,8} | digit alphanum{3})
  42. if ((subtag.length() >= 5) && (subtag.length() <= 8))
  43. return all_of(subtag, is_ascii_alphanumeric);
  44. if (subtag.length() == 4)
  45. return is_ascii_digit(subtag[0]) && all_of(subtag.substring_view(1), is_ascii_alphanumeric);
  46. return false;
  47. }
  48. static bool is_key(StringView key)
  49. {
  50. // key = alphanum alpha
  51. if (key.length() != 2)
  52. return false;
  53. return is_ascii_alphanumeric(key[0]) && is_ascii_alpha(key[1]);
  54. }
  55. static bool is_single_type(StringView type)
  56. {
  57. // type = alphanum{3,8} (sep alphanum{3,8})*
  58. // Note: Consecutive types are not handled here, that is left to the caller.
  59. if ((type.length() < 3) || (type.length() > 8))
  60. return false;
  61. return all_of(type, is_ascii_alphanumeric);
  62. }
  63. static bool is_attribute(StringView type)
  64. {
  65. // attribute = alphanum{3,8}
  66. if ((type.length() < 3) || (type.length() > 8))
  67. return false;
  68. return all_of(type, is_ascii_alphanumeric);
  69. }
  70. static bool is_transformed_key(StringView key)
  71. {
  72. // tkey = alpha digit
  73. if (key.length() != 2)
  74. return false;
  75. return is_ascii_alpha(key[0]) && is_ascii_digit(key[1]);
  76. }
  77. static bool is_single_transformed_value(StringView value)
  78. {
  79. // tvalue = (sep alphanum{3,8})+
  80. // Note: Consecutive values are not handled here, that is left to the caller.
  81. if ((value.length() < 3) || (value.length() > 8))
  82. return false;
  83. return all_of(value, is_ascii_alphanumeric);
  84. }
  85. static Optional<StringView> consume_next_segment(GenericLexer& lexer, bool with_separator = true)
  86. {
  87. constexpr auto is_separator = is_any_of("-_"sv);
  88. if (with_separator) {
  89. if (!lexer.next_is(is_separator))
  90. return {};
  91. lexer.ignore();
  92. }
  93. auto segment = lexer.consume_until(is_separator);
  94. if (segment.is_empty()) {
  95. lexer.retreat(with_separator);
  96. return {};
  97. }
  98. return segment;
  99. }
  100. static Optional<LanguageID> parse_unicode_language_id(GenericLexer& lexer)
  101. {
  102. // https://unicode.org/reports/tr35/#Unicode_language_identifier
  103. //
  104. // unicode_language_id = "root"
  105. // OR
  106. // unicode_language_id = ((unicode_language_subtag (sep unicode_script_subtag)?) | unicode_script_subtag)
  107. // (sep unicode_region_subtag)?
  108. // (sep unicode_variant_subtag)*
  109. LanguageID language_id {};
  110. if (lexer.consume_specific("root"sv)) {
  111. language_id.is_root = true;
  112. return language_id;
  113. }
  114. enum class ParseState {
  115. ParsingLanguageOrScript,
  116. ParsingScript,
  117. ParsingRegion,
  118. ParsingVariant,
  119. Done,
  120. };
  121. auto state = ParseState::ParsingLanguageOrScript;
  122. while (!lexer.is_eof() && (state != ParseState::Done)) {
  123. auto segment = consume_next_segment(lexer, state != ParseState::ParsingLanguageOrScript);
  124. if (!segment.has_value())
  125. return {};
  126. switch (state) {
  127. case ParseState::ParsingLanguageOrScript:
  128. if (is_unicode_language_subtag(*segment)) {
  129. state = ParseState::ParsingScript;
  130. language_id.language = *segment;
  131. } else if (is_unicode_script_subtag(*segment)) {
  132. state = ParseState::ParsingRegion;
  133. language_id.script = *segment;
  134. } else {
  135. return {};
  136. }
  137. break;
  138. case ParseState::ParsingScript:
  139. if (is_unicode_script_subtag(*segment)) {
  140. state = ParseState::ParsingRegion;
  141. language_id.script = *segment;
  142. break;
  143. }
  144. state = ParseState::ParsingRegion;
  145. [[fallthrough]];
  146. case ParseState::ParsingRegion:
  147. if (is_unicode_region_subtag(*segment)) {
  148. state = ParseState::ParsingVariant;
  149. language_id.region = *segment;
  150. break;
  151. }
  152. state = ParseState::ParsingVariant;
  153. [[fallthrough]];
  154. case ParseState::ParsingVariant:
  155. if (is_unicode_variant_subtag(*segment)) {
  156. language_id.variants.append(*segment);
  157. } else {
  158. lexer.retreat(segment->length() + 1);
  159. state = ParseState::Done;
  160. }
  161. break;
  162. default:
  163. VERIFY_NOT_REACHED();
  164. }
  165. }
  166. return language_id;
  167. }
  168. static Optional<LocaleExtension> parse_unicode_locale_extension(GenericLexer& lexer)
  169. {
  170. // https://unicode.org/reports/tr35/#unicode_locale_extensions
  171. //
  172. // unicode_locale_extensions = sep [uU] ((sep keyword)+ | (sep attribute)+ (sep keyword)*)
  173. LocaleExtension locale_extension {};
  174. enum class ParseState {
  175. ParsingAttributeOrKeyword,
  176. ParsingAttribute,
  177. ParsingKeyword,
  178. Done,
  179. };
  180. auto state = ParseState::ParsingAttributeOrKeyword;
  181. while (!lexer.is_eof() && (state != ParseState::Done)) {
  182. auto segment = consume_next_segment(lexer);
  183. if (!segment.has_value())
  184. return {};
  185. if (state == ParseState::ParsingAttributeOrKeyword)
  186. state = is_key(*segment) ? ParseState::ParsingKeyword : ParseState::ParsingAttribute;
  187. switch (state) {
  188. case ParseState::ParsingAttribute:
  189. if (is_attribute(*segment)) {
  190. locale_extension.attributes.append(*segment);
  191. break;
  192. }
  193. state = ParseState::ParsingKeyword;
  194. [[fallthrough]];
  195. case ParseState::ParsingKeyword: {
  196. // keyword = key (sep type)?
  197. Keyword keyword { .key = *segment };
  198. if (!is_key(*segment)) {
  199. lexer.retreat(segment->length() + 1);
  200. state = ParseState::Done;
  201. break;
  202. }
  203. while (true) {
  204. auto type = consume_next_segment(lexer);
  205. if (!type.has_value() || !is_single_type(*type)) {
  206. if (type.has_value())
  207. lexer.retreat(type->length() + 1);
  208. break;
  209. }
  210. keyword.types.append(*type);
  211. }
  212. locale_extension.keywords.append(move(keyword));
  213. break;
  214. }
  215. default:
  216. VERIFY_NOT_REACHED();
  217. }
  218. }
  219. if (locale_extension.attributes.is_empty() && locale_extension.keywords.is_empty())
  220. return {};
  221. return locale_extension;
  222. }
  223. static Optional<TransformedExtension> parse_transformed_extension(GenericLexer& lexer)
  224. {
  225. // https://unicode.org/reports/tr35/#transformed_extensions
  226. //
  227. // transformed_extensions = sep [tT] ((sep tlang (sep tfield)*) | (sep tfield)+)
  228. TransformedExtension transformed_extension {};
  229. enum class ParseState {
  230. ParsingLanguageOrField,
  231. ParsingLanguage,
  232. ParsingField,
  233. Done,
  234. };
  235. auto state = ParseState::ParsingLanguageOrField;
  236. while (!lexer.is_eof() && (state != ParseState::Done)) {
  237. auto segment = consume_next_segment(lexer);
  238. if (!segment.has_value())
  239. return {};
  240. if (state == ParseState::ParsingLanguageOrField)
  241. state = is_unicode_language_subtag(*segment) ? ParseState::ParsingLanguage : ParseState::ParsingField;
  242. switch (state) {
  243. case ParseState::ParsingLanguage:
  244. lexer.retreat(segment->length());
  245. if (auto language_id = parse_unicode_language_id(lexer); language_id.has_value()) {
  246. transformed_extension.language = language_id.release_value();
  247. state = ParseState::ParsingField;
  248. break;
  249. }
  250. return {};
  251. case ParseState::ParsingField: {
  252. // tfield = tkey tvalue;
  253. TransformedField field { .key = *segment };
  254. if (!is_transformed_key(*segment)) {
  255. lexer.retreat(segment->length() + 1);
  256. state = ParseState::Done;
  257. break;
  258. }
  259. while (true) {
  260. auto value = consume_next_segment(lexer);
  261. if (!value.has_value() || !is_single_transformed_value(*value)) {
  262. if (value.has_value())
  263. lexer.retreat(value->length() + 1);
  264. break;
  265. }
  266. field.values.append(*value);
  267. }
  268. if (field.values.is_empty())
  269. return {};
  270. transformed_extension.fields.append(move(field));
  271. break;
  272. }
  273. default:
  274. VERIFY_NOT_REACHED();
  275. }
  276. }
  277. if (!transformed_extension.language.has_value() && transformed_extension.fields.is_empty())
  278. return {};
  279. return transformed_extension;
  280. }
  281. static Optional<OtherExtension> parse_other_extension(char key, GenericLexer& lexer)
  282. {
  283. // https://unicode.org/reports/tr35/#other_extensions
  284. //
  285. // other_extensions = sep [alphanum-[tTuUxX]] (sep alphanum{2,8})+ ;
  286. OtherExtension other_extension { .key = key };
  287. if (!is_ascii_alphanumeric(key) || (key == 'x') || (key == 'X'))
  288. return {};
  289. while (true) {
  290. auto segment = consume_next_segment(lexer);
  291. if (!segment.has_value())
  292. break;
  293. if ((segment->length() < 2) || (segment->length() > 8) || !all_of(*segment, is_ascii_alphanumeric)) {
  294. lexer.retreat(segment->length() + 1);
  295. break;
  296. }
  297. other_extension.values.append(*segment);
  298. }
  299. if (other_extension.values.is_empty())
  300. return {};
  301. return other_extension;
  302. }
  303. static Optional<Extension> parse_extension(GenericLexer& lexer)
  304. {
  305. // https://unicode.org/reports/tr35/#extensions
  306. //
  307. // extensions = unicode_locale_extensions | transformed_extensions | other_extensions
  308. size_t starting_position = lexer.tell();
  309. if (auto header = consume_next_segment(lexer); header.has_value() && (header->length() == 1)) {
  310. switch (char key = (*header)[0]) {
  311. case 'u':
  312. case 'U':
  313. if (auto extension = parse_unicode_locale_extension(lexer); extension.has_value())
  314. return Extension { extension.release_value() };
  315. break;
  316. case 't':
  317. case 'T':
  318. if (auto extension = parse_transformed_extension(lexer); extension.has_value())
  319. return Extension { extension.release_value() };
  320. break;
  321. default:
  322. if (auto extension = parse_other_extension(key, lexer); extension.has_value())
  323. return Extension { extension.release_value() };
  324. break;
  325. }
  326. }
  327. lexer.retreat(lexer.tell() - starting_position);
  328. return {};
  329. }
  330. static Vector<String> parse_private_use_extensions(GenericLexer& lexer)
  331. {
  332. // https://unicode.org/reports/tr35/#pu_extensions
  333. //
  334. // pu_extensions = = sep [xX] (sep alphanum{1,8})+ ;
  335. size_t starting_position = lexer.tell();
  336. auto header = consume_next_segment(lexer);
  337. if (!header.has_value())
  338. return {};
  339. auto parse_values = [&]() -> Vector<String> {
  340. Vector<String> extensions;
  341. while (true) {
  342. auto segment = consume_next_segment(lexer);
  343. if (!segment.has_value())
  344. break;
  345. if ((segment->length() < 1) || (segment->length() > 8) || !all_of(*segment, is_ascii_alphanumeric)) {
  346. lexer.retreat(segment->length() + 1);
  347. break;
  348. }
  349. extensions.append(*segment);
  350. }
  351. return extensions;
  352. };
  353. if ((header->length() == 1) && (((*header)[0] == 'x') || ((*header)[0] == 'X'))) {
  354. if (auto extensions = parse_values(); !extensions.is_empty())
  355. return extensions;
  356. }
  357. lexer.retreat(lexer.tell() - starting_position);
  358. return {};
  359. }
  360. Optional<LanguageID> parse_unicode_language_id(StringView language)
  361. {
  362. GenericLexer lexer { language };
  363. auto language_id = parse_unicode_language_id(lexer);
  364. if (!lexer.is_eof())
  365. return {};
  366. return language_id;
  367. }
  368. Optional<LocaleID> parse_unicode_locale_id(StringView locale)
  369. {
  370. GenericLexer lexer { locale };
  371. // https://unicode.org/reports/tr35/#Unicode_locale_identifier
  372. //
  373. // unicode_locale_id = unicode_language_id
  374. // extensions*
  375. // pu_extensions?
  376. auto language_id = parse_unicode_language_id(lexer);
  377. if (!language_id.has_value())
  378. return {};
  379. LocaleID locale_id { language_id.release_value() };
  380. while (true) {
  381. auto extension = parse_extension(lexer);
  382. if (!extension.has_value())
  383. break;
  384. locale_id.extensions.append(extension.release_value());
  385. }
  386. locale_id.private_use_extensions = parse_private_use_extensions(lexer);
  387. if (!lexer.is_eof())
  388. return {};
  389. return locale_id;
  390. }
  391. static void perform_hard_coded_key_value_substitutions(String& key, String& value)
  392. {
  393. // FIXME: In the XML export of CLDR, there are some aliases defined in the following files:
  394. // https://github.com/unicode-org/cldr-staging/blob/master/production/common/bcp47/calendar.xml
  395. // https://github.com/unicode-org/cldr-staging/blob/master/production/common/bcp47/collation.xml
  396. // https://github.com/unicode-org/cldr-staging/blob/master/production/common/bcp47/measure.xml
  397. // https://github.com/unicode-org/cldr-staging/blob/master/production/common/bcp47/timezone.xml
  398. // https://github.com/unicode-org/cldr-staging/blob/master/production/common/bcp47/transform.xml
  399. //
  400. // There doesn't seem to be a counterpart in the JSON export. Since there aren't many such
  401. // aliases, until an XML parser is implemented, those aliases are implemented here.
  402. if ((key == "ca"sv) && (value == "islamicc"sv)) {
  403. value = "islamic-civil"sv;
  404. } else if (key.is_one_of("kb"sv, "kc"sv, "kh"sv, "kk"sv, "kn"sv) && (value == "yes"sv)) {
  405. value = "true"sv;
  406. } else if (key == "ks"sv) {
  407. if (value == "primary"sv)
  408. value = "level1"sv;
  409. else if (value == "tertiary"sv)
  410. value = "level3"sv;
  411. // Note: There are also aliases for "secondary", "quaternary", "quarternary", and "identical",
  412. // but those are semantically incorrect values (they are too long), so they can be skipped.
  413. } else if ((key == "m0"sv) && (value == "names"sv)) {
  414. value = "prprname"sv;
  415. } else if ((key == "ms"sv) && (value == "imperial"sv)) {
  416. value = "uksystem"sv;
  417. } else if (key == "tz"sv) {
  418. // Formatter disabled because this block is easier to read / check against timezone.xml as one-liners.
  419. // clang-format off
  420. if (value == "aqams"sv) value = "nzakl"sv;
  421. else if (value == "cnckg"sv) value = "cnsha"sv;
  422. else if (value == "cnhrb"sv) value = "cnsha"sv;
  423. else if (value == "cnkhg"sv) value = "cnurc"sv;
  424. else if (value == "cuba"sv) value = "cuhav"sv;
  425. else if (value == "egypt"sv) value = "egcai"sv;
  426. else if (value == "eire"sv) value = "iedub"sv;
  427. else if (value == "est"sv) value = "utcw05"sv;
  428. else if (value == "gmt0"sv) value = "gmt"sv;
  429. else if (value == "hongkong"sv) value = "hkhkg"sv;
  430. else if (value == "hst"sv) value = "utcw10"sv;
  431. else if (value == "iceland"sv) value = "isrey"sv;
  432. else if (value == "iran"sv) value = "irthr"sv;
  433. else if (value == "israel"sv) value = "jeruslm"sv;
  434. else if (value == "jamaica"sv) value = "jmkin"sv;
  435. else if (value == "japan"sv) value = "jptyo"sv;
  436. else if (value == "kwajalein"sv) value = "mhkwa"sv;
  437. else if (value == "libya"sv) value = "lytip"sv;
  438. else if (value == "mst"sv) value = "utcw07"sv;
  439. else if (value == "navajo"sv) value = "usden"sv;
  440. else if (value == "poland"sv) value = "plwaw"sv;
  441. else if (value == "portugal"sv) value = "ptlis"sv;
  442. else if (value == "prc"sv) value = "cnsha"sv;
  443. else if (value == "roc"sv) value = "twtpe"sv;
  444. else if (value == "rok"sv) value = "krsel"sv;
  445. else if (value == "singapore"sv) value = "sgsin"sv;
  446. else if (value == "turkey"sv) value = "trist"sv;
  447. else if (value == "uct"sv) value = "utc"sv;
  448. else if (value == "usnavajo"sv) value = "usden"sv;
  449. else if (value == "zulu"sv) value = "utc"sv;
  450. // clang-format on
  451. }
  452. }
  453. static void perform_hard_coded_key_multi_value_substitutions(String const& key, Vector<String>& values)
  454. {
  455. // Similar to perform_hard_coded_key_value_substitutions, some aliases depend on multiple
  456. // variants being present in the original locale. Those are canonicalized separately here.
  457. // https://github.com/unicode-org/cldr-staging/blob/master/production/common/bcp47/calendar.xml
  458. if ((key != "ca"sv) || (values.size() != 3))
  459. return;
  460. static Vector<String> ethiopic_amete_alem { "ethiopic"sv, "amete"sv, "alem"sv };
  461. if (values == ethiopic_amete_alem)
  462. values = { "ethioaa"sv };
  463. }
  464. static void transform_unicode_locale_id_to_canonical_syntax(LocaleID& locale_id)
  465. {
  466. auto canonicalize_language = [](LanguageID& language_id, bool force_lowercase) {
  467. language_id.language = language_id.language->to_lowercase();
  468. if (language_id.script.has_value())
  469. language_id.script = language_id.script->to_titlecase();
  470. if (language_id.region.has_value())
  471. language_id.region = language_id.region->to_uppercase();
  472. for (auto& variant : language_id.variants)
  473. variant = variant.to_lowercase();
  474. #if ENABLE_UNICODE_DATA
  475. Detail::resolve_complex_language_aliases(language_id);
  476. #endif
  477. if (auto alias = resolve_language_alias(*language_id.language); alias.has_value()) {
  478. auto language_alias = parse_unicode_language_id(*alias);
  479. VERIFY(language_alias.has_value());
  480. language_id.language = move(language_alias->language);
  481. if (!language_id.script.has_value() && language_alias->script.has_value())
  482. language_id.script = move(language_alias->script);
  483. if (!language_id.region.has_value() && language_alias->region.has_value())
  484. language_id.region = move(language_alias->region);
  485. if (language_id.variants.is_empty() && !language_alias->variants.is_empty())
  486. language_id.variants = move(language_alias->variants);
  487. }
  488. if (language_id.script.has_value()) {
  489. if (auto alias = resolve_script_tag_alias(*language_id.script); alias.has_value())
  490. language_id.script = move(*alias);
  491. }
  492. if (language_id.region.has_value()) {
  493. if (auto alias = resolve_territory_alias(*language_id.region); alias.has_value()) {
  494. auto aliases = alias->split_view(' ');
  495. // FIXME: Territory subtag aliases should also consult the CLDR likelySubtags.json file.
  496. // For now, implement the spec's recommendation of using just the first alias.
  497. language_id.region = aliases[0].to_string();
  498. }
  499. }
  500. quick_sort(language_id.variants);
  501. for (auto& variant : language_id.variants) {
  502. variant = variant.to_lowercase();
  503. if (auto alias = resolve_variant_alias(variant); alias.has_value())
  504. variant = move(*alias);
  505. }
  506. if (force_lowercase) {
  507. if (language_id.script.has_value())
  508. language_id.script = language_id.script->to_lowercase();
  509. if (language_id.region.has_value())
  510. language_id.region = language_id.region->to_lowercase();
  511. }
  512. };
  513. auto canonicalize_key_value_list = [&](auto& key, auto& values, bool remove_true_values) {
  514. key = key.to_lowercase();
  515. auto raw_values = move(values);
  516. for (auto& value : raw_values) {
  517. value = value.to_lowercase();
  518. perform_hard_coded_key_value_substitutions(key, value);
  519. // Note: The spec says to remove "true" type and tfield values but that is believed to be a bug in the spec
  520. // because, for tvalues, that would result in invalid syntax:
  521. // https://unicode-org.atlassian.net/browse/CLDR-14318
  522. // This has also been noted by test262:
  523. // https://github.com/tc39/test262/blob/18bb955771669541c56c28748603f6afdb2e25ff/test/intl402/Intl/getCanonicalLocales/transformed-ext-canonical.js
  524. if (remove_true_values && (value == "true"sv))
  525. continue;
  526. if (key.is_one_of("sd"sv, "rg"sv)) {
  527. if (auto alias = resolve_subdivision_alias(value); alias.has_value()) {
  528. auto aliases = alias->split_view(' ');
  529. // FIXME: Subdivision subtags do not appear in the CLDR likelySubtags.json file.
  530. // Implement the spec's recommendation of using just the first alias for now,
  531. // but we should determine if there's anything else needed here.
  532. values.append(aliases[0].to_string());
  533. continue;
  534. }
  535. }
  536. values.append(move(value));
  537. }
  538. perform_hard_coded_key_multi_value_substitutions(key, values);
  539. };
  540. canonicalize_language(locale_id.language_id, false);
  541. quick_sort(locale_id.extensions, [](auto const& left, auto const& right) {
  542. auto key = [](auto const& extension) {
  543. return extension.visit(
  544. [](LocaleExtension const&) { return 'u'; },
  545. [](TransformedExtension const&) { return 't'; },
  546. [](OtherExtension const& ext) { return static_cast<char>(to_ascii_lowercase(ext.key)); });
  547. };
  548. return key(left) < key(right);
  549. });
  550. for (auto& extension : locale_id.extensions) {
  551. extension.visit(
  552. [&](LocaleExtension& ext) {
  553. for (auto& attribute : ext.attributes)
  554. attribute = attribute.to_lowercase();
  555. for (auto& keyword : ext.keywords)
  556. canonicalize_key_value_list(keyword.key, keyword.types, true);
  557. quick_sort(ext.attributes);
  558. quick_sort(ext.keywords, [](auto const& a, auto const& b) { return a.key < b.key; });
  559. },
  560. [&](TransformedExtension& ext) {
  561. if (ext.language.has_value())
  562. canonicalize_language(*ext.language, true);
  563. for (auto& field : ext.fields)
  564. canonicalize_key_value_list(field.key, field.values, false);
  565. quick_sort(ext.fields, [](auto const& a, auto const& b) { return a.key < b.key; });
  566. },
  567. [&](OtherExtension& ext) {
  568. ext.key = static_cast<char>(to_ascii_lowercase(ext.key));
  569. for (auto& value : ext.values)
  570. value = value.to_lowercase();
  571. });
  572. }
  573. for (auto& extension : locale_id.private_use_extensions)
  574. extension = extension.to_lowercase();
  575. }
  576. Optional<String> canonicalize_unicode_locale_id(LocaleID& locale_id)
  577. {
  578. // https://unicode.org/reports/tr35/#Canonical_Unicode_Locale_Identifiers
  579. StringBuilder builder;
  580. auto append_sep_and_string = [&](Optional<String> const& string) {
  581. if (!string.has_value())
  582. return;
  583. builder.appendff("-{}", *string);
  584. };
  585. if (!locale_id.language_id.language.has_value())
  586. return {};
  587. transform_unicode_locale_id_to_canonical_syntax(locale_id);
  588. builder.append(locale_id.language_id.language->to_lowercase());
  589. append_sep_and_string(locale_id.language_id.script);
  590. append_sep_and_string(locale_id.language_id.region);
  591. for (auto const& variant : locale_id.language_id.variants)
  592. append_sep_and_string(variant);
  593. auto append_key_value_list = [&](auto const& key, auto& values) {
  594. append_sep_and_string(key);
  595. for (auto& value : values)
  596. append_sep_and_string(value);
  597. };
  598. for (auto const& extension : locale_id.extensions) {
  599. extension.visit(
  600. [&](LocaleExtension const& ext) {
  601. builder.append("-u"sv);
  602. for (auto const& attribute : ext.attributes)
  603. append_sep_and_string(attribute);
  604. for (auto const& keyword : ext.keywords)
  605. append_key_value_list(keyword.key, keyword.types);
  606. },
  607. [&](TransformedExtension const& ext) {
  608. builder.append("-t"sv);
  609. if (ext.language.has_value()) {
  610. append_sep_and_string(ext.language->language);
  611. append_sep_and_string(ext.language->script);
  612. append_sep_and_string(ext.language->region);
  613. for (auto const& variant : ext.language->variants)
  614. append_sep_and_string(variant);
  615. }
  616. for (auto const& field : ext.fields)
  617. append_key_value_list(field.key, field.values);
  618. },
  619. [&](OtherExtension const& ext) {
  620. builder.appendff("-{:c}", to_ascii_lowercase(ext.key));
  621. for (auto const& value : ext.values)
  622. append_sep_and_string(value);
  623. });
  624. }
  625. if (!locale_id.private_use_extensions.is_empty()) {
  626. builder.append("-x"sv);
  627. for (auto const& extension : locale_id.private_use_extensions)
  628. append_sep_and_string(extension);
  629. }
  630. return builder.build();
  631. }
  632. String const& default_locale()
  633. {
  634. static String locale = "en"sv;
  635. return locale;
  636. }
  637. bool is_locale_available([[maybe_unused]] StringView locale)
  638. {
  639. #if ENABLE_UNICODE_DATA
  640. return Detail::locale_from_string(locale).has_value();
  641. #else
  642. return false;
  643. #endif
  644. }
  645. Optional<StringView> get_locale_language_mapping([[maybe_unused]] StringView locale, [[maybe_unused]] StringView language)
  646. {
  647. #if ENABLE_UNICODE_DATA
  648. return Detail::get_locale_language_mapping(locale, language);
  649. #else
  650. return {};
  651. #endif
  652. }
  653. Optional<StringView> get_locale_territory_mapping([[maybe_unused]] StringView locale, [[maybe_unused]] StringView territory)
  654. {
  655. #if ENABLE_UNICODE_DATA
  656. return Detail::get_locale_territory_mapping(locale, territory);
  657. #else
  658. return {};
  659. #endif
  660. }
  661. Optional<StringView> get_locale_script_mapping([[maybe_unused]] StringView locale, [[maybe_unused]] StringView script)
  662. {
  663. #if ENABLE_UNICODE_DATA
  664. return Detail::get_locale_script_tag_mapping(locale, script);
  665. #else
  666. return {};
  667. #endif
  668. }
  669. Optional<StringView> get_locale_currency_mapping([[maybe_unused]] StringView locale, [[maybe_unused]] StringView currency)
  670. {
  671. #if ENABLE_UNICODE_DATA
  672. return Detail::get_locale_currency_mapping(locale, currency);
  673. #else
  674. return {};
  675. #endif
  676. }
  677. Optional<StringView> resolve_language_alias(StringView language)
  678. {
  679. #if ENABLE_UNICODE_DATA
  680. return Detail::resolve_language_alias(language);
  681. #else
  682. return language;
  683. #endif
  684. }
  685. Optional<StringView> resolve_territory_alias(StringView territory)
  686. {
  687. #if ENABLE_UNICODE_DATA
  688. return Detail::resolve_territory_alias(territory);
  689. #else
  690. return territory;
  691. #endif
  692. }
  693. Optional<StringView> resolve_script_tag_alias(StringView script_tag)
  694. {
  695. #if ENABLE_UNICODE_DATA
  696. return Detail::resolve_script_tag_alias(script_tag);
  697. #else
  698. return script_tag;
  699. #endif
  700. }
  701. Optional<StringView> resolve_variant_alias(StringView variant)
  702. {
  703. #if ENABLE_UNICODE_DATA
  704. return Detail::resolve_variant_alias(variant);
  705. #else
  706. return variant;
  707. #endif
  708. }
  709. Optional<StringView> resolve_subdivision_alias(StringView subdivision)
  710. {
  711. #if ENABLE_UNICODE_DATA
  712. return Detail::resolve_subdivision_alias(subdivision);
  713. #else
  714. return subdivision;
  715. #endif
  716. }
  717. String resolve_most_likely_territory([[maybe_unused]] LanguageID const& language_id, StringView territory_alias)
  718. {
  719. auto aliases = territory_alias.split_view(' ');
  720. #if ENABLE_UNICODE_DATA
  721. if (aliases.size() > 1) {
  722. auto territory = Detail::resolve_most_likely_territory(language_id);
  723. if (territory.has_value() && aliases.contains_slow(*territory))
  724. return territory.release_value();
  725. }
  726. #endif
  727. return aliases[0].to_string();
  728. }
  729. }