GenerateUnicodeDateTimeFormat.cpp 83 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109
  1. /*
  2. * Copyright (c) 2021, Tim Flynn <trflynn89@pm.me>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include "GeneratorUtil.h"
  7. #include <AK/AllOf.h>
  8. #include <AK/CharacterTypes.h>
  9. #include <AK/Format.h>
  10. #include <AK/GenericLexer.h>
  11. #include <AK/HashFunctions.h>
  12. #include <AK/HashMap.h>
  13. #include <AK/JsonObject.h>
  14. #include <AK/JsonParser.h>
  15. #include <AK/JsonValue.h>
  16. #include <AK/LexicalPath.h>
  17. #include <AK/SourceGenerator.h>
  18. #include <AK/String.h>
  19. #include <AK/StringBuilder.h>
  20. #include <AK/Traits.h>
  21. #include <AK/Utf8View.h>
  22. #include <LibCore/ArgsParser.h>
  23. #include <LibCore/DirIterator.h>
  24. #include <LibCore/File.h>
  25. #include <LibUnicode/DateTimeFormat.h>
  26. using StringIndexType = u16;
  27. constexpr auto s_string_index_type = "u16"sv;
  28. using CalendarPatternIndexType = u16;
  29. constexpr auto s_calendar_pattern_index_type = "u16"sv;
  30. using CalendarPatternListIndexType = u8;
  31. constexpr auto s_calendar_pattern_list_index_type = "u8"sv;
  32. using CalendarRangePatternIndexType = u16;
  33. constexpr auto s_calendar_range_pattern_index_type = "u16"sv;
  34. using CalendarRangePatternListIndexType = u8;
  35. constexpr auto s_calendar_range_pattern_list_index_type = "u8"sv;
  36. using CalendarFormatIndexType = u8;
  37. constexpr auto s_calendar_format_index_type = "u8"sv;
  38. using SymbolListIndexType = u16;
  39. constexpr auto s_symbol_list_index_type = "u16"sv;
  40. using CalendarSymbolsIndexType = u16;
  41. constexpr auto s_calendar_symbols_index_type = "u16"sv;
  42. using CalendarSymbolsListIndexType = u8;
  43. constexpr auto s_calendar_symbols_list_index_type = "u8"sv;
  44. using CalendarIndexType = u8;
  45. constexpr auto s_calendar_index_type = "u8"sv;
  46. using TimeZoneIndexType = u16;
  47. constexpr auto s_time_zone_index_type = "u16"sv;
  48. using TimeZoneListIndexType = u8;
  49. constexpr auto s_time_zone_list_index_type = "u8"sv;
  50. using DayPeriodIndexType = u8;
  51. constexpr auto s_day_period_index_type = "u8"sv;
  52. using DayPeriodListIndexType = u8;
  53. constexpr auto s_day_period_list_index_type = "u8"sv;
  54. using HourCycleListIndexType = u8;
  55. constexpr auto s_hour_cycle_list_index_type = "u8"sv;
  56. struct CalendarPattern : public Unicode::CalendarPattern {
  57. bool contains_only_date_fields() const
  58. {
  59. return !day_period.has_value() && !hour.has_value() && !minute.has_value() && !second.has_value() && !fractional_second_digits.has_value() && !time_zone_name.has_value();
  60. }
  61. bool contains_only_time_fields() const
  62. {
  63. return !weekday.has_value() && !era.has_value() && !year.has_value() && !month.has_value() && !day.has_value();
  64. }
  65. unsigned hash() const
  66. {
  67. auto hash = pair_int_hash(pattern_index, pattern12_index);
  68. hash = pair_int_hash(hash, skeleton_index);
  69. auto hash_field = [&](auto const& field) {
  70. if (field.has_value())
  71. hash = pair_int_hash(hash, static_cast<u8>(*field));
  72. else
  73. hash = pair_int_hash(hash, -1);
  74. };
  75. hash_field(era);
  76. hash_field(year);
  77. hash_field(month);
  78. hash_field(weekday);
  79. hash_field(day);
  80. hash_field(day_period);
  81. hash_field(hour);
  82. hash_field(minute);
  83. hash_field(second);
  84. hash_field(fractional_second_digits);
  85. hash_field(time_zone_name);
  86. return hash;
  87. }
  88. bool operator==(CalendarPattern const& other) const
  89. {
  90. return (skeleton_index == other.skeleton_index)
  91. && (pattern_index == other.pattern_index)
  92. && (pattern12_index == other.pattern12_index)
  93. && (era == other.era)
  94. && (year == other.year)
  95. && (month == other.month)
  96. && (weekday == other.weekday)
  97. && (day == other.day)
  98. && (day_period == other.day_period)
  99. && (hour == other.hour)
  100. && (minute == other.minute)
  101. && (second == other.second)
  102. && (fractional_second_digits == other.fractional_second_digits)
  103. && (time_zone_name == other.time_zone_name);
  104. }
  105. StringIndexType skeleton_index { 0 };
  106. StringIndexType pattern_index { 0 };
  107. StringIndexType pattern12_index { 0 };
  108. };
  109. template<>
  110. struct AK::Formatter<CalendarPattern> : Formatter<FormatString> {
  111. ErrorOr<void> format(FormatBuilder& builder, CalendarPattern const& pattern)
  112. {
  113. auto field_to_i8 = [](auto const& field) -> i8 {
  114. if (!field.has_value())
  115. return -1;
  116. return static_cast<i8>(*field);
  117. };
  118. return Formatter<FormatString>::format(builder,
  119. "{{ {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {} }}",
  120. pattern.skeleton_index,
  121. pattern.pattern_index,
  122. pattern.pattern12_index,
  123. field_to_i8(pattern.era),
  124. field_to_i8(pattern.year),
  125. field_to_i8(pattern.month),
  126. field_to_i8(pattern.weekday),
  127. field_to_i8(pattern.day),
  128. field_to_i8(pattern.day_period),
  129. field_to_i8(pattern.hour),
  130. field_to_i8(pattern.minute),
  131. field_to_i8(pattern.second),
  132. field_to_i8(pattern.fractional_second_digits),
  133. field_to_i8(pattern.time_zone_name));
  134. }
  135. };
  136. template<>
  137. struct AK::Traits<CalendarPattern> : public GenericTraits<CalendarPattern> {
  138. static unsigned hash(CalendarPattern const& c) { return c.hash(); }
  139. };
  140. struct CalendarRangePattern : public CalendarPattern {
  141. unsigned hash() const
  142. {
  143. auto hash = CalendarPattern::hash();
  144. if (field.has_value())
  145. hash = pair_int_hash(hash, static_cast<u8>(*field));
  146. hash = pair_int_hash(hash, start_range);
  147. hash = pair_int_hash(hash, separator);
  148. hash = pair_int_hash(hash, end_range);
  149. return hash;
  150. }
  151. bool operator==(CalendarRangePattern const& other) const
  152. {
  153. if (!CalendarPattern::operator==(other))
  154. return false;
  155. return (field == other.field)
  156. && (start_range == other.start_range)
  157. && (separator == other.separator)
  158. && (end_range == other.end_range);
  159. }
  160. Optional<Unicode::CalendarRangePattern::Field> field {};
  161. StringIndexType start_range { 0 };
  162. StringIndexType separator { 0 };
  163. StringIndexType end_range { 0 };
  164. };
  165. template<>
  166. struct AK::Formatter<CalendarRangePattern> : Formatter<FormatString> {
  167. ErrorOr<void> format(FormatBuilder& builder, CalendarRangePattern const& pattern)
  168. {
  169. auto field_to_i8 = [](auto const& field) -> i8 {
  170. if (!field.has_value())
  171. return -1;
  172. return static_cast<i8>(*field);
  173. };
  174. return Formatter<FormatString>::format(builder,
  175. "{{ {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {} }}",
  176. pattern.skeleton_index,
  177. field_to_i8(pattern.field),
  178. pattern.start_range,
  179. pattern.separator,
  180. pattern.end_range,
  181. field_to_i8(pattern.era),
  182. field_to_i8(pattern.year),
  183. field_to_i8(pattern.month),
  184. field_to_i8(pattern.weekday),
  185. field_to_i8(pattern.day),
  186. field_to_i8(pattern.day_period),
  187. field_to_i8(pattern.hour),
  188. field_to_i8(pattern.minute),
  189. field_to_i8(pattern.second),
  190. field_to_i8(pattern.fractional_second_digits),
  191. field_to_i8(pattern.time_zone_name));
  192. }
  193. };
  194. template<>
  195. struct AK::Traits<CalendarRangePattern> : public GenericTraits<CalendarRangePattern> {
  196. static unsigned hash(CalendarRangePattern const& c) { return c.hash(); }
  197. };
  198. struct CalendarFormat {
  199. unsigned hash() const
  200. {
  201. auto hash = pair_int_hash(full_format, long_format);
  202. hash = pair_int_hash(hash, medium_format);
  203. hash = pair_int_hash(hash, short_format);
  204. return hash;
  205. }
  206. bool operator==(CalendarFormat const& other) const
  207. {
  208. return (full_format == other.full_format)
  209. && (long_format == other.long_format)
  210. && (medium_format == other.medium_format)
  211. && (short_format == other.short_format);
  212. }
  213. CalendarPatternIndexType full_format { 0 };
  214. CalendarPatternIndexType long_format { 0 };
  215. CalendarPatternIndexType medium_format { 0 };
  216. CalendarPatternIndexType short_format { 0 };
  217. };
  218. template<>
  219. struct AK::Formatter<CalendarFormat> : Formatter<FormatString> {
  220. ErrorOr<void> format(FormatBuilder& builder, CalendarFormat const& pattern)
  221. {
  222. return Formatter<FormatString>::format(builder,
  223. "{{ {}, {}, {}, {} }}",
  224. pattern.full_format,
  225. pattern.long_format,
  226. pattern.medium_format,
  227. pattern.short_format);
  228. }
  229. };
  230. template<>
  231. struct AK::Traits<CalendarFormat> : public GenericTraits<CalendarFormat> {
  232. static unsigned hash(CalendarFormat const& c) { return c.hash(); }
  233. };
  234. using SymbolList = Vector<StringIndexType>;
  235. struct CalendarSymbols {
  236. unsigned hash() const
  237. {
  238. auto hash = pair_int_hash(narrow_symbols, short_symbols);
  239. hash = pair_int_hash(hash, long_symbols);
  240. return hash;
  241. }
  242. bool operator==(CalendarSymbols const& other) const
  243. {
  244. return (narrow_symbols == other.narrow_symbols)
  245. && (short_symbols == other.short_symbols)
  246. && (long_symbols == other.long_symbols);
  247. }
  248. SymbolListIndexType narrow_symbols { 0 };
  249. SymbolListIndexType short_symbols { 0 };
  250. SymbolListIndexType long_symbols { 0 };
  251. };
  252. template<>
  253. struct AK::Formatter<CalendarSymbols> : Formatter<FormatString> {
  254. ErrorOr<void> format(FormatBuilder& builder, CalendarSymbols const& symbols)
  255. {
  256. return Formatter<FormatString>::format(builder,
  257. "{{ {}, {}, {} }}",
  258. symbols.narrow_symbols,
  259. symbols.short_symbols,
  260. symbols.long_symbols);
  261. }
  262. };
  263. template<>
  264. struct AK::Traits<CalendarSymbols> : public GenericTraits<CalendarSymbols> {
  265. static unsigned hash(CalendarSymbols const& c) { return c.hash(); }
  266. };
  267. using CalendarPatternList = Vector<CalendarPatternIndexType>;
  268. using CalendarRangePatternList = Vector<CalendarRangePatternIndexType>;
  269. using CalendarSymbolsList = Vector<CalendarSymbolsIndexType>;
  270. struct Calendar {
  271. unsigned hash() const
  272. {
  273. auto hash = int_hash(date_formats);
  274. hash = pair_int_hash(hash, time_formats);
  275. hash = pair_int_hash(hash, date_time_formats);
  276. hash = pair_int_hash(hash, available_formats);
  277. hash = pair_int_hash(hash, default_range_format);
  278. hash = pair_int_hash(hash, range_formats);
  279. hash = pair_int_hash(hash, range12_formats);
  280. hash = pair_int_hash(hash, symbols);
  281. return hash;
  282. }
  283. bool operator==(Calendar const& other) const
  284. {
  285. return (date_formats == other.date_formats)
  286. && (time_formats == other.time_formats)
  287. && (date_time_formats == other.date_time_formats)
  288. && (available_formats == other.available_formats)
  289. && (default_range_format == other.default_range_format)
  290. && (range_formats == other.range_formats)
  291. && (range12_formats == other.range12_formats)
  292. && (symbols == other.symbols);
  293. }
  294. CalendarFormatIndexType date_formats { 0 };
  295. CalendarFormatIndexType time_formats { 0 };
  296. CalendarFormatIndexType date_time_formats { 0 };
  297. CalendarPatternListIndexType available_formats { 0 };
  298. CalendarRangePatternIndexType default_range_format { 0 };
  299. CalendarRangePatternListIndexType range_formats { 0 };
  300. CalendarRangePatternListIndexType range12_formats { 0 };
  301. CalendarSymbolsListIndexType symbols { 0 };
  302. };
  303. template<>
  304. struct AK::Formatter<Calendar> : Formatter<FormatString> {
  305. ErrorOr<void> format(FormatBuilder& builder, Calendar const& calendar)
  306. {
  307. return Formatter<FormatString>::format(builder,
  308. "{{ {}, {}, {}, {}, {}, {}, {}, {} }}",
  309. calendar.date_formats,
  310. calendar.time_formats,
  311. calendar.date_time_formats,
  312. calendar.available_formats,
  313. calendar.default_range_format,
  314. calendar.range_formats,
  315. calendar.range12_formats,
  316. calendar.symbols);
  317. }
  318. };
  319. template<>
  320. struct AK::Traits<Calendar> : public GenericTraits<Calendar> {
  321. static unsigned hash(Calendar const& c) { return c.hash(); }
  322. };
  323. struct TimeZone {
  324. unsigned hash() const
  325. {
  326. return pair_int_hash(long_name, short_name);
  327. }
  328. bool operator==(TimeZone const& other) const
  329. {
  330. return (long_name == other.long_name) && (short_name == other.short_name);
  331. }
  332. StringIndexType long_name { 0 };
  333. StringIndexType short_name { 0 };
  334. };
  335. template<>
  336. struct AK::Formatter<TimeZone> : Formatter<FormatString> {
  337. ErrorOr<void> format(FormatBuilder& builder, TimeZone const& time_zone)
  338. {
  339. return Formatter<FormatString>::format(builder,
  340. "{{ {}, {} }}",
  341. time_zone.long_name,
  342. time_zone.short_name);
  343. }
  344. };
  345. template<>
  346. struct AK::Traits<TimeZone> : public GenericTraits<TimeZone> {
  347. static unsigned hash(TimeZone const& t) { return t.hash(); }
  348. };
  349. struct DayPeriod {
  350. unsigned hash() const
  351. {
  352. auto hash = int_hash(static_cast<u8>(day_period));
  353. hash = pair_int_hash(hash, begin);
  354. hash = pair_int_hash(hash, end);
  355. return hash;
  356. }
  357. bool operator==(DayPeriod const& other) const
  358. {
  359. return (day_period == other.day_period)
  360. && (begin == other.begin)
  361. && (end == other.end);
  362. }
  363. Unicode::DayPeriod day_period {};
  364. u8 begin { 0 };
  365. u8 end { 0 };
  366. };
  367. template<>
  368. struct AK::Formatter<DayPeriod> : Formatter<FormatString> {
  369. ErrorOr<void> format(FormatBuilder& builder, DayPeriod const& day_period)
  370. {
  371. return Formatter<FormatString>::format(builder,
  372. "{{ {}, {}, {} }}",
  373. static_cast<u8>(day_period.day_period),
  374. day_period.begin,
  375. day_period.end);
  376. }
  377. };
  378. template<>
  379. struct AK::Traits<DayPeriod> : public GenericTraits<DayPeriod> {
  380. static unsigned hash(DayPeriod const& d) { return d.hash(); }
  381. };
  382. using TimeZoneList = Vector<TimeZoneIndexType>;
  383. using DayPeriodList = Vector<DayPeriodIndexType>;
  384. using HourCycleList = Vector<Unicode::HourCycle>;
  385. template<>
  386. struct AK::Formatter<Unicode::HourCycle> : Formatter<FormatString> {
  387. ErrorOr<void> format(FormatBuilder& builder, Unicode::HourCycle hour_cycle)
  388. {
  389. return Formatter<FormatString>::format(builder, "{}", to_underlying(hour_cycle));
  390. }
  391. };
  392. struct Locale {
  393. HashMap<String, CalendarIndexType> calendars;
  394. TimeZoneListIndexType time_zones { 0 };
  395. DayPeriodListIndexType day_periods { 0 };
  396. };
  397. struct UnicodeLocaleData {
  398. UniqueStringStorage<StringIndexType> unique_strings;
  399. UniqueStorage<CalendarPattern, CalendarPatternIndexType> unique_patterns;
  400. UniqueStorage<CalendarPatternList, CalendarPatternListIndexType> unique_pattern_lists;
  401. UniqueStorage<CalendarRangePattern, CalendarRangePatternIndexType> unique_range_patterns;
  402. UniqueStorage<CalendarRangePatternList, CalendarRangePatternListIndexType> unique_range_pattern_lists;
  403. UniqueStorage<CalendarFormat, CalendarFormatIndexType> unique_formats;
  404. UniqueStorage<SymbolList, SymbolListIndexType> unique_symbol_lists;
  405. UniqueStorage<CalendarSymbols, CalendarSymbolsIndexType> unique_calendar_symbols;
  406. UniqueStorage<CalendarSymbolsList, CalendarSymbolsListIndexType> unique_calendar_symbols_lists;
  407. UniqueStorage<Calendar, CalendarIndexType> unique_calendars;
  408. UniqueStorage<TimeZone, TimeZoneIndexType> unique_time_zones;
  409. UniqueStorage<TimeZoneList, TimeZoneListIndexType> unique_time_zone_lists;
  410. UniqueStorage<DayPeriod, DayPeriodIndexType> unique_day_periods;
  411. UniqueStorage<DayPeriodList, DayPeriodListIndexType> unique_day_period_lists;
  412. UniqueStorage<HourCycleList, HourCycleListIndexType> unique_hour_cycle_lists;
  413. HashMap<String, Locale> locales;
  414. HashMap<String, HourCycleListIndexType> hour_cycles;
  415. Vector<String> hour_cycle_regions;
  416. HashMap<String, String> meta_zones;
  417. Vector<String> time_zones { "UTC"sv };
  418. Vector<String> calendars;
  419. Vector<Alias> calendar_aliases {
  420. // FIXME: Aliases should come from BCP47. See: https://unicode-org.atlassian.net/browse/CLDR-15158
  421. { "gregorian"sv, "gregory"sv },
  422. };
  423. };
  424. static Optional<Unicode::DayPeriod> day_period_from_string(StringView day_period)
  425. {
  426. if (day_period == "am"sv)
  427. return Unicode::DayPeriod::AM;
  428. if (day_period == "pm"sv)
  429. return Unicode::DayPeriod::PM;
  430. if (day_period == "morning1"sv)
  431. return Unicode::DayPeriod::Morning1;
  432. if (day_period == "morning2"sv)
  433. return Unicode::DayPeriod::Morning2;
  434. if (day_period == "afternoon1"sv)
  435. return Unicode::DayPeriod::Afternoon1;
  436. if (day_period == "afternoon2"sv)
  437. return Unicode::DayPeriod::Afternoon2;
  438. if (day_period == "evening1"sv)
  439. return Unicode::DayPeriod::Evening1;
  440. if (day_period == "evening2"sv)
  441. return Unicode::DayPeriod::Evening2;
  442. if (day_period == "night1"sv)
  443. return Unicode::DayPeriod::Night1;
  444. if (day_period == "night2"sv)
  445. return Unicode::DayPeriod::Night2;
  446. return {};
  447. };
  448. static ErrorOr<void> parse_hour_cycles(String core_path, UnicodeLocaleData& locale_data)
  449. {
  450. // https://unicode.org/reports/tr35/tr35-dates.html#Time_Data
  451. LexicalPath time_data_path(move(core_path));
  452. time_data_path = time_data_path.append("supplemental"sv);
  453. time_data_path = time_data_path.append("timeData.json"sv);
  454. auto time_data_file = TRY(Core::File::open(time_data_path.string(), Core::OpenMode::ReadOnly));
  455. auto time_data = TRY(JsonValue::from_string(time_data_file->read_all()));
  456. auto const& supplemental_object = time_data.as_object().get("supplemental"sv);
  457. auto const& time_data_object = supplemental_object.as_object().get("timeData"sv);
  458. auto parse_hour_cycle = [](StringView hour_cycle) -> Optional<Unicode::HourCycle> {
  459. if (hour_cycle == "h"sv)
  460. return Unicode::HourCycle::H12;
  461. if (hour_cycle == "H"sv)
  462. return Unicode::HourCycle::H23;
  463. if (hour_cycle == "K"sv)
  464. return Unicode::HourCycle::H11;
  465. if (hour_cycle == "k"sv)
  466. return Unicode::HourCycle::H24;
  467. return {};
  468. };
  469. time_data_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  470. auto allowed_hour_cycles_string = value.as_object().get("_allowed"sv).as_string();
  471. auto allowed_hour_cycles = allowed_hour_cycles_string.split_view(' ');
  472. Vector<Unicode::HourCycle> hour_cycles;
  473. for (auto allowed_hour_cycle : allowed_hour_cycles) {
  474. if (auto hour_cycle = parse_hour_cycle(allowed_hour_cycle); hour_cycle.has_value())
  475. hour_cycles.append(*hour_cycle);
  476. }
  477. auto hour_cycles_index = locale_data.unique_hour_cycle_lists.ensure(move(hour_cycles));
  478. locale_data.hour_cycles.set(key, hour_cycles_index);
  479. if (!locale_data.hour_cycle_regions.contains_slow(key))
  480. locale_data.hour_cycle_regions.append(key);
  481. });
  482. return {};
  483. };
  484. static ErrorOr<void> parse_meta_zones(String core_path, UnicodeLocaleData& locale_data)
  485. {
  486. // https://unicode.org/reports/tr35/tr35-dates.html#Metazones
  487. LexicalPath meta_zone_path(move(core_path));
  488. meta_zone_path = meta_zone_path.append("supplemental"sv);
  489. meta_zone_path = meta_zone_path.append("metaZones.json"sv);
  490. auto meta_zone_file = TRY(Core::File::open(meta_zone_path.string(), Core::OpenMode::ReadOnly));
  491. auto meta_zone = TRY(JsonValue::from_string(meta_zone_file->read_all()));
  492. auto const& supplemental_object = meta_zone.as_object().get("supplemental"sv);
  493. auto const& meta_zone_object = supplemental_object.as_object().get("metaZones"sv);
  494. auto const& meta_zone_array = meta_zone_object.as_object().get("metazones"sv);
  495. meta_zone_array.as_array().for_each([&](JsonValue const& value) {
  496. auto const& mapping = value.as_object().get("mapZone"sv);
  497. auto const& meta_zone = mapping.as_object().get("_other"sv);
  498. auto const& golden_zone = mapping.as_object().get("_type"sv);
  499. locale_data.meta_zones.set(meta_zone.as_string(), golden_zone.as_string());
  500. });
  501. // UTC does not appear in metaZones.json. Define it for convenience so other parsers don't need to check for its existence.
  502. locale_data.meta_zones.set("UTC"sv, "UTC"sv);
  503. return {};
  504. };
  505. static constexpr auto is_char(char ch)
  506. {
  507. return [ch](auto c) { return c == ch; };
  508. }
  509. // For patterns that are 12-hour aware, we need to generate two patterns: one with the day period
  510. // (e.g. {ampm}) in the pattern, and one without the day period. We need to take care to remove
  511. // extra spaces around the day period. Some example expected removals:
  512. //
  513. // "{hour}:{minute} {ampm}" becomes "{hour}:{minute}" (remove the space before {ampm})
  514. // "{ampm} {hour}" becomes "{hour}" (remove the space after {ampm})
  515. // "{hour}:{minute} {ampm} {timeZoneName}" becomes "{hour}:{minute} {timeZoneName}" (remove one of the spaces around {ampm})
  516. static String remove_period_from_pattern(String pattern)
  517. {
  518. for (auto remove : AK::Array { "({ampm})"sv, "{ampm}"sv, "({dayPeriod})"sv, "{dayPeriod}"sv }) {
  519. auto index = pattern.find(remove);
  520. if (!index.has_value())
  521. continue;
  522. constexpr u32 space = ' ';
  523. constexpr u32 open = '{';
  524. constexpr u32 close = '}';
  525. Utf8View utf8_pattern { pattern };
  526. Optional<u32> before_removal;
  527. Optional<u32> after_removal;
  528. for (auto it = utf8_pattern.begin(); utf8_pattern.byte_offset_of(it) < *index; ++it)
  529. before_removal = *it;
  530. if (auto it = utf8_pattern.iterator_at_byte_offset(*index + remove.length()); it != utf8_pattern.end())
  531. after_removal = *it;
  532. if ((before_removal == space) && (after_removal != open)) {
  533. pattern = String::formatted("{}{}",
  534. pattern.substring_view(0, *index - 1),
  535. pattern.substring_view(*index + remove.length()));
  536. } else if ((after_removal == space) && (before_removal != close)) {
  537. pattern = String::formatted("{}{}",
  538. pattern.substring_view(0, *index),
  539. pattern.substring_view(*index + remove.length() + 1));
  540. } else {
  541. pattern = String::formatted("{}{}",
  542. pattern.substring_view(0, *index),
  543. pattern.substring_view(*index + remove.length()));
  544. }
  545. }
  546. return pattern;
  547. }
  548. static Optional<CalendarPattern> parse_date_time_pattern_raw(String pattern, String skeleton, UnicodeLocaleData& locale_data)
  549. {
  550. // https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table
  551. using Unicode::CalendarPatternStyle;
  552. CalendarPattern format {};
  553. if (!skeleton.is_empty())
  554. format.skeleton_index = locale_data.unique_strings.ensure(move(skeleton));
  555. GenericLexer lexer { pattern };
  556. StringBuilder builder;
  557. bool hour12 { false };
  558. while (!lexer.is_eof()) {
  559. // Literal strings enclosed by quotes are to be appended to the pattern as-is without further
  560. // processing (this just avoids conflicts with the patterns below).
  561. if (lexer.next_is(is_quote)) {
  562. builder.append(lexer.consume_quoted_string());
  563. continue;
  564. }
  565. auto starting_char = lexer.peek();
  566. auto segment = lexer.consume_while([&](char ch) { return ch == starting_char; });
  567. // Era
  568. if (all_of(segment, is_char('G'))) {
  569. builder.append("{era}");
  570. if (segment.length() <= 3)
  571. format.era = CalendarPatternStyle::Short;
  572. else if (segment.length() == 4)
  573. format.era = CalendarPatternStyle::Long;
  574. else
  575. format.era = CalendarPatternStyle::Narrow;
  576. }
  577. // Year
  578. else if (all_of(segment, is_any_of("yYuUr"sv))) {
  579. builder.append("{year}");
  580. if (segment.length() == 2)
  581. format.year = CalendarPatternStyle::TwoDigit;
  582. else
  583. format.year = CalendarPatternStyle::Numeric;
  584. }
  585. // Quarter
  586. else if (all_of(segment, is_any_of("qQ"sv))) {
  587. // Intl.DateTimeFormat does not support quarter formatting, so drop these patterns.
  588. return {};
  589. }
  590. // Month
  591. else if (all_of(segment, is_any_of("ML"sv))) {
  592. builder.append("{month}");
  593. if (segment.length() == 1)
  594. format.month = CalendarPatternStyle::Numeric;
  595. else if (segment.length() == 2)
  596. format.month = CalendarPatternStyle::TwoDigit;
  597. else if (segment.length() == 3)
  598. format.month = CalendarPatternStyle::Short;
  599. else if (segment.length() == 4)
  600. format.month = CalendarPatternStyle::Long;
  601. else if (segment.length() == 5)
  602. format.month = CalendarPatternStyle::Narrow;
  603. } else if (all_of(segment, is_char('l'))) {
  604. // Using 'l' for month formatting is deprecated by TR-35, ensure it is not used.
  605. return {};
  606. }
  607. // Week
  608. else if (all_of(segment, is_any_of("wW"sv))) {
  609. // Intl.DateTimeFormat does not support week formatting, so drop these patterns.
  610. return {};
  611. }
  612. // Day
  613. else if (all_of(segment, is_char('d'))) {
  614. builder.append("{day}");
  615. if (segment.length() == 1)
  616. format.day = CalendarPatternStyle::Numeric;
  617. else
  618. format.day = CalendarPatternStyle::TwoDigit;
  619. } else if (all_of(segment, is_any_of("DFg"sv))) {
  620. builder.append("{day}");
  621. format.day = CalendarPatternStyle::Numeric;
  622. }
  623. // Weekday
  624. else if (all_of(segment, is_char('E'))) {
  625. builder.append("{weekday}");
  626. if (segment.length() == 4)
  627. format.weekday = CalendarPatternStyle::Long;
  628. else if (segment.length() == 5)
  629. format.weekday = CalendarPatternStyle::Narrow;
  630. else
  631. format.weekday = CalendarPatternStyle::Short;
  632. } else if (all_of(segment, is_any_of("ec"sv))) {
  633. builder.append("{weekday}");
  634. // TR-35 defines "e", "c", and "cc" as as numeric, and "ee" as 2-digit, but those
  635. // pattern styles are not supported by Intl.DateTimeFormat.
  636. if (segment.length() <= 2)
  637. return {};
  638. if (segment.length() == 4)
  639. format.weekday = CalendarPatternStyle::Long;
  640. else if (segment.length() == 5)
  641. format.weekday = CalendarPatternStyle::Narrow;
  642. else
  643. format.weekday = CalendarPatternStyle::Short;
  644. }
  645. // Period
  646. else if (all_of(segment, is_any_of("ab"sv))) {
  647. builder.append("{ampm}");
  648. hour12 = true;
  649. } else if (all_of(segment, is_char('B'))) {
  650. builder.append("{dayPeriod}");
  651. hour12 = true;
  652. if (segment.length() == 4)
  653. format.day_period = CalendarPatternStyle::Long;
  654. else if (segment.length() == 5)
  655. format.day_period = CalendarPatternStyle::Narrow;
  656. else
  657. format.day_period = CalendarPatternStyle::Short;
  658. }
  659. // Hour
  660. else if (all_of(segment, is_any_of("hHKk"sv))) {
  661. builder.append("{hour}");
  662. if ((segment[0] == 'h') || (segment[0] == 'K'))
  663. hour12 = true;
  664. if (segment.length() == 1)
  665. format.hour = CalendarPatternStyle::Numeric;
  666. else
  667. format.hour = CalendarPatternStyle::TwoDigit;
  668. } else if (all_of(segment, is_any_of("jJC"sv))) {
  669. // TR-35 indicates these should not be used.
  670. return {};
  671. }
  672. // Minute
  673. else if (all_of(segment, is_char('m'))) {
  674. builder.append("{minute}");
  675. if (segment.length() == 1)
  676. format.minute = CalendarPatternStyle::Numeric;
  677. else
  678. format.minute = CalendarPatternStyle::TwoDigit;
  679. }
  680. // Second
  681. else if (all_of(segment, is_char('s'))) {
  682. builder.append("{second}");
  683. if (segment.length() == 1)
  684. format.second = CalendarPatternStyle::Numeric;
  685. else
  686. format.second = CalendarPatternStyle::TwoDigit;
  687. } else if (all_of(segment, is_char('S'))) {
  688. builder.append("{fractionalSecondDigits}");
  689. VERIFY(segment.length() <= 3);
  690. format.fractional_second_digits = static_cast<u8>(segment.length());
  691. } else if (all_of(segment, is_char('A'))) {
  692. // Intl.DateTimeFormat does not support millisecond formatting, so drop these patterns.
  693. return {};
  694. }
  695. // Zone
  696. else if (all_of(segment, is_any_of("zZOvVXx"))) {
  697. builder.append("{timeZoneName}");
  698. if (segment.length() < 4)
  699. format.time_zone_name = CalendarPatternStyle::Short;
  700. else
  701. format.time_zone_name = CalendarPatternStyle::Long;
  702. }
  703. // Non-patterns
  704. else {
  705. builder.append(segment);
  706. }
  707. }
  708. pattern = builder.build();
  709. if (hour12) {
  710. format.pattern = remove_period_from_pattern(pattern);
  711. format.pattern12 = move(pattern);
  712. } else {
  713. format.pattern = move(pattern);
  714. }
  715. return format;
  716. }
  717. static Optional<CalendarPatternIndexType> parse_date_time_pattern(String pattern, String skeleton, UnicodeLocaleData& locale_data)
  718. {
  719. auto format = parse_date_time_pattern_raw(move(pattern), move(skeleton), locale_data);
  720. if (!format.has_value())
  721. return {};
  722. format->pattern_index = locale_data.unique_strings.ensure(move(format->pattern));
  723. if (format->pattern12.has_value())
  724. format->pattern12_index = locale_data.unique_strings.ensure(format->pattern12.release_value());
  725. return locale_data.unique_patterns.ensure(format.release_value());
  726. }
  727. template<typename... Chars>
  728. static constexpr bool char_is_one_of(char ch, Chars&&... chars)
  729. {
  730. return ((ch == chars) || ...);
  731. }
  732. static void parse_interval_patterns(Calendar& calendar, JsonObject const& interval_formats_object, UnicodeLocaleData& locale_data)
  733. {
  734. // https://unicode.org/reports/tr35/tr35-dates.html#intervalFormats
  735. CalendarRangePatternList range_formats {};
  736. CalendarRangePatternList range12_formats {};
  737. auto name_of_field = [&](char field) {
  738. if (char_is_one_of(field, 'G'))
  739. return Unicode::CalendarRangePattern::Field::Era;
  740. if (char_is_one_of(field, 'y', 'Y', 'u', 'U', 'r'))
  741. return Unicode::CalendarRangePattern::Field::Year;
  742. if (char_is_one_of(field, 'M', 'L'))
  743. return Unicode::CalendarRangePattern::Field::Month;
  744. if (char_is_one_of(field, 'd', 'D', 'F', 'g'))
  745. return Unicode::CalendarRangePattern::Field::Day;
  746. if (char_is_one_of(field, 'a', 'b'))
  747. return Unicode::CalendarRangePattern::Field::AmPm;
  748. if (char_is_one_of(field, 'B'))
  749. return Unicode::CalendarRangePattern::Field::DayPeriod;
  750. if (char_is_one_of(field, 'h', 'H', 'K', 'k'))
  751. return Unicode::CalendarRangePattern::Field::Hour;
  752. if (char_is_one_of(field, 'm'))
  753. return Unicode::CalendarRangePattern::Field::Minute;
  754. if (char_is_one_of(field, 's'))
  755. return Unicode::CalendarRangePattern::Field::Second;
  756. if (char_is_one_of(field, 'S'))
  757. return Unicode::CalendarRangePattern::Field::FractionalSecondDigits;
  758. VERIFY_NOT_REACHED();
  759. };
  760. auto split_default_range_pattern = [&](auto skeleton, auto const& pattern) {
  761. auto start_range_end = pattern.find('}').value() + 1;
  762. auto end_range_begin = pattern.find_last('{').value();
  763. auto start_range = pattern.substring_view(0, start_range_end);
  764. auto separator = pattern.substring_view(start_range_end, end_range_begin - start_range_end);
  765. auto end_range = pattern.substring_view(end_range_begin);
  766. CalendarRangePattern format {};
  767. format.skeleton_index = locale_data.unique_strings.ensure(skeleton);
  768. format.start_range = locale_data.unique_strings.ensure(start_range);
  769. format.separator = locale_data.unique_strings.ensure(separator);
  770. format.end_range = locale_data.unique_strings.ensure(end_range);
  771. return format;
  772. };
  773. auto split_range_pattern = [&](auto skeleton, auto field, auto const& pattern, auto const& parsed_fields) {
  774. HashMap<StringView, size_t> partitions;
  775. StringView last_partition;
  776. auto begin_index = pattern.find('{');
  777. size_t end_index = 0;
  778. while (begin_index.has_value()) {
  779. end_index = pattern.find('}', *begin_index).value();
  780. auto partition = pattern.substring_view(*begin_index, end_index - *begin_index);
  781. if (partitions.contains(partition))
  782. break;
  783. partitions.set(partition, *begin_index);
  784. last_partition = partition;
  785. begin_index = pattern.find('{', end_index + 1);
  786. }
  787. VERIFY(begin_index.has_value() && !last_partition.is_empty());
  788. auto start_range_end = partitions.get(last_partition).value() + last_partition.length() + 1;
  789. auto start_range = pattern.substring_view(0, start_range_end);
  790. auto separator = pattern.substring_view(start_range_end, *begin_index - start_range_end);
  791. auto end_range = pattern.substring_view(*begin_index);
  792. CalendarRangePattern format {};
  793. format.skeleton_index = locale_data.unique_strings.ensure(skeleton);
  794. format.field = field;
  795. format.start_range = locale_data.unique_strings.ensure(start_range);
  796. format.separator = locale_data.unique_strings.ensure(separator);
  797. format.end_range = locale_data.unique_strings.ensure(end_range);
  798. format.for_each_calendar_field_zipped_with(parsed_fields, [](auto& format_field, auto const& parsed_field, auto) {
  799. format_field = parsed_field;
  800. });
  801. return format;
  802. };
  803. interval_formats_object.for_each_member([&](auto const& skeleton, auto const& value) {
  804. if (skeleton == "intervalFormatFallback"sv) {
  805. auto range_format = split_default_range_pattern(skeleton, value.as_string());
  806. calendar.default_range_format = locale_data.unique_range_patterns.ensure(move(range_format));
  807. return;
  808. }
  809. value.as_object().for_each_member([&](auto const& field, auto const& pattern) {
  810. if (field.ends_with("alt-variant"sv))
  811. return;
  812. VERIFY(field.length() == 1);
  813. auto name = name_of_field(field[0]);
  814. auto format = parse_date_time_pattern_raw(pattern.as_string(), skeleton, locale_data).release_value();
  815. auto range_format = split_range_pattern(skeleton, name, format.pattern, format);
  816. range_formats.append(locale_data.unique_range_patterns.ensure(move(range_format)));
  817. if (format.pattern12.has_value()) {
  818. auto range12_pattern = split_range_pattern(skeleton, name, *format.pattern12, format);
  819. range12_formats.append(locale_data.unique_range_patterns.ensure(move(range12_pattern)));
  820. } else {
  821. range12_formats.append(range_formats.last());
  822. }
  823. });
  824. });
  825. calendar.range_formats = locale_data.unique_range_pattern_lists.ensure(move(range_formats));
  826. calendar.range12_formats = locale_data.unique_range_pattern_lists.ensure(move(range12_formats));
  827. }
  828. static void generate_missing_patterns(Calendar& calendar, CalendarPatternList& formats, Vector<CalendarPattern> date_formats, Vector<CalendarPattern> time_formats, UnicodeLocaleData& locale_data)
  829. {
  830. // https://unicode.org/reports/tr35/tr35-dates.html#Missing_Skeleton_Fields
  831. auto replace_pattern = [&](auto format, auto time_format, auto date_format) {
  832. auto pattern = locale_data.unique_strings.get(format);
  833. auto time_pattern = locale_data.unique_strings.get(time_format);
  834. auto date_pattern = locale_data.unique_strings.get(date_format);
  835. auto new_pattern = pattern.replace("{0}", time_pattern).replace("{1}", date_pattern);
  836. return locale_data.unique_strings.ensure(move(new_pattern));
  837. };
  838. auto inject_fractional_second_digits = [&](auto format) {
  839. auto pattern = locale_data.unique_strings.get(format);
  840. auto new_pattern = pattern.replace("{second}"sv, "{second}{decimal}{fractionalSecondDigits}"sv);
  841. return locale_data.unique_strings.ensure(move(new_pattern));
  842. };
  843. auto append_if_unique = [&](auto format) {
  844. auto format_index = locale_data.unique_patterns.ensure(move(format));
  845. if (!formats.contains_slow(format_index))
  846. formats.append(format_index);
  847. };
  848. Vector<CalendarPattern> time_formats_with_fractional_second_digits;
  849. for (auto const& format : date_formats)
  850. append_if_unique(format);
  851. for (auto const& format : time_formats) {
  852. append_if_unique(format);
  853. if (format.second.has_value() && !format.fractional_second_digits.has_value()) {
  854. auto new_format = format;
  855. new_format.fractional_second_digits = 2;
  856. new_format.pattern_index = inject_fractional_second_digits(new_format.pattern_index);
  857. if (new_format.pattern12_index != 0)
  858. new_format.pattern12_index = inject_fractional_second_digits(new_format.pattern12_index);
  859. time_formats_with_fractional_second_digits.append(new_format);
  860. append_if_unique(move(new_format));
  861. }
  862. }
  863. time_formats.extend(move(time_formats_with_fractional_second_digits));
  864. for (auto const& date_format : date_formats) {
  865. auto const& date_time_formats = locale_data.unique_formats.get(calendar.date_time_formats);
  866. CalendarPatternIndexType date_time_format_index = 0;
  867. if (date_format.month == Unicode::CalendarPatternStyle::Long) {
  868. if (date_format.weekday.has_value())
  869. date_time_format_index = date_time_formats.full_format;
  870. else
  871. date_time_format_index = date_time_formats.long_format;
  872. } else if (date_format.month == Unicode::CalendarPatternStyle::Short) {
  873. date_time_format_index = date_time_formats.medium_format;
  874. } else {
  875. date_time_format_index = date_time_formats.short_format;
  876. }
  877. for (auto const& time_format : time_formats) {
  878. auto format = locale_data.unique_patterns.get(date_time_format_index);
  879. if (time_format.pattern12_index != 0)
  880. format.pattern12_index = replace_pattern(format.pattern_index, time_format.pattern12_index, date_format.pattern_index);
  881. format.pattern_index = replace_pattern(format.pattern_index, time_format.pattern_index, date_format.pattern_index);
  882. format.for_each_calendar_field_zipped_with(date_format, [](auto& field, auto const& date_field, auto) {
  883. if (date_field.has_value())
  884. field = date_field;
  885. });
  886. format.for_each_calendar_field_zipped_with(time_format, [](auto& field, auto const& time_field, auto) {
  887. if (time_field.has_value())
  888. field = time_field;
  889. });
  890. append_if_unique(move(format));
  891. }
  892. }
  893. }
  894. static void parse_calendar_symbols(Calendar& calendar, JsonObject const& calendar_object, UnicodeLocaleData& locale_data)
  895. {
  896. auto create_symbol_lists = [](size_t size) {
  897. SymbolList narrow_symbol_list;
  898. SymbolList short_symbol_list;
  899. SymbolList long_symbol_list;
  900. narrow_symbol_list.resize(size);
  901. short_symbol_list.resize(size);
  902. long_symbol_list.resize(size);
  903. return Array<SymbolList, 3> { {
  904. move(narrow_symbol_list),
  905. move(short_symbol_list),
  906. move(long_symbol_list),
  907. } };
  908. };
  909. CalendarSymbolsList symbols_list {};
  910. auto store_symbol_lists = [&](auto symbol, auto symbol_lists) {
  911. auto symbol_index = to_underlying(symbol);
  912. if (symbol_index >= symbols_list.size())
  913. symbols_list.resize(symbol_index + 1);
  914. CalendarSymbols symbols {};
  915. symbols.narrow_symbols = locale_data.unique_symbol_lists.ensure(move(symbol_lists[0]));
  916. symbols.short_symbols = locale_data.unique_symbol_lists.ensure(move(symbol_lists[1]));
  917. symbols.long_symbols = locale_data.unique_symbol_lists.ensure(move(symbol_lists[2]));
  918. auto calendar_symbols_index = locale_data.unique_calendar_symbols.ensure(move(symbols));
  919. symbols_list[symbol_index] = calendar_symbols_index;
  920. };
  921. auto parse_era_symbols = [&](auto const& symbols_object) {
  922. auto const& narrow_symbols = symbols_object.get("eraNarrow"sv).as_object();
  923. auto const& short_symbols = symbols_object.get("eraAbbr"sv).as_object();
  924. auto const& long_symbols = symbols_object.get("eraNames"sv).as_object();
  925. auto symbol_lists = create_symbol_lists(2);
  926. auto append_symbol = [&](auto& symbols, auto const& key, auto symbol) {
  927. if (auto key_index = key.to_uint(); key_index.has_value())
  928. symbols[*key_index] = locale_data.unique_strings.ensure(move(symbol));
  929. };
  930. narrow_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  931. append_symbol(symbol_lists[0], key, value.as_string());
  932. });
  933. short_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  934. append_symbol(symbol_lists[1], key, value.as_string());
  935. });
  936. long_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  937. append_symbol(symbol_lists[2], key, value.as_string());
  938. });
  939. store_symbol_lists(Unicode::CalendarSymbol::Era, move(symbol_lists));
  940. };
  941. auto parse_month_symbols = [&](auto const& symbols_object) {
  942. auto const& narrow_symbols = symbols_object.get("narrow"sv).as_object();
  943. auto const& short_symbols = symbols_object.get("abbreviated"sv).as_object();
  944. auto const& long_symbols = symbols_object.get("wide"sv).as_object();
  945. auto symbol_lists = create_symbol_lists(12);
  946. auto append_symbol = [&](auto& symbols, auto const& key, auto symbol) {
  947. auto key_index = key.to_uint().value() - 1;
  948. symbols[key_index] = locale_data.unique_strings.ensure(move(symbol));
  949. };
  950. narrow_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  951. append_symbol(symbol_lists[0], key, value.as_string());
  952. });
  953. short_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  954. append_symbol(symbol_lists[1], key, value.as_string());
  955. });
  956. long_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  957. append_symbol(symbol_lists[2], key, value.as_string());
  958. });
  959. store_symbol_lists(Unicode::CalendarSymbol::Month, move(symbol_lists));
  960. };
  961. auto parse_weekday_symbols = [&](auto const& symbols_object) {
  962. auto const& narrow_symbols = symbols_object.get("narrow"sv).as_object();
  963. auto const& short_symbols = symbols_object.get("abbreviated"sv).as_object();
  964. auto const& long_symbols = symbols_object.get("wide"sv).as_object();
  965. auto symbol_lists = create_symbol_lists(7);
  966. auto append_symbol = [&](auto& symbols, auto const& key, auto symbol) {
  967. if (key == "sun"sv)
  968. symbols[to_underlying(Unicode::Weekday::Sunday)] = locale_data.unique_strings.ensure(move(symbol));
  969. else if (key == "mon"sv)
  970. symbols[to_underlying(Unicode::Weekday::Monday)] = locale_data.unique_strings.ensure(move(symbol));
  971. else if (key == "tue"sv)
  972. symbols[to_underlying(Unicode::Weekday::Tuesday)] = locale_data.unique_strings.ensure(move(symbol));
  973. else if (key == "wed"sv)
  974. symbols[to_underlying(Unicode::Weekday::Wednesday)] = locale_data.unique_strings.ensure(move(symbol));
  975. else if (key == "thu"sv)
  976. symbols[to_underlying(Unicode::Weekday::Thursday)] = locale_data.unique_strings.ensure(move(symbol));
  977. else if (key == "fri"sv)
  978. symbols[to_underlying(Unicode::Weekday::Friday)] = locale_data.unique_strings.ensure(move(symbol));
  979. else if (key == "sat"sv)
  980. symbols[to_underlying(Unicode::Weekday::Saturday)] = locale_data.unique_strings.ensure(move(symbol));
  981. };
  982. narrow_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  983. append_symbol(symbol_lists[0], key, value.as_string());
  984. });
  985. short_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  986. append_symbol(symbol_lists[1], key, value.as_string());
  987. });
  988. long_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  989. append_symbol(symbol_lists[2], key, value.as_string());
  990. });
  991. store_symbol_lists(Unicode::CalendarSymbol::Weekday, move(symbol_lists));
  992. };
  993. auto parse_day_period_symbols = [&](auto const& symbols_object) {
  994. auto const& narrow_symbols = symbols_object.get("narrow"sv).as_object();
  995. auto const& short_symbols = symbols_object.get("abbreviated"sv).as_object();
  996. auto const& long_symbols = symbols_object.get("wide"sv).as_object();
  997. auto symbol_lists = create_symbol_lists(10);
  998. auto append_symbol = [&](auto& symbols, auto const& key, auto symbol) {
  999. if (auto day_period = day_period_from_string(key); day_period.has_value())
  1000. symbols[to_underlying(*day_period)] = locale_data.unique_strings.ensure(move(symbol));
  1001. };
  1002. narrow_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  1003. append_symbol(symbol_lists[0], key, value.as_string());
  1004. });
  1005. short_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  1006. append_symbol(symbol_lists[1], key, value.as_string());
  1007. });
  1008. long_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  1009. append_symbol(symbol_lists[2], key, value.as_string());
  1010. });
  1011. store_symbol_lists(Unicode::CalendarSymbol::DayPeriod, move(symbol_lists));
  1012. };
  1013. parse_era_symbols(calendar_object.get("eras"sv).as_object());
  1014. parse_month_symbols(calendar_object.get("months"sv).as_object().get("format"sv).as_object());
  1015. parse_weekday_symbols(calendar_object.get("days"sv).as_object().get("format"sv).as_object());
  1016. parse_day_period_symbols(calendar_object.get("dayPeriods"sv).as_object().get("format"sv).as_object());
  1017. calendar.symbols = locale_data.unique_calendar_symbols_lists.ensure(move(symbols_list));
  1018. }
  1019. static ErrorOr<void> parse_calendars(String locale_calendars_path, UnicodeLocaleData& locale_data, Locale& locale)
  1020. {
  1021. LexicalPath calendars_path(move(locale_calendars_path));
  1022. if (!calendars_path.basename().starts_with("ca-"sv))
  1023. return {};
  1024. auto calendars_file = TRY(Core::File::open(calendars_path.string(), Core::OpenMode::ReadOnly));
  1025. auto calendars = TRY(JsonValue::from_string(calendars_file->read_all()));
  1026. auto const& main_object = calendars.as_object().get("main"sv);
  1027. auto const& locale_object = main_object.as_object().get(calendars_path.parent().basename());
  1028. auto const& dates_object = locale_object.as_object().get("dates"sv);
  1029. auto const& calendars_object = dates_object.as_object().get("calendars"sv);
  1030. auto parse_patterns = [&](auto const& patterns_object, auto const& skeletons_object, Vector<CalendarPattern>* patterns) {
  1031. auto parse_pattern = [&](auto name) {
  1032. auto format = patterns_object.get(name);
  1033. auto skeleton = skeletons_object.get(name);
  1034. auto format_index = parse_date_time_pattern(format.as_string(), skeleton.as_string_or(String::empty()), locale_data).value();
  1035. if (patterns)
  1036. patterns->append(locale_data.unique_patterns.get(format_index));
  1037. return format_index;
  1038. };
  1039. CalendarFormat formats {};
  1040. formats.full_format = parse_pattern("full"sv);
  1041. formats.long_format = parse_pattern("long"sv);
  1042. formats.medium_format = parse_pattern("medium"sv);
  1043. formats.short_format = parse_pattern("short"sv);
  1044. return locale_data.unique_formats.ensure(move(formats));
  1045. };
  1046. calendars_object.as_object().for_each_member([&](auto const& calendar_name, JsonValue const& value) {
  1047. // The generic calendar is not a supported Unicode calendar key, so skip it:
  1048. // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/Locale/calendar#unicode_calendar_keys
  1049. if (calendar_name == "generic"sv)
  1050. return;
  1051. Calendar calendar {};
  1052. CalendarPatternList available_formats {};
  1053. if (!locale_data.calendars.contains_slow(calendar_name))
  1054. locale_data.calendars.append(calendar_name);
  1055. Vector<CalendarPattern> date_formats;
  1056. Vector<CalendarPattern> time_formats;
  1057. auto const& date_formats_object = value.as_object().get("dateFormats"sv);
  1058. auto const& date_skeletons_object = value.as_object().get("dateSkeletons"sv);
  1059. calendar.date_formats = parse_patterns(date_formats_object.as_object(), date_skeletons_object.as_object(), &date_formats);
  1060. auto const& time_formats_object = value.as_object().get("timeFormats"sv);
  1061. auto const& time_skeletons_object = value.as_object().get("timeSkeletons"sv);
  1062. calendar.time_formats = parse_patterns(time_formats_object.as_object(), time_skeletons_object.as_object(), &time_formats);
  1063. auto const& date_time_formats_object = value.as_object().get("dateTimeFormats"sv);
  1064. calendar.date_time_formats = parse_patterns(date_time_formats_object.as_object(), JsonObject {}, nullptr);
  1065. auto const& available_formats_object = date_time_formats_object.as_object().get("availableFormats"sv);
  1066. available_formats_object.as_object().for_each_member([&](auto const& skeleton, JsonValue const& pattern) {
  1067. auto pattern_index = parse_date_time_pattern(pattern.as_string(), skeleton, locale_data);
  1068. if (!pattern_index.has_value())
  1069. return;
  1070. auto const& format = locale_data.unique_patterns.get(*pattern_index);
  1071. if (format.contains_only_date_fields())
  1072. date_formats.append(format);
  1073. else if (format.contains_only_time_fields())
  1074. time_formats.append(format);
  1075. if (!available_formats.contains_slow(*pattern_index))
  1076. available_formats.append(*pattern_index);
  1077. });
  1078. auto const& interval_formats_object = date_time_formats_object.as_object().get("intervalFormats"sv);
  1079. parse_interval_patterns(calendar, interval_formats_object.as_object(), locale_data);
  1080. generate_missing_patterns(calendar, available_formats, move(date_formats), move(time_formats), locale_data);
  1081. parse_calendar_symbols(calendar, value.as_object(), locale_data);
  1082. calendar.available_formats = locale_data.unique_pattern_lists.ensure(move(available_formats));
  1083. locale.calendars.set(calendar_name, locale_data.unique_calendars.ensure(move(calendar)));
  1084. });
  1085. return {};
  1086. }
  1087. static ErrorOr<void> parse_time_zone_names(String locale_time_zone_names_path, UnicodeLocaleData& locale_data, Locale& locale)
  1088. {
  1089. LexicalPath time_zone_names_path(move(locale_time_zone_names_path));
  1090. time_zone_names_path = time_zone_names_path.append("timeZoneNames.json"sv);
  1091. auto time_zone_names_file = TRY(Core::File::open(time_zone_names_path.string(), Core::OpenMode::ReadOnly));
  1092. auto time_zone_names = TRY(JsonValue::from_string(time_zone_names_file->read_all()));
  1093. auto const& main_object = time_zone_names.as_object().get("main"sv);
  1094. auto const& locale_object = main_object.as_object().get(time_zone_names_path.parent().basename());
  1095. auto const& dates_object = locale_object.as_object().get("dates"sv);
  1096. auto const& time_zone_names_object = dates_object.as_object().get("timeZoneNames"sv);
  1097. auto const& meta_zone_object = time_zone_names_object.as_object().get("metazone"sv);
  1098. if (meta_zone_object.is_null())
  1099. return {};
  1100. auto parse_name = [&](StringView type, JsonObject const& meta_zone_object) -> Optional<StringIndexType> {
  1101. auto const& names = meta_zone_object.get(type);
  1102. if (!names.is_object())
  1103. return {};
  1104. auto const& daylight = names.as_object().get("daylight"sv);
  1105. if (daylight.is_string())
  1106. return locale_data.unique_strings.ensure(daylight.as_string());
  1107. auto const& standard = names.as_object().get("standard"sv);
  1108. if (standard.is_string())
  1109. return locale_data.unique_strings.ensure(standard.as_string());
  1110. return {};
  1111. };
  1112. TimeZoneList time_zones;
  1113. auto parse_time_zone = [&](StringView meta_zone, JsonObject const& meta_zone_object) {
  1114. auto const& golden_zone = locale_data.meta_zones.find(meta_zone)->value;
  1115. TimeZone time_zone {};
  1116. if (auto long_name = parse_name("long"sv, meta_zone_object); long_name.has_value())
  1117. time_zone.long_name = long_name.value();
  1118. if (auto short_name = parse_name("short"sv, meta_zone_object); short_name.has_value())
  1119. time_zone.short_name = short_name.value();
  1120. auto time_zone_index = locale_data.time_zones.find_first_index(golden_zone).value();
  1121. time_zones[time_zone_index] = locale_data.unique_time_zones.ensure(move(time_zone));
  1122. };
  1123. meta_zone_object.as_object().for_each_member([&](auto const& meta_zone, JsonValue const&) {
  1124. auto const& golden_zone = locale_data.meta_zones.find(meta_zone)->value;
  1125. if (!locale_data.time_zones.contains_slow(golden_zone))
  1126. locale_data.time_zones.append(golden_zone);
  1127. });
  1128. time_zones.resize(locale_data.time_zones.size());
  1129. quick_sort(locale_data.time_zones);
  1130. meta_zone_object.as_object().for_each_member([&](auto const& meta_zone, JsonValue const& value) {
  1131. parse_time_zone(meta_zone, value.as_object());
  1132. });
  1133. // The long and short names for UTC are not under the "timeZoneNames/metazone" object, but are under "timeZoneNames/zone/Etc".
  1134. auto const& zone_object = time_zone_names_object.as_object().get("zone"sv);
  1135. auto const& etc_object = zone_object.as_object().get("Etc"sv);
  1136. auto const& utc_object = etc_object.as_object().get("UTC"sv);
  1137. parse_time_zone("UTC"sv, utc_object.as_object());
  1138. locale.time_zones = locale_data.unique_time_zone_lists.ensure(move(time_zones));
  1139. return {};
  1140. }
  1141. static ErrorOr<void> parse_day_periods(String core_path, UnicodeLocaleData& locale_data)
  1142. {
  1143. // https://unicode.org/reports/tr35/tr35-dates.html#Day_Period_Rule_Sets
  1144. LexicalPath day_periods_path(move(core_path));
  1145. day_periods_path = day_periods_path.append("supplemental"sv);
  1146. day_periods_path = day_periods_path.append("dayPeriods.json"sv);
  1147. auto day_periods_file = TRY(Core::File::open(day_periods_path.string(), Core::OpenMode::ReadOnly));
  1148. auto locale_day_periods = TRY(JsonValue::from_string(day_periods_file->read_all()));
  1149. auto const& supplemental_object = locale_day_periods.as_object().get("supplemental"sv);
  1150. auto const& day_periods_object = supplemental_object.as_object().get("dayPeriodRuleSet"sv);
  1151. auto parse_hour = [](auto const& time) {
  1152. auto hour_end_index = time.find(':').value();
  1153. // The times are in the form "hours:minutes", but we only need the hour segment.
  1154. // TR-35 explicitly states that minutes other than :00 must not be used.
  1155. VERIFY(time.substring_view(hour_end_index) == ":00"sv);
  1156. auto hour = time.substring_view(0, hour_end_index);
  1157. return hour.template to_uint<u8>().value();
  1158. };
  1159. auto parse_day_period = [&](auto const& symbol, auto const& ranges) -> Optional<DayPeriod> {
  1160. auto day_period = day_period_from_string(symbol);
  1161. if (!day_period.has_value())
  1162. return {};
  1163. auto begin = parse_hour(ranges.get("_from"sv).as_string());
  1164. auto end = parse_hour(ranges.get("_before"sv).as_string());
  1165. return DayPeriod { *day_period, begin, end };
  1166. };
  1167. day_periods_object.as_object().for_each_member([&](auto const& language, JsonValue const& value) {
  1168. auto locale = locale_data.locales.find(language);
  1169. if (locale == locale_data.locales.end())
  1170. return;
  1171. DayPeriodList day_periods;
  1172. value.as_object().for_each_member([&](auto const& symbol, JsonValue const& ranges) {
  1173. if (auto day_period = parse_day_period(symbol, ranges.as_object()); day_period.has_value()) {
  1174. auto day_period_index = locale_data.unique_day_periods.ensure(day_period.release_value());
  1175. day_periods.append(day_period_index);
  1176. }
  1177. });
  1178. locale->value.day_periods = locale_data.unique_day_period_lists.ensure(move(day_periods));
  1179. });
  1180. return {};
  1181. };
  1182. static ErrorOr<void> parse_all_locales(String core_path, String dates_path, UnicodeLocaleData& locale_data)
  1183. {
  1184. TRY(parse_hour_cycles(core_path, locale_data));
  1185. TRY(parse_meta_zones(core_path, locale_data));
  1186. auto dates_iterator = TRY(path_to_dir_iterator(move(dates_path)));
  1187. auto remove_variants_from_path = [&](String path) -> ErrorOr<String> {
  1188. auto parsed_locale = TRY(CanonicalLanguageID<StringIndexType>::parse(locale_data.unique_strings, LexicalPath::basename(path)));
  1189. StringBuilder builder;
  1190. builder.append(locale_data.unique_strings.get(parsed_locale.language));
  1191. if (auto script = locale_data.unique_strings.get(parsed_locale.script); !script.is_empty())
  1192. builder.appendff("-{}", script);
  1193. if (auto region = locale_data.unique_strings.get(parsed_locale.region); !region.is_empty())
  1194. builder.appendff("-{}", region);
  1195. return builder.build();
  1196. };
  1197. while (dates_iterator.has_next()) {
  1198. auto dates_path = TRY(next_path_from_dir_iterator(dates_iterator));
  1199. auto calendars_iterator = TRY(path_to_dir_iterator(dates_path, {}));
  1200. auto language = TRY(remove_variants_from_path(dates_path));
  1201. auto& locale = locale_data.locales.ensure(language);
  1202. while (calendars_iterator.has_next()) {
  1203. auto calendars_path = TRY(next_path_from_dir_iterator(calendars_iterator));
  1204. TRY(parse_calendars(move(calendars_path), locale_data, locale));
  1205. }
  1206. TRY(parse_time_zone_names(move(dates_path), locale_data, locale));
  1207. }
  1208. TRY(parse_day_periods(move(core_path), locale_data));
  1209. return {};
  1210. }
  1211. static String format_identifier(StringView owner, String identifier)
  1212. {
  1213. identifier = identifier.replace("-"sv, "_"sv, true);
  1214. identifier = identifier.replace("/"sv, "_"sv, true);
  1215. if (all_of(identifier, is_ascii_digit))
  1216. return String::formatted("{}_{}", owner[0], identifier);
  1217. if (is_ascii_lower_alpha(identifier[0]))
  1218. return String::formatted("{:c}{}", to_ascii_uppercase(identifier[0]), identifier.substring_view(1));
  1219. return identifier;
  1220. }
  1221. static void generate_unicode_locale_header(Core::File& file, UnicodeLocaleData& locale_data)
  1222. {
  1223. StringBuilder builder;
  1224. SourceGenerator generator { builder };
  1225. generator.append(R"~~~(
  1226. #pragma once
  1227. #include <AK/Optional.h>
  1228. #include <AK/StringView.h>
  1229. #include <LibUnicode/Forward.h>
  1230. namespace Unicode {
  1231. )~~~");
  1232. generate_enum(generator, format_identifier, "Calendar"sv, {}, locale_data.calendars, locale_data.calendar_aliases);
  1233. generate_enum(generator, format_identifier, "HourCycleRegion"sv, {}, locale_data.hour_cycle_regions);
  1234. generate_enum(generator, format_identifier, "TimeZone"sv, {}, locale_data.time_zones);
  1235. generator.append(R"~~~(
  1236. namespace Detail {
  1237. Optional<Calendar> calendar_from_string(StringView calendar);
  1238. Optional<HourCycleRegion> hour_cycle_region_from_string(StringView hour_cycle_region);
  1239. Vector<Unicode::HourCycle> get_regional_hour_cycles(StringView region);
  1240. Optional<Unicode::CalendarFormat> get_calendar_date_format(StringView locale, StringView calendar);
  1241. Optional<Unicode::CalendarFormat> get_calendar_time_format(StringView locale, StringView calendar);
  1242. Optional<Unicode::CalendarFormat> get_calendar_date_time_format(StringView locale, StringView calendar);
  1243. Vector<Unicode::CalendarPattern> get_calendar_available_formats(StringView locale, StringView calendar);
  1244. Optional<Unicode::CalendarRangePattern> get_calendar_default_range_format(StringView locale, StringView calendar);
  1245. Vector<Unicode::CalendarRangePattern> get_calendar_range_formats(StringView locale, StringView calendar, StringView skeleton);
  1246. Vector<Unicode::CalendarRangePattern> get_calendar_range12_formats(StringView locale, StringView calendar, StringView skeleton);
  1247. Optional<StringView> get_calendar_era_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Era value);
  1248. Optional<StringView> get_calendar_month_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Month value);
  1249. Optional<StringView> get_calendar_weekday_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Weekday value);
  1250. Optional<StringView> get_calendar_day_period_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::DayPeriod value);
  1251. Optional<StringView> get_calendar_day_period_symbol_for_hour(StringView locale, StringView calendar, CalendarPatternStyle style, u8 hour);
  1252. Optional<TimeZone> time_zone_from_string(StringView time_zone);
  1253. Optional<StringView> get_time_zone_name(StringView locale, StringView time_zone, CalendarPatternStyle style);
  1254. }
  1255. }
  1256. )~~~");
  1257. VERIFY(file.write(generator.as_string_view()));
  1258. }
  1259. static void generate_unicode_locale_implementation(Core::File& file, UnicodeLocaleData& locale_data)
  1260. {
  1261. StringBuilder builder;
  1262. SourceGenerator generator { builder };
  1263. generator.set("string_index_type"sv, s_string_index_type);
  1264. generator.set("calendar_pattern_index_type"sv, s_calendar_pattern_index_type);
  1265. generator.set("calendar_pattern_list_index_type"sv, s_calendar_pattern_list_index_type);
  1266. generator.set("calendar_range_pattern_index_type"sv, s_calendar_range_pattern_index_type);
  1267. generator.set("calendar_range_pattern_list_index_type"sv, s_calendar_range_pattern_list_index_type);
  1268. generator.set("calendar_format_index_type"sv, s_calendar_format_index_type);
  1269. generator.set("symbol_list_index_type"sv, s_symbol_list_index_type);
  1270. generator.set("calendar_symbols_index_type"sv, s_calendar_symbols_index_type);
  1271. generator.set("calendar_symbols_list_index_type"sv, s_calendar_symbols_list_index_type);
  1272. generator.set("calendar_index_type"sv, s_calendar_index_type);
  1273. generator.set("time_zone_index_type"sv, s_time_zone_index_type);
  1274. generator.set("time_zone_list_index_type"sv, s_time_zone_list_index_type);
  1275. generator.set("day_period_index_type"sv, s_day_period_index_type);
  1276. generator.set("day_period_list_index_type"sv, s_day_period_list_index_type);
  1277. generator.append(R"~~~(
  1278. #include <AK/Array.h>
  1279. #include <AK/BinarySearch.h>
  1280. #include <LibUnicode/DateTimeFormat.h>
  1281. #include <LibUnicode/Locale.h>
  1282. #include <LibUnicode/UnicodeDateTimeFormat.h>
  1283. namespace Unicode::Detail {
  1284. )~~~");
  1285. locale_data.unique_strings.generate(generator);
  1286. generator.append(R"~~~(
  1287. template <typename SourceType, typename TargetType>
  1288. void convert_calendar_fields(SourceType const& source, TargetType& target)
  1289. {
  1290. if (source.era != -1)
  1291. target.era = static_cast<Unicode::CalendarPatternStyle>(source.era);
  1292. if (source.year != -1)
  1293. target.year = static_cast<Unicode::CalendarPatternStyle>(source.year);
  1294. if (source.month != -1)
  1295. target.month = static_cast<Unicode::CalendarPatternStyle>(source.month);
  1296. if (source.weekday != -1)
  1297. target.weekday = static_cast<Unicode::CalendarPatternStyle>(source.weekday);
  1298. if (source.day != -1)
  1299. target.day = static_cast<Unicode::CalendarPatternStyle>(source.day);
  1300. if (source.day_period != -1)
  1301. target.day_period = static_cast<Unicode::CalendarPatternStyle>(source.day_period);
  1302. if (source.hour != -1)
  1303. target.hour = static_cast<Unicode::CalendarPatternStyle>(source.hour);
  1304. if (source.minute != -1)
  1305. target.minute = static_cast<Unicode::CalendarPatternStyle>(source.minute);
  1306. if (source.second != -1)
  1307. target.second = static_cast<Unicode::CalendarPatternStyle>(source.second);
  1308. if (source.fractional_second_digits != -1)
  1309. target.fractional_second_digits = static_cast<u8>(source.fractional_second_digits);
  1310. if (source.time_zone_name != -1)
  1311. target.time_zone_name = static_cast<Unicode::CalendarPatternStyle>(source.time_zone_name);
  1312. }
  1313. struct CalendarPattern {
  1314. Unicode::CalendarPattern to_unicode_calendar_pattern() const {
  1315. Unicode::CalendarPattern calendar_pattern {};
  1316. calendar_pattern.skeleton = s_string_list[skeleton];
  1317. calendar_pattern.pattern = s_string_list[pattern];
  1318. if (pattern12 != 0)
  1319. calendar_pattern.pattern12 = s_string_list[pattern12];
  1320. convert_calendar_fields(*this, calendar_pattern);
  1321. return calendar_pattern;
  1322. }
  1323. @string_index_type@ skeleton { 0 };
  1324. @string_index_type@ pattern { 0 };
  1325. @string_index_type@ pattern12 { 0 };
  1326. i8 era { -1 };
  1327. i8 year { -1 };
  1328. i8 month { -1 };
  1329. i8 weekday { -1 };
  1330. i8 day { -1 };
  1331. i8 day_period { -1 };
  1332. i8 hour { -1 };
  1333. i8 minute { -1 };
  1334. i8 second { -1 };
  1335. i8 fractional_second_digits { -1 };
  1336. i8 time_zone_name { -1 };
  1337. };
  1338. struct CalendarRangePattern {
  1339. Unicode::CalendarRangePattern to_unicode_calendar_range_pattern() const {
  1340. Unicode::CalendarRangePattern calendar_range_pattern {};
  1341. if (field != -1)
  1342. calendar_range_pattern.field = static_cast<Unicode::CalendarRangePattern::Field>(field);
  1343. calendar_range_pattern.start_range = s_string_list[start_range];
  1344. calendar_range_pattern.separator = s_string_list[separator];
  1345. calendar_range_pattern.end_range = s_string_list[end_range];
  1346. convert_calendar_fields(*this, calendar_range_pattern);
  1347. return calendar_range_pattern;
  1348. }
  1349. @string_index_type@ skeleton { 0 };
  1350. i8 field { -1 };
  1351. @string_index_type@ start_range { 0 };
  1352. @string_index_type@ separator { 0 };
  1353. @string_index_type@ end_range { 0 };
  1354. i8 era { -1 };
  1355. i8 year { -1 };
  1356. i8 month { -1 };
  1357. i8 weekday { -1 };
  1358. i8 day { -1 };
  1359. i8 day_period { -1 };
  1360. i8 hour { -1 };
  1361. i8 minute { -1 };
  1362. i8 second { -1 };
  1363. i8 fractional_second_digits { -1 };
  1364. i8 time_zone_name { -1 };
  1365. };
  1366. )~~~");
  1367. locale_data.unique_patterns.generate(generator, "CalendarPattern"sv, "s_calendar_patterns"sv, 10);
  1368. locale_data.unique_pattern_lists.generate(generator, s_calendar_pattern_index_type, "s_calendar_pattern_lists"sv);
  1369. locale_data.unique_range_patterns.generate(generator, "CalendarRangePattern"sv, "s_calendar_range_patterns"sv, 10);
  1370. locale_data.unique_range_pattern_lists.generate(generator, s_calendar_range_pattern_index_type, "s_calendar_range_pattern_lists"sv);
  1371. generator.append(R"~~~(
  1372. struct CalendarFormat {
  1373. Unicode::CalendarFormat to_unicode_calendar_format() const {
  1374. Unicode::CalendarFormat calendar_format {};
  1375. calendar_format.full_format = s_calendar_patterns[full_format].to_unicode_calendar_pattern();
  1376. calendar_format.long_format = s_calendar_patterns[long_format].to_unicode_calendar_pattern();
  1377. calendar_format.medium_format = s_calendar_patterns[medium_format].to_unicode_calendar_pattern();
  1378. calendar_format.short_format = s_calendar_patterns[short_format].to_unicode_calendar_pattern();
  1379. return calendar_format;
  1380. }
  1381. @calendar_pattern_index_type@ full_format { 0 };
  1382. @calendar_pattern_index_type@ long_format { 0 };
  1383. @calendar_pattern_index_type@ medium_format { 0 };
  1384. @calendar_pattern_index_type@ short_format { 0 };
  1385. };
  1386. struct CalendarSymbols {
  1387. @symbol_list_index_type@ narrow_symbols { 0 };
  1388. @symbol_list_index_type@ short_symbols { 0 };
  1389. @symbol_list_index_type@ long_symbols { 0 };
  1390. };
  1391. struct CalendarData {
  1392. @calendar_format_index_type@ date_formats { 0 };
  1393. @calendar_format_index_type@ time_formats { 0 };
  1394. @calendar_format_index_type@ date_time_formats { 0 };
  1395. @calendar_pattern_list_index_type@ available_formats { 0 };
  1396. @calendar_range_pattern_index_type@ default_range_format { 0 };
  1397. @calendar_range_pattern_list_index_type@ range_formats { 0 };
  1398. @calendar_range_pattern_list_index_type@ range12_formats { 0 };
  1399. @calendar_symbols_list_index_type@ symbols { 0 };
  1400. };
  1401. struct TimeZoneData {
  1402. @string_index_type@ long_name { 0 };
  1403. @string_index_type@ short_name { 0 };
  1404. };
  1405. struct DayPeriodData {
  1406. u8 day_period { 0 };
  1407. u8 begin { 0 };
  1408. u8 end { 0 };
  1409. };
  1410. )~~~");
  1411. locale_data.unique_formats.generate(generator, "CalendarFormat"sv, "s_calendar_formats"sv, 10);
  1412. locale_data.unique_symbol_lists.generate(generator, s_string_index_type, "s_symbol_lists"sv);
  1413. locale_data.unique_calendar_symbols.generate(generator, "CalendarSymbols"sv, "s_calendar_symbols"sv, 10);
  1414. locale_data.unique_calendar_symbols_lists.generate(generator, s_calendar_symbols_index_type, "s_calendar_symbol_lists"sv);
  1415. locale_data.unique_calendars.generate(generator, "CalendarData"sv, "s_calendars"sv, 10);
  1416. locale_data.unique_time_zones.generate(generator, "TimeZoneData"sv, "s_time_zones"sv, 30);
  1417. locale_data.unique_time_zone_lists.generate(generator, s_time_zone_index_type, "s_time_zone_lists"sv);
  1418. locale_data.unique_day_periods.generate(generator, "DayPeriodData"sv, "s_day_periods"sv, 30);
  1419. locale_data.unique_day_period_lists.generate(generator, s_day_period_index_type, "s_day_period_lists"sv);
  1420. locale_data.unique_hour_cycle_lists.generate(generator, "u8"sv, "s_hour_cycle_lists"sv);
  1421. auto append_calendars = [&](String name, auto const& calendars) {
  1422. generator.set("name", name);
  1423. generator.set("size", String::number(calendars.size()));
  1424. generator.append(R"~~~(
  1425. static constexpr Array<@calendar_index_type@, @size@> @name@ { {)~~~");
  1426. bool first = true;
  1427. for (auto const& calendar_key : locale_data.calendars) {
  1428. auto calendar = calendars.find(calendar_key)->value;
  1429. generator.append(first ? " " : ", ");
  1430. generator.append(String::number(calendar));
  1431. first = false;
  1432. }
  1433. generator.append(" } };");
  1434. };
  1435. auto append_mapping = [&](auto const& keys, auto const& map, auto type, auto name, auto mapping_getter) {
  1436. generator.set("type", type);
  1437. generator.set("name", name);
  1438. generator.set("size", String::number(keys.size()));
  1439. generator.append(R"~~~(
  1440. static constexpr Array<@type@, @size@> @name@ { {)~~~");
  1441. bool first = true;
  1442. for (auto const& key : keys) {
  1443. auto const& value = map.find(key)->value;
  1444. auto mapping = mapping_getter(value);
  1445. generator.append(first ? " " : ", ");
  1446. generator.append(String::number(mapping));
  1447. first = false;
  1448. }
  1449. generator.append(" } };");
  1450. };
  1451. auto locales = locale_data.locales.keys();
  1452. quick_sort(locales);
  1453. generate_mapping(generator, locale_data.locales, s_calendar_index_type, "s_locale_calendars"sv, "s_calendars_{}", [&](auto const& name, auto const& value) { append_calendars(name, value.calendars); });
  1454. append_mapping(locales, locale_data.locales, s_time_zone_index_type, "s_locale_time_zones"sv, [](auto const& locale) { return locale.time_zones; });
  1455. append_mapping(locales, locale_data.locales, s_day_period_index_type, "s_locale_day_periods"sv, [](auto const& locale) { return locale.day_periods; });
  1456. append_mapping(locale_data.hour_cycle_regions, locale_data.hour_cycles, s_hour_cycle_list_index_type, "s_hour_cycles"sv, [](auto const& hour_cycles) { return hour_cycles; });
  1457. generator.append("\n");
  1458. auto append_from_string = [&](StringView enum_title, StringView enum_snake, auto const& values, Vector<Alias> const& aliases = {}) {
  1459. HashValueMap<String> hashes;
  1460. hashes.ensure_capacity(values.size());
  1461. for (auto const& value : values)
  1462. hashes.set(value.hash(), format_identifier(enum_title, value));
  1463. for (auto const& alias : aliases)
  1464. hashes.set(alias.alias.hash(), format_identifier(enum_title, alias.alias));
  1465. generate_value_from_string(generator, "{}_from_string"sv, enum_title, enum_snake, move(hashes));
  1466. };
  1467. append_from_string("Calendar"sv, "calendar"sv, locale_data.calendars, locale_data.calendar_aliases);
  1468. append_from_string("HourCycleRegion"sv, "hour_cycle_region"sv, locale_data.hour_cycle_regions);
  1469. append_from_string("TimeZone"sv, "time_zone"sv, locale_data.time_zones);
  1470. generator.append(R"~~~(
  1471. Vector<Unicode::HourCycle> get_regional_hour_cycles(StringView region)
  1472. {
  1473. auto region_value = hour_cycle_region_from_string(region);
  1474. if (!region_value.has_value())
  1475. return {};
  1476. auto region_index = to_underlying(*region_value);
  1477. auto regional_hour_cycles_index = s_hour_cycles.at(region_index);
  1478. auto const& regional_hour_cycles = s_hour_cycle_lists.at(regional_hour_cycles_index);
  1479. Vector<Unicode::HourCycle> hour_cycles;
  1480. hour_cycles.ensure_capacity(regional_hour_cycles.size());
  1481. for (auto hour_cycle : regional_hour_cycles)
  1482. hour_cycles.unchecked_append(static_cast<Unicode::HourCycle>(hour_cycle));
  1483. return hour_cycles;
  1484. }
  1485. static CalendarData const* find_calendar_data(StringView locale, StringView calendar)
  1486. {
  1487. auto locale_value = locale_from_string(locale);
  1488. if (!locale_value.has_value())
  1489. return nullptr;
  1490. auto calendar_value = calendar_from_string(calendar);
  1491. if (!calendar_value.has_value())
  1492. return nullptr;
  1493. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1494. size_t calendar_index = to_underlying(*calendar_value);
  1495. auto const& calendar_indices = s_locale_calendars.at(locale_index);
  1496. calendar_index = calendar_indices[calendar_index];
  1497. return &s_calendars[calendar_index];
  1498. }
  1499. Optional<Unicode::CalendarFormat> get_calendar_date_format(StringView locale, StringView calendar)
  1500. {
  1501. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1502. auto const& formats = s_calendar_formats.at(data->date_formats);
  1503. return formats.to_unicode_calendar_format();
  1504. }
  1505. return {};
  1506. }
  1507. Optional<Unicode::CalendarFormat> get_calendar_time_format(StringView locale, StringView calendar)
  1508. {
  1509. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1510. auto const& formats = s_calendar_formats.at(data->time_formats);
  1511. return formats.to_unicode_calendar_format();
  1512. }
  1513. return {};
  1514. }
  1515. Optional<Unicode::CalendarFormat> get_calendar_date_time_format(StringView locale, StringView calendar)
  1516. {
  1517. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1518. auto const& formats = s_calendar_formats.at(data->date_time_formats);
  1519. return formats.to_unicode_calendar_format();
  1520. }
  1521. return {};
  1522. }
  1523. Vector<Unicode::CalendarPattern> get_calendar_available_formats(StringView locale, StringView calendar)
  1524. {
  1525. Vector<Unicode::CalendarPattern> result {};
  1526. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1527. auto const& available_formats = s_calendar_pattern_lists.at(data->available_formats);
  1528. result.ensure_capacity(available_formats.size());
  1529. for (auto const& format : available_formats)
  1530. result.unchecked_append(s_calendar_patterns[format].to_unicode_calendar_pattern());
  1531. }
  1532. return result;
  1533. }
  1534. Optional<Unicode::CalendarRangePattern> get_calendar_default_range_format(StringView locale, StringView calendar)
  1535. {
  1536. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1537. auto const& pattern = s_calendar_range_patterns[data->default_range_format];
  1538. return pattern.to_unicode_calendar_range_pattern();
  1539. }
  1540. return {};
  1541. }
  1542. Vector<Unicode::CalendarRangePattern> get_calendar_range_formats(StringView locale, StringView calendar, StringView skeleton) {
  1543. Vector<Unicode::CalendarRangePattern> result {};
  1544. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1545. auto const& range_formats = s_calendar_range_pattern_lists.at(data->range_formats);
  1546. for (auto format : range_formats) {
  1547. auto const& pattern = s_calendar_range_patterns[format];
  1548. if (skeleton == s_string_list[pattern.skeleton])
  1549. result.append(pattern.to_unicode_calendar_range_pattern());
  1550. }
  1551. }
  1552. return result;
  1553. }
  1554. Vector<Unicode::CalendarRangePattern> get_calendar_range12_formats(StringView locale, StringView calendar, StringView skeleton) {
  1555. Vector<Unicode::CalendarRangePattern> result {};
  1556. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1557. auto const& range12_formats = s_calendar_range_pattern_lists.at(data->range12_formats);
  1558. for (auto format : range12_formats) {
  1559. auto const& pattern = s_calendar_range_patterns[format];
  1560. if (skeleton == s_string_list[pattern.skeleton])
  1561. result.append(pattern.to_unicode_calendar_range_pattern());
  1562. }
  1563. }
  1564. return result;
  1565. }
  1566. static Span<@string_index_type@ const> find_calendar_symbols(StringView locale, StringView calendar, CalendarSymbol symbol, CalendarPatternStyle style)
  1567. {
  1568. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1569. auto const& symbols_list = s_calendar_symbol_lists[data->symbols];
  1570. auto symbol_index = to_underlying(symbol);
  1571. auto calendar_symbols_index = symbols_list.at(symbol_index);
  1572. auto const& symbols = s_calendar_symbols.at(calendar_symbols_index);
  1573. @symbol_list_index_type@ symbol_list_index = 0;
  1574. switch (style) {
  1575. case CalendarPatternStyle::Narrow:
  1576. symbol_list_index = symbols.narrow_symbols;
  1577. break;
  1578. case CalendarPatternStyle::Short:
  1579. symbol_list_index = symbols.short_symbols;
  1580. break;
  1581. case CalendarPatternStyle::Long:
  1582. symbol_list_index = symbols.long_symbols;
  1583. break;
  1584. default:
  1585. VERIFY_NOT_REACHED();
  1586. }
  1587. return s_symbol_lists.at(symbol_list_index);
  1588. }
  1589. return {};
  1590. }
  1591. Optional<StringView> get_calendar_era_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Era value)
  1592. {
  1593. auto symbols = find_calendar_symbols(locale, calendar, CalendarSymbol::Era, style);
  1594. if (auto value_index = to_underlying(value); value_index < symbols.size())
  1595. return s_string_list[symbols.at(value_index)];
  1596. return {};
  1597. }
  1598. Optional<StringView> get_calendar_month_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Month value)
  1599. {
  1600. auto symbols = find_calendar_symbols(locale, calendar, CalendarSymbol::Month, style);
  1601. if (auto value_index = to_underlying(value); value_index < symbols.size())
  1602. return s_string_list[symbols.at(value_index)];
  1603. return {};
  1604. }
  1605. Optional<StringView> get_calendar_weekday_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Weekday value)
  1606. {
  1607. auto symbols = find_calendar_symbols(locale, calendar, CalendarSymbol::Weekday, style);
  1608. if (auto value_index = to_underlying(value); value_index < symbols.size())
  1609. return s_string_list[symbols.at(value_index)];
  1610. return {};
  1611. }
  1612. Optional<StringView> get_calendar_day_period_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::DayPeriod value)
  1613. {
  1614. auto symbols = find_calendar_symbols(locale, calendar, CalendarSymbol::DayPeriod, style);
  1615. if (auto value_index = to_underlying(value); value_index < symbols.size())
  1616. return s_string_list[symbols.at(value_index)];
  1617. return {};
  1618. }
  1619. Optional<StringView> get_calendar_day_period_symbol_for_hour(StringView locale, StringView calendar, CalendarPatternStyle style, u8 hour)
  1620. {
  1621. auto locale_value = locale_from_string(locale);
  1622. if (!locale_value.has_value())
  1623. return {};
  1624. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1625. auto day_periods_index = s_locale_day_periods[locale_index];
  1626. auto day_periods = s_day_period_lists[day_periods_index];
  1627. for (auto day_period_index : day_periods) {
  1628. auto const& day_period = s_day_periods[day_period_index];
  1629. if ((day_period.begin <= hour) && (hour < day_period.end)) {
  1630. auto period = static_cast<Unicode::DayPeriod>(day_period.day_period);
  1631. return Detail::get_calendar_day_period_symbol(locale, calendar, style, period);
  1632. }
  1633. }
  1634. // Fallback to fixed periods if the locale does not have flexible day periods.
  1635. // TR-35 states that the meaning of AM and PM does not change with locale.
  1636. if (hour < 12)
  1637. return Detail::get_calendar_day_period_symbol(locale, calendar, style, Unicode::DayPeriod::AM);
  1638. return Detail::get_calendar_day_period_symbol(locale, calendar, style, Unicode::DayPeriod::PM);
  1639. }
  1640. static TimeZoneData const* find_time_zone_data(StringView locale, StringView time_zone)
  1641. {
  1642. auto locale_value = locale_from_string(locale);
  1643. if (!locale_value.has_value())
  1644. return nullptr;
  1645. auto time_zone_value = time_zone_from_string(time_zone);
  1646. if (!time_zone_value.has_value())
  1647. return nullptr;
  1648. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1649. size_t time_zone_index = to_underlying(*time_zone_value);
  1650. auto time_zone_list_index = s_locale_time_zones.at(locale_index);
  1651. auto const& time_zone_list = s_time_zone_lists.at(time_zone_list_index);
  1652. time_zone_index = time_zone_list.at(time_zone_index);
  1653. return &s_time_zones[time_zone_index];
  1654. }
  1655. Optional<StringView> get_time_zone_name(StringView locale, StringView time_zone, CalendarPatternStyle style)
  1656. {
  1657. if (auto const* data = find_time_zone_data(locale, time_zone); data != nullptr) {
  1658. @string_index_type@ time_zone_index = 0;
  1659. switch (style) {
  1660. case CalendarPatternStyle::Long:
  1661. time_zone_index = data->long_name;
  1662. break;
  1663. case CalendarPatternStyle::Short:
  1664. time_zone_index = data->short_name;
  1665. break;
  1666. default:
  1667. VERIFY_NOT_REACHED();
  1668. }
  1669. if (time_zone_index != 0)
  1670. return s_string_list[time_zone_index];
  1671. }
  1672. return {};
  1673. }
  1674. }
  1675. )~~~");
  1676. VERIFY(file.write(generator.as_string_view()));
  1677. }
  1678. ErrorOr<int> serenity_main(Main::Arguments arguments)
  1679. {
  1680. StringView generated_header_path;
  1681. StringView generated_implementation_path;
  1682. StringView core_path;
  1683. StringView dates_path;
  1684. Core::ArgsParser args_parser;
  1685. args_parser.add_option(generated_header_path, "Path to the Unicode locale header file to generate", "generated-header-path", 'h', "generated-header-path");
  1686. args_parser.add_option(generated_implementation_path, "Path to the Unicode locale implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
  1687. args_parser.add_option(core_path, "Path to cldr-core directory", "core-path", 'r', "core-path");
  1688. args_parser.add_option(dates_path, "Path to cldr-dates directory", "dates-path", 'd', "dates-path");
  1689. args_parser.parse(arguments);
  1690. auto open_file = [&](StringView path) -> ErrorOr<NonnullRefPtr<Core::File>> {
  1691. if (path.is_empty()) {
  1692. args_parser.print_usage(stderr, arguments.argv[0]);
  1693. return Error::from_string_literal("Must provide all command line options"sv);
  1694. }
  1695. return Core::File::open(path, Core::OpenMode::ReadWrite);
  1696. };
  1697. auto generated_header_file = TRY(open_file(generated_header_path));
  1698. auto generated_implementation_file = TRY(open_file(generated_implementation_path));
  1699. UnicodeLocaleData locale_data;
  1700. TRY(parse_all_locales(core_path, dates_path, locale_data));
  1701. generate_unicode_locale_header(generated_header_file, locale_data);
  1702. generate_unicode_locale_implementation(generated_implementation_file, locale_data);
  1703. return 0;
  1704. }