GenerateUnicodeDateTimeFormat.cpp 82 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069
  1. /*
  2. * Copyright (c) 2021, Tim Flynn <trflynn89@pm.me>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include "GeneratorUtil.h"
  7. #include <AK/AllOf.h>
  8. #include <AK/CharacterTypes.h>
  9. #include <AK/Format.h>
  10. #include <AK/GenericLexer.h>
  11. #include <AK/HashFunctions.h>
  12. #include <AK/HashMap.h>
  13. #include <AK/JsonObject.h>
  14. #include <AK/JsonParser.h>
  15. #include <AK/JsonValue.h>
  16. #include <AK/LexicalPath.h>
  17. #include <AK/SourceGenerator.h>
  18. #include <AK/String.h>
  19. #include <AK/StringBuilder.h>
  20. #include <AK/Traits.h>
  21. #include <AK/Utf8View.h>
  22. #include <LibCore/ArgsParser.h>
  23. #include <LibCore/DirIterator.h>
  24. #include <LibCore/File.h>
  25. #include <LibUnicode/DateTimeFormat.h>
  26. using StringIndexType = u16;
  27. constexpr auto s_string_index_type = "u16"sv;
  28. using CalendarPatternIndexType = u16;
  29. constexpr auto s_calendar_pattern_index_type = "u16"sv;
  30. using CalendarPatternListIndexType = u8;
  31. constexpr auto s_calendar_pattern_list_index_type = "u8"sv;
  32. using CalendarRangePatternIndexType = u16;
  33. constexpr auto s_calendar_range_pattern_index_type = "u16"sv;
  34. using CalendarRangePatternListIndexType = u8;
  35. constexpr auto s_calendar_range_pattern_list_index_type = "u8"sv;
  36. using CalendarFormatIndexType = u8;
  37. constexpr auto s_calendar_format_index_type = "u8"sv;
  38. using SymbolListIndexType = u16;
  39. constexpr auto s_symbol_list_index_type = "u16"sv;
  40. using CalendarSymbolsIndexType = u16;
  41. constexpr auto s_calendar_symbols_index_type = "u16"sv;
  42. using CalendarSymbolsListIndexType = u8;
  43. constexpr auto s_calendar_symbols_list_index_type = "u8"sv;
  44. using CalendarIndexType = u8;
  45. constexpr auto s_calendar_index_type = "u8"sv;
  46. using TimeZoneIndexType = u16;
  47. constexpr auto s_time_zone_index_type = "u16"sv;
  48. struct CalendarPattern : public Unicode::CalendarPattern {
  49. bool contains_only_date_fields() const
  50. {
  51. return !day_period.has_value() && !hour.has_value() && !minute.has_value() && !second.has_value() && !fractional_second_digits.has_value() && !time_zone_name.has_value();
  52. }
  53. bool contains_only_time_fields() const
  54. {
  55. return !weekday.has_value() && !era.has_value() && !year.has_value() && !month.has_value() && !day.has_value();
  56. }
  57. unsigned hash() const
  58. {
  59. auto hash = pair_int_hash(pattern_index, pattern12_index);
  60. hash = pair_int_hash(hash, skeleton_index);
  61. auto hash_field = [&](auto const& field) {
  62. if (field.has_value())
  63. hash = pair_int_hash(hash, static_cast<u8>(*field));
  64. else
  65. hash = pair_int_hash(hash, -1);
  66. };
  67. hash_field(era);
  68. hash_field(year);
  69. hash_field(month);
  70. hash_field(weekday);
  71. hash_field(day);
  72. hash_field(day_period);
  73. hash_field(hour);
  74. hash_field(minute);
  75. hash_field(second);
  76. hash_field(fractional_second_digits);
  77. hash_field(time_zone_name);
  78. return hash;
  79. }
  80. bool operator==(CalendarPattern const& other) const
  81. {
  82. return (skeleton_index == other.skeleton_index)
  83. && (pattern_index == other.pattern_index)
  84. && (pattern12_index == other.pattern12_index)
  85. && (era == other.era)
  86. && (year == other.year)
  87. && (month == other.month)
  88. && (weekday == other.weekday)
  89. && (day == other.day)
  90. && (day_period == other.day_period)
  91. && (hour == other.hour)
  92. && (minute == other.minute)
  93. && (second == other.second)
  94. && (fractional_second_digits == other.fractional_second_digits)
  95. && (time_zone_name == other.time_zone_name);
  96. }
  97. StringIndexType skeleton_index { 0 };
  98. StringIndexType pattern_index { 0 };
  99. StringIndexType pattern12_index { 0 };
  100. };
  101. template<>
  102. struct AK::Formatter<CalendarPattern> : Formatter<FormatString> {
  103. ErrorOr<void> format(FormatBuilder& builder, CalendarPattern const& pattern)
  104. {
  105. auto field_to_i8 = [](auto const& field) -> i8 {
  106. if (!field.has_value())
  107. return -1;
  108. return static_cast<i8>(*field);
  109. };
  110. return Formatter<FormatString>::format(builder,
  111. "{{ {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {} }}",
  112. pattern.skeleton_index,
  113. pattern.pattern_index,
  114. pattern.pattern12_index,
  115. field_to_i8(pattern.era),
  116. field_to_i8(pattern.year),
  117. field_to_i8(pattern.month),
  118. field_to_i8(pattern.weekday),
  119. field_to_i8(pattern.day),
  120. field_to_i8(pattern.day_period),
  121. field_to_i8(pattern.hour),
  122. field_to_i8(pattern.minute),
  123. field_to_i8(pattern.second),
  124. field_to_i8(pattern.fractional_second_digits),
  125. field_to_i8(pattern.time_zone_name));
  126. }
  127. };
  128. template<>
  129. struct AK::Traits<CalendarPattern> : public GenericTraits<CalendarPattern> {
  130. static unsigned hash(CalendarPattern const& c) { return c.hash(); }
  131. };
  132. struct CalendarRangePattern : public CalendarPattern {
  133. unsigned hash() const
  134. {
  135. auto hash = CalendarPattern::hash();
  136. if (field.has_value())
  137. hash = pair_int_hash(hash, static_cast<u8>(*field));
  138. hash = pair_int_hash(hash, start_range);
  139. hash = pair_int_hash(hash, separator);
  140. hash = pair_int_hash(hash, end_range);
  141. return hash;
  142. }
  143. bool operator==(CalendarRangePattern const& other) const
  144. {
  145. if (!CalendarPattern::operator==(other))
  146. return false;
  147. return (field == other.field)
  148. && (start_range == other.start_range)
  149. && (separator == other.separator)
  150. && (end_range == other.end_range);
  151. }
  152. Optional<Unicode::CalendarRangePattern::Field> field {};
  153. StringIndexType start_range { 0 };
  154. StringIndexType separator { 0 };
  155. StringIndexType end_range { 0 };
  156. };
  157. template<>
  158. struct AK::Formatter<CalendarRangePattern> : Formatter<FormatString> {
  159. ErrorOr<void> format(FormatBuilder& builder, CalendarRangePattern const& pattern)
  160. {
  161. auto field_to_i8 = [](auto const& field) -> i8 {
  162. if (!field.has_value())
  163. return -1;
  164. return static_cast<i8>(*field);
  165. };
  166. return Formatter<FormatString>::format(builder,
  167. "{{ {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {} }}",
  168. pattern.skeleton_index,
  169. field_to_i8(pattern.field),
  170. pattern.start_range,
  171. pattern.separator,
  172. pattern.end_range,
  173. field_to_i8(pattern.era),
  174. field_to_i8(pattern.year),
  175. field_to_i8(pattern.month),
  176. field_to_i8(pattern.weekday),
  177. field_to_i8(pattern.day),
  178. field_to_i8(pattern.day_period),
  179. field_to_i8(pattern.hour),
  180. field_to_i8(pattern.minute),
  181. field_to_i8(pattern.second),
  182. field_to_i8(pattern.fractional_second_digits),
  183. field_to_i8(pattern.time_zone_name));
  184. }
  185. };
  186. template<>
  187. struct AK::Traits<CalendarRangePattern> : public GenericTraits<CalendarRangePattern> {
  188. static unsigned hash(CalendarRangePattern const& c) { return c.hash(); }
  189. };
  190. struct CalendarFormat {
  191. unsigned hash() const
  192. {
  193. auto hash = pair_int_hash(full_format, long_format);
  194. hash = pair_int_hash(hash, medium_format);
  195. hash = pair_int_hash(hash, short_format);
  196. return hash;
  197. }
  198. bool operator==(CalendarFormat const& other) const
  199. {
  200. return (full_format == other.full_format)
  201. && (long_format == other.long_format)
  202. && (medium_format == other.medium_format)
  203. && (short_format == other.short_format);
  204. }
  205. CalendarPatternIndexType full_format { 0 };
  206. CalendarPatternIndexType long_format { 0 };
  207. CalendarPatternIndexType medium_format { 0 };
  208. CalendarPatternIndexType short_format { 0 };
  209. };
  210. template<>
  211. struct AK::Formatter<CalendarFormat> : Formatter<FormatString> {
  212. ErrorOr<void> format(FormatBuilder& builder, CalendarFormat const& pattern)
  213. {
  214. return Formatter<FormatString>::format(builder,
  215. "{{ {}, {}, {}, {} }}",
  216. pattern.full_format,
  217. pattern.long_format,
  218. pattern.medium_format,
  219. pattern.short_format);
  220. }
  221. };
  222. template<>
  223. struct AK::Traits<CalendarFormat> : public GenericTraits<CalendarFormat> {
  224. static unsigned hash(CalendarFormat const& c) { return c.hash(); }
  225. };
  226. using SymbolList = Vector<StringIndexType>;
  227. struct CalendarSymbols {
  228. unsigned hash() const
  229. {
  230. auto hash = pair_int_hash(narrow_symbols, short_symbols);
  231. hash = pair_int_hash(hash, long_symbols);
  232. return hash;
  233. }
  234. bool operator==(CalendarSymbols const& other) const
  235. {
  236. return (narrow_symbols == other.narrow_symbols)
  237. && (short_symbols == other.short_symbols)
  238. && (long_symbols == other.long_symbols);
  239. }
  240. SymbolListIndexType narrow_symbols { 0 };
  241. SymbolListIndexType short_symbols { 0 };
  242. SymbolListIndexType long_symbols { 0 };
  243. };
  244. template<>
  245. struct AK::Formatter<CalendarSymbols> : Formatter<FormatString> {
  246. ErrorOr<void> format(FormatBuilder& builder, CalendarSymbols const& symbols)
  247. {
  248. return Formatter<FormatString>::format(builder,
  249. "{{ {}, {}, {} }}",
  250. symbols.narrow_symbols,
  251. symbols.short_symbols,
  252. symbols.long_symbols);
  253. }
  254. };
  255. template<>
  256. struct AK::Traits<CalendarSymbols> : public GenericTraits<CalendarSymbols> {
  257. static unsigned hash(CalendarSymbols const& c) { return c.hash(); }
  258. };
  259. using CalendarPatternList = Vector<CalendarPatternIndexType>;
  260. using CalendarRangePatternList = Vector<CalendarRangePatternIndexType>;
  261. using CalendarSymbolsList = Vector<CalendarSymbolsIndexType>;
  262. struct Calendar {
  263. unsigned hash() const
  264. {
  265. auto hash = int_hash(calendar);
  266. hash = pair_int_hash(hash, date_formats);
  267. hash = pair_int_hash(hash, time_formats);
  268. hash = pair_int_hash(hash, date_time_formats);
  269. hash = pair_int_hash(hash, available_formats);
  270. hash = pair_int_hash(hash, default_range_format);
  271. hash = pair_int_hash(hash, range_formats);
  272. hash = pair_int_hash(hash, range12_formats);
  273. hash = pair_int_hash(hash, symbols);
  274. return hash;
  275. }
  276. bool operator==(Calendar const& other) const
  277. {
  278. return (calendar == other.calendar)
  279. && (date_formats == other.date_formats)
  280. && (time_formats == other.time_formats)
  281. && (date_time_formats == other.date_time_formats)
  282. && (available_formats == other.available_formats)
  283. && (default_range_format == other.default_range_format)
  284. && (range_formats == other.range_formats)
  285. && (range12_formats == other.range12_formats)
  286. && (symbols == other.symbols);
  287. }
  288. StringIndexType calendar { 0 };
  289. CalendarFormatIndexType date_formats { 0 };
  290. CalendarFormatIndexType time_formats { 0 };
  291. CalendarFormatIndexType date_time_formats { 0 };
  292. CalendarPatternListIndexType available_formats { 0 };
  293. CalendarRangePatternIndexType default_range_format { 0 };
  294. CalendarRangePatternListIndexType range_formats { 0 };
  295. CalendarRangePatternListIndexType range12_formats { 0 };
  296. CalendarSymbolsListIndexType symbols { 0 };
  297. };
  298. template<>
  299. struct AK::Formatter<Calendar> : Formatter<FormatString> {
  300. ErrorOr<void> format(FormatBuilder& builder, Calendar const& calendar)
  301. {
  302. return Formatter<FormatString>::format(builder,
  303. "{{ {}, {}, {}, {}, {}, {}, {}, {}, {} }}",
  304. calendar.calendar,
  305. calendar.date_formats,
  306. calendar.time_formats,
  307. calendar.date_time_formats,
  308. calendar.available_formats,
  309. calendar.default_range_format,
  310. calendar.range_formats,
  311. calendar.range12_formats,
  312. calendar.symbols);
  313. }
  314. };
  315. template<>
  316. struct AK::Traits<Calendar> : public GenericTraits<Calendar> {
  317. static unsigned hash(Calendar const& c) { return c.hash(); }
  318. };
  319. struct TimeZone {
  320. unsigned hash() const
  321. {
  322. auto hash = int_hash(time_zone);
  323. hash = pair_int_hash(hash, long_name);
  324. hash = pair_int_hash(hash, short_name);
  325. return hash;
  326. }
  327. bool operator==(TimeZone const& other) const
  328. {
  329. return (time_zone == other.time_zone)
  330. && (long_name == other.long_name)
  331. && (short_name == other.short_name);
  332. }
  333. StringIndexType time_zone { 0 };
  334. StringIndexType long_name { 0 };
  335. StringIndexType short_name { 0 };
  336. };
  337. template<>
  338. struct AK::Formatter<TimeZone> : Formatter<FormatString> {
  339. ErrorOr<void> format(FormatBuilder& builder, TimeZone const& time_zone)
  340. {
  341. return Formatter<FormatString>::format(builder,
  342. "{{ {}, {}, {} }}",
  343. time_zone.time_zone,
  344. time_zone.long_name,
  345. time_zone.short_name);
  346. }
  347. };
  348. template<>
  349. struct AK::Traits<TimeZone> : public GenericTraits<TimeZone> {
  350. static unsigned hash(TimeZone const& t) { return t.hash(); }
  351. };
  352. struct DayPeriod {
  353. Unicode::DayPeriod day_period {};
  354. u8 begin { 0 };
  355. u8 end { 0 };
  356. };
  357. struct Locale {
  358. HashMap<String, CalendarIndexType> calendars;
  359. HashMap<String, TimeZoneIndexType> time_zones;
  360. Vector<DayPeriod> day_periods;
  361. };
  362. struct UnicodeLocaleData {
  363. UniqueStringStorage<StringIndexType> unique_strings;
  364. UniqueStorage<CalendarPattern, CalendarPatternIndexType> unique_patterns;
  365. UniqueStorage<CalendarPatternList, CalendarPatternListIndexType> unique_pattern_lists;
  366. UniqueStorage<CalendarRangePattern, CalendarRangePatternIndexType> unique_range_patterns;
  367. UniqueStorage<CalendarRangePatternList, CalendarRangePatternListIndexType> unique_range_pattern_lists;
  368. UniqueStorage<CalendarFormat, CalendarFormatIndexType> unique_formats;
  369. UniqueStorage<SymbolList, SymbolListIndexType> unique_symbol_lists;
  370. UniqueStorage<CalendarSymbols, CalendarSymbolsIndexType> unique_calendar_symbols;
  371. UniqueStorage<CalendarSymbolsList, CalendarSymbolsListIndexType> unique_calendar_symbols_lists;
  372. UniqueStorage<Calendar, CalendarIndexType> unique_calendars;
  373. UniqueStorage<TimeZone, TimeZoneIndexType> unique_time_zones;
  374. HashMap<String, Locale> locales;
  375. HashMap<String, Vector<Unicode::HourCycle>> hour_cycles;
  376. Vector<String> hour_cycle_regions;
  377. HashMap<String, StringIndexType> meta_zones;
  378. Vector<String> time_zones;
  379. Vector<String> calendars;
  380. Vector<Alias> calendar_aliases {
  381. // FIXME: Aliases should come from BCP47. See: https://unicode-org.atlassian.net/browse/CLDR-15158
  382. { "gregorian"sv, "gregory"sv },
  383. };
  384. };
  385. static Optional<Unicode::DayPeriod> day_period_from_string(StringView day_period)
  386. {
  387. if (day_period == "am"sv)
  388. return Unicode::DayPeriod::AM;
  389. if (day_period == "pm"sv)
  390. return Unicode::DayPeriod::PM;
  391. if (day_period == "morning1"sv)
  392. return Unicode::DayPeriod::Morning1;
  393. if (day_period == "morning2"sv)
  394. return Unicode::DayPeriod::Morning2;
  395. if (day_period == "afternoon1"sv)
  396. return Unicode::DayPeriod::Afternoon1;
  397. if (day_period == "afternoon2"sv)
  398. return Unicode::DayPeriod::Afternoon2;
  399. if (day_period == "evening1"sv)
  400. return Unicode::DayPeriod::Evening1;
  401. if (day_period == "evening2"sv)
  402. return Unicode::DayPeriod::Evening2;
  403. if (day_period == "night1"sv)
  404. return Unicode::DayPeriod::Night1;
  405. if (day_period == "night2"sv)
  406. return Unicode::DayPeriod::Night2;
  407. return {};
  408. };
  409. static ErrorOr<void> parse_hour_cycles(String core_path, UnicodeLocaleData& locale_data)
  410. {
  411. // https://unicode.org/reports/tr35/tr35-dates.html#Time_Data
  412. LexicalPath time_data_path(move(core_path));
  413. time_data_path = time_data_path.append("supplemental"sv);
  414. time_data_path = time_data_path.append("timeData.json"sv);
  415. auto time_data_file = TRY(Core::File::open(time_data_path.string(), Core::OpenMode::ReadOnly));
  416. auto time_data = TRY(JsonValue::from_string(time_data_file->read_all()));
  417. auto const& supplemental_object = time_data.as_object().get("supplemental"sv);
  418. auto const& time_data_object = supplemental_object.as_object().get("timeData"sv);
  419. auto parse_hour_cycle = [](StringView hour_cycle) -> Optional<Unicode::HourCycle> {
  420. if (hour_cycle == "h"sv)
  421. return Unicode::HourCycle::H12;
  422. if (hour_cycle == "H"sv)
  423. return Unicode::HourCycle::H23;
  424. if (hour_cycle == "K"sv)
  425. return Unicode::HourCycle::H11;
  426. if (hour_cycle == "k"sv)
  427. return Unicode::HourCycle::H24;
  428. return {};
  429. };
  430. time_data_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
  431. auto allowed_hour_cycles_string = value.as_object().get("_allowed"sv).as_string();
  432. auto allowed_hour_cycles = allowed_hour_cycles_string.split_view(' ');
  433. Vector<Unicode::HourCycle> hour_cycles;
  434. for (auto allowed_hour_cycle : allowed_hour_cycles) {
  435. if (auto hour_cycle = parse_hour_cycle(allowed_hour_cycle); hour_cycle.has_value())
  436. hour_cycles.append(*hour_cycle);
  437. }
  438. locale_data.hour_cycles.set(key, move(hour_cycles));
  439. if (!locale_data.hour_cycle_regions.contains_slow(key))
  440. locale_data.hour_cycle_regions.append(key);
  441. });
  442. return {};
  443. };
  444. static ErrorOr<void> parse_meta_zones(String core_path, UnicodeLocaleData& locale_data)
  445. {
  446. // https://unicode.org/reports/tr35/tr35-dates.html#Metazones
  447. LexicalPath meta_zone_path(move(core_path));
  448. meta_zone_path = meta_zone_path.append("supplemental"sv);
  449. meta_zone_path = meta_zone_path.append("metaZones.json"sv);
  450. auto meta_zone_file = TRY(Core::File::open(meta_zone_path.string(), Core::OpenMode::ReadOnly));
  451. auto meta_zone = TRY(JsonValue::from_string(meta_zone_file->read_all()));
  452. auto const& supplemental_object = meta_zone.as_object().get("supplemental"sv);
  453. auto const& meta_zone_object = supplemental_object.as_object().get("metaZones"sv);
  454. auto const& meta_zone_array = meta_zone_object.as_object().get("metazones"sv);
  455. meta_zone_array.as_array().for_each([&](JsonValue const& value) {
  456. auto const& mapping = value.as_object().get("mapZone"sv);
  457. auto const& meta_zone = mapping.as_object().get("_other"sv);
  458. auto const& golden_zone = mapping.as_object().get("_type"sv);
  459. auto golden_zone_index = locale_data.unique_strings.ensure(golden_zone.as_string());
  460. locale_data.meta_zones.set(meta_zone.as_string(), golden_zone_index);
  461. });
  462. // UTC does not appear in metaZones.json. Define it for convenience so other parsers don't need to check for its existence.
  463. locale_data.meta_zones.set("UTC"sv, locale_data.unique_strings.ensure("UTC"sv));
  464. return {};
  465. };
  466. static constexpr auto is_char(char ch)
  467. {
  468. return [ch](auto c) { return c == ch; };
  469. }
  470. // For patterns that are 12-hour aware, we need to generate two patterns: one with the day period
  471. // (e.g. {ampm}) in the pattern, and one without the day period. We need to take care to remove
  472. // extra spaces around the day period. Some example expected removals:
  473. //
  474. // "{hour}:{minute} {ampm}" becomes "{hour}:{minute}" (remove the space before {ampm})
  475. // "{ampm} {hour}" becomes "{hour}" (remove the space after {ampm})
  476. // "{hour}:{minute} {ampm} {timeZoneName}" becomes "{hour}:{minute} {timeZoneName}" (remove one of the spaces around {ampm})
  477. static String remove_period_from_pattern(String pattern)
  478. {
  479. for (auto remove : AK::Array { "({ampm})"sv, "{ampm}"sv, "({dayPeriod})"sv, "{dayPeriod}"sv }) {
  480. auto index = pattern.find(remove);
  481. if (!index.has_value())
  482. continue;
  483. constexpr u32 space = ' ';
  484. constexpr u32 open = '{';
  485. constexpr u32 close = '}';
  486. Utf8View utf8_pattern { pattern };
  487. Optional<u32> before_removal;
  488. Optional<u32> after_removal;
  489. for (auto it = utf8_pattern.begin(); utf8_pattern.byte_offset_of(it) < *index; ++it)
  490. before_removal = *it;
  491. if (auto it = utf8_pattern.iterator_at_byte_offset(*index + remove.length()); it != utf8_pattern.end())
  492. after_removal = *it;
  493. if ((before_removal == space) && (after_removal != open)) {
  494. pattern = String::formatted("{}{}",
  495. pattern.substring_view(0, *index - 1),
  496. pattern.substring_view(*index + remove.length()));
  497. } else if ((after_removal == space) && (before_removal != close)) {
  498. pattern = String::formatted("{}{}",
  499. pattern.substring_view(0, *index),
  500. pattern.substring_view(*index + remove.length() + 1));
  501. } else {
  502. pattern = String::formatted("{}{}",
  503. pattern.substring_view(0, *index),
  504. pattern.substring_view(*index + remove.length()));
  505. }
  506. }
  507. return pattern;
  508. }
  509. static Optional<CalendarPattern> parse_date_time_pattern_raw(String pattern, String skeleton, UnicodeLocaleData& locale_data)
  510. {
  511. // https://unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table
  512. using Unicode::CalendarPatternStyle;
  513. CalendarPattern format {};
  514. if (!skeleton.is_empty())
  515. format.skeleton_index = locale_data.unique_strings.ensure(move(skeleton));
  516. GenericLexer lexer { pattern };
  517. StringBuilder builder;
  518. bool hour12 { false };
  519. while (!lexer.is_eof()) {
  520. // Literal strings enclosed by quotes are to be appended to the pattern as-is without further
  521. // processing (this just avoids conflicts with the patterns below).
  522. if (lexer.next_is(is_quote)) {
  523. builder.append(lexer.consume_quoted_string());
  524. continue;
  525. }
  526. auto starting_char = lexer.peek();
  527. auto segment = lexer.consume_while([&](char ch) { return ch == starting_char; });
  528. // Era
  529. if (all_of(segment, is_char('G'))) {
  530. builder.append("{era}");
  531. if (segment.length() <= 3)
  532. format.era = CalendarPatternStyle::Short;
  533. else if (segment.length() == 4)
  534. format.era = CalendarPatternStyle::Long;
  535. else
  536. format.era = CalendarPatternStyle::Narrow;
  537. }
  538. // Year
  539. else if (all_of(segment, is_any_of("yYuUr"sv))) {
  540. builder.append("{year}");
  541. if (segment.length() == 2)
  542. format.year = CalendarPatternStyle::TwoDigit;
  543. else
  544. format.year = CalendarPatternStyle::Numeric;
  545. }
  546. // Quarter
  547. else if (all_of(segment, is_any_of("qQ"sv))) {
  548. // Intl.DateTimeFormat does not support quarter formatting, so drop these patterns.
  549. return {};
  550. }
  551. // Month
  552. else if (all_of(segment, is_any_of("ML"sv))) {
  553. builder.append("{month}");
  554. if (segment.length() == 1)
  555. format.month = CalendarPatternStyle::Numeric;
  556. else if (segment.length() == 2)
  557. format.month = CalendarPatternStyle::TwoDigit;
  558. else if (segment.length() == 3)
  559. format.month = CalendarPatternStyle::Short;
  560. else if (segment.length() == 4)
  561. format.month = CalendarPatternStyle::Long;
  562. else if (segment.length() == 5)
  563. format.month = CalendarPatternStyle::Narrow;
  564. } else if (all_of(segment, is_char('l'))) {
  565. // Using 'l' for month formatting is deprecated by TR-35, ensure it is not used.
  566. return {};
  567. }
  568. // Week
  569. else if (all_of(segment, is_any_of("wW"sv))) {
  570. // Intl.DateTimeFormat does not support week formatting, so drop these patterns.
  571. return {};
  572. }
  573. // Day
  574. else if (all_of(segment, is_char('d'))) {
  575. builder.append("{day}");
  576. if (segment.length() == 1)
  577. format.day = CalendarPatternStyle::Numeric;
  578. else
  579. format.day = CalendarPatternStyle::TwoDigit;
  580. } else if (all_of(segment, is_any_of("DFg"sv))) {
  581. builder.append("{day}");
  582. format.day = CalendarPatternStyle::Numeric;
  583. }
  584. // Weekday
  585. else if (all_of(segment, is_char('E'))) {
  586. builder.append("{weekday}");
  587. if (segment.length() == 4)
  588. format.weekday = CalendarPatternStyle::Long;
  589. else if (segment.length() == 5)
  590. format.weekday = CalendarPatternStyle::Narrow;
  591. else
  592. format.weekday = CalendarPatternStyle::Short;
  593. } else if (all_of(segment, is_any_of("ec"sv))) {
  594. builder.append("{weekday}");
  595. // TR-35 defines "e", "c", and "cc" as as numeric, and "ee" as 2-digit, but those
  596. // pattern styles are not supported by Intl.DateTimeFormat.
  597. if (segment.length() <= 2)
  598. return {};
  599. if (segment.length() == 4)
  600. format.weekday = CalendarPatternStyle::Long;
  601. else if (segment.length() == 5)
  602. format.weekday = CalendarPatternStyle::Narrow;
  603. else
  604. format.weekday = CalendarPatternStyle::Short;
  605. }
  606. // Period
  607. else if (all_of(segment, is_any_of("ab"sv))) {
  608. builder.append("{ampm}");
  609. hour12 = true;
  610. } else if (all_of(segment, is_char('B'))) {
  611. builder.append("{dayPeriod}");
  612. hour12 = true;
  613. if (segment.length() == 4)
  614. format.day_period = CalendarPatternStyle::Long;
  615. else if (segment.length() == 5)
  616. format.day_period = CalendarPatternStyle::Narrow;
  617. else
  618. format.day_period = CalendarPatternStyle::Short;
  619. }
  620. // Hour
  621. else if (all_of(segment, is_any_of("hHKk"sv))) {
  622. builder.append("{hour}");
  623. if ((segment[0] == 'h') || (segment[0] == 'K'))
  624. hour12 = true;
  625. if (segment.length() == 1)
  626. format.hour = CalendarPatternStyle::Numeric;
  627. else
  628. format.hour = CalendarPatternStyle::TwoDigit;
  629. } else if (all_of(segment, is_any_of("jJC"sv))) {
  630. // TR-35 indicates these should not be used.
  631. return {};
  632. }
  633. // Minute
  634. else if (all_of(segment, is_char('m'))) {
  635. builder.append("{minute}");
  636. if (segment.length() == 1)
  637. format.minute = CalendarPatternStyle::Numeric;
  638. else
  639. format.minute = CalendarPatternStyle::TwoDigit;
  640. }
  641. // Second
  642. else if (all_of(segment, is_char('s'))) {
  643. builder.append("{second}");
  644. if (segment.length() == 1)
  645. format.second = CalendarPatternStyle::Numeric;
  646. else
  647. format.second = CalendarPatternStyle::TwoDigit;
  648. } else if (all_of(segment, is_char('S'))) {
  649. builder.append("{fractionalSecondDigits}");
  650. VERIFY(segment.length() <= 3);
  651. format.fractional_second_digits = static_cast<u8>(segment.length());
  652. } else if (all_of(segment, is_char('A'))) {
  653. // Intl.DateTimeFormat does not support millisecond formatting, so drop these patterns.
  654. return {};
  655. }
  656. // Zone
  657. else if (all_of(segment, is_any_of("zZOvVXx"))) {
  658. builder.append("{timeZoneName}");
  659. if (segment.length() < 4)
  660. format.time_zone_name = CalendarPatternStyle::Short;
  661. else
  662. format.time_zone_name = CalendarPatternStyle::Long;
  663. }
  664. // Non-patterns
  665. else {
  666. builder.append(segment);
  667. }
  668. }
  669. pattern = builder.build();
  670. if (hour12) {
  671. format.pattern = remove_period_from_pattern(pattern);
  672. format.pattern12 = move(pattern);
  673. } else {
  674. format.pattern = move(pattern);
  675. }
  676. return format;
  677. }
  678. static Optional<CalendarPatternIndexType> parse_date_time_pattern(String pattern, String skeleton, UnicodeLocaleData& locale_data)
  679. {
  680. auto format = parse_date_time_pattern_raw(move(pattern), move(skeleton), locale_data);
  681. if (!format.has_value())
  682. return {};
  683. format->pattern_index = locale_data.unique_strings.ensure(move(format->pattern));
  684. if (format->pattern12.has_value())
  685. format->pattern12_index = locale_data.unique_strings.ensure(format->pattern12.release_value());
  686. return locale_data.unique_patterns.ensure(format.release_value());
  687. }
  688. template<typename... Chars>
  689. static constexpr bool char_is_one_of(char ch, Chars&&... chars)
  690. {
  691. return ((ch == chars) || ...);
  692. }
  693. static void parse_interval_patterns(Calendar& calendar, JsonObject const& interval_formats_object, UnicodeLocaleData& locale_data)
  694. {
  695. // https://unicode.org/reports/tr35/tr35-dates.html#intervalFormats
  696. CalendarRangePatternList range_formats {};
  697. CalendarRangePatternList range12_formats {};
  698. auto name_of_field = [&](char field) {
  699. if (char_is_one_of(field, 'G'))
  700. return Unicode::CalendarRangePattern::Field::Era;
  701. if (char_is_one_of(field, 'y', 'Y', 'u', 'U', 'r'))
  702. return Unicode::CalendarRangePattern::Field::Year;
  703. if (char_is_one_of(field, 'M', 'L'))
  704. return Unicode::CalendarRangePattern::Field::Month;
  705. if (char_is_one_of(field, 'd', 'D', 'F', 'g'))
  706. return Unicode::CalendarRangePattern::Field::Day;
  707. if (char_is_one_of(field, 'a', 'b'))
  708. return Unicode::CalendarRangePattern::Field::AmPm;
  709. if (char_is_one_of(field, 'B'))
  710. return Unicode::CalendarRangePattern::Field::DayPeriod;
  711. if (char_is_one_of(field, 'h', 'H', 'K', 'k'))
  712. return Unicode::CalendarRangePattern::Field::Hour;
  713. if (char_is_one_of(field, 'm'))
  714. return Unicode::CalendarRangePattern::Field::Minute;
  715. if (char_is_one_of(field, 's'))
  716. return Unicode::CalendarRangePattern::Field::Second;
  717. if (char_is_one_of(field, 'S'))
  718. return Unicode::CalendarRangePattern::Field::FractionalSecondDigits;
  719. VERIFY_NOT_REACHED();
  720. };
  721. auto split_default_range_pattern = [&](auto skeleton, auto const& pattern) {
  722. auto start_range_end = pattern.find('}').value() + 1;
  723. auto end_range_begin = pattern.find_last('{').value();
  724. auto start_range = pattern.substring_view(0, start_range_end);
  725. auto separator = pattern.substring_view(start_range_end, end_range_begin - start_range_end);
  726. auto end_range = pattern.substring_view(end_range_begin);
  727. CalendarRangePattern format {};
  728. format.skeleton_index = locale_data.unique_strings.ensure(skeleton);
  729. format.start_range = locale_data.unique_strings.ensure(start_range);
  730. format.separator = locale_data.unique_strings.ensure(separator);
  731. format.end_range = locale_data.unique_strings.ensure(end_range);
  732. return format;
  733. };
  734. auto split_range_pattern = [&](auto skeleton, auto field, auto const& pattern, auto const& parsed_fields) {
  735. HashMap<StringView, size_t> partitions;
  736. StringView last_partition;
  737. auto begin_index = pattern.find('{');
  738. size_t end_index = 0;
  739. while (begin_index.has_value()) {
  740. end_index = pattern.find('}', *begin_index).value();
  741. auto partition = pattern.substring_view(*begin_index, end_index - *begin_index);
  742. if (partitions.contains(partition))
  743. break;
  744. partitions.set(partition, *begin_index);
  745. last_partition = partition;
  746. begin_index = pattern.find('{', end_index + 1);
  747. }
  748. VERIFY(begin_index.has_value() && !last_partition.is_empty());
  749. auto start_range_end = partitions.get(last_partition).value() + last_partition.length() + 1;
  750. auto start_range = pattern.substring_view(0, start_range_end);
  751. auto separator = pattern.substring_view(start_range_end, *begin_index - start_range_end);
  752. auto end_range = pattern.substring_view(*begin_index);
  753. CalendarRangePattern format {};
  754. format.skeleton_index = locale_data.unique_strings.ensure(skeleton);
  755. format.field = field;
  756. format.start_range = locale_data.unique_strings.ensure(start_range);
  757. format.separator = locale_data.unique_strings.ensure(separator);
  758. format.end_range = locale_data.unique_strings.ensure(end_range);
  759. format.for_each_calendar_field_zipped_with(parsed_fields, [](auto& format_field, auto const& parsed_field, auto) {
  760. format_field = parsed_field;
  761. });
  762. return format;
  763. };
  764. interval_formats_object.for_each_member([&](auto const& skeleton, auto const& value) {
  765. if (skeleton == "intervalFormatFallback"sv) {
  766. auto range_format = split_default_range_pattern(skeleton, value.as_string());
  767. calendar.default_range_format = locale_data.unique_range_patterns.ensure(move(range_format));
  768. return;
  769. }
  770. value.as_object().for_each_member([&](auto const& field, auto const& pattern) {
  771. if (field.ends_with("alt-variant"sv))
  772. return;
  773. VERIFY(field.length() == 1);
  774. auto name = name_of_field(field[0]);
  775. auto format = parse_date_time_pattern_raw(pattern.as_string(), skeleton, locale_data).release_value();
  776. auto range_format = split_range_pattern(skeleton, name, format.pattern, format);
  777. range_formats.append(locale_data.unique_range_patterns.ensure(move(range_format)));
  778. if (format.pattern12.has_value()) {
  779. auto range12_pattern = split_range_pattern(skeleton, name, *format.pattern12, format);
  780. range12_formats.append(locale_data.unique_range_patterns.ensure(move(range12_pattern)));
  781. } else {
  782. range12_formats.append(range_formats.last());
  783. }
  784. });
  785. });
  786. calendar.range_formats = locale_data.unique_range_pattern_lists.ensure(move(range_formats));
  787. calendar.range12_formats = locale_data.unique_range_pattern_lists.ensure(move(range12_formats));
  788. }
  789. static void generate_missing_patterns(Calendar& calendar, CalendarPatternList& formats, Vector<CalendarPattern> date_formats, Vector<CalendarPattern> time_formats, UnicodeLocaleData& locale_data)
  790. {
  791. // https://unicode.org/reports/tr35/tr35-dates.html#Missing_Skeleton_Fields
  792. auto replace_pattern = [&](auto format, auto time_format, auto date_format) {
  793. auto pattern = locale_data.unique_strings.get(format);
  794. auto time_pattern = locale_data.unique_strings.get(time_format);
  795. auto date_pattern = locale_data.unique_strings.get(date_format);
  796. auto new_pattern = pattern.replace("{0}", time_pattern).replace("{1}", date_pattern);
  797. return locale_data.unique_strings.ensure(move(new_pattern));
  798. };
  799. auto inject_fractional_second_digits = [&](auto format) {
  800. auto pattern = locale_data.unique_strings.get(format);
  801. auto new_pattern = pattern.replace("{second}"sv, "{second}{decimal}{fractionalSecondDigits}"sv);
  802. return locale_data.unique_strings.ensure(move(new_pattern));
  803. };
  804. auto append_if_unique = [&](auto format) {
  805. auto format_index = locale_data.unique_patterns.ensure(move(format));
  806. if (!formats.contains_slow(format_index))
  807. formats.append(format_index);
  808. };
  809. Vector<CalendarPattern> time_formats_with_fractional_second_digits;
  810. for (auto const& format : date_formats)
  811. append_if_unique(format);
  812. for (auto const& format : time_formats) {
  813. append_if_unique(format);
  814. if (format.second.has_value() && !format.fractional_second_digits.has_value()) {
  815. auto new_format = format;
  816. new_format.fractional_second_digits = 2;
  817. new_format.pattern_index = inject_fractional_second_digits(new_format.pattern_index);
  818. if (new_format.pattern12_index != 0)
  819. new_format.pattern12_index = inject_fractional_second_digits(new_format.pattern12_index);
  820. time_formats_with_fractional_second_digits.append(new_format);
  821. append_if_unique(move(new_format));
  822. }
  823. }
  824. time_formats.extend(move(time_formats_with_fractional_second_digits));
  825. for (auto const& date_format : date_formats) {
  826. auto const& date_time_formats = locale_data.unique_formats.get(calendar.date_time_formats);
  827. CalendarPatternIndexType date_time_format_index = 0;
  828. if (date_format.month == Unicode::CalendarPatternStyle::Long) {
  829. if (date_format.weekday.has_value())
  830. date_time_format_index = date_time_formats.full_format;
  831. else
  832. date_time_format_index = date_time_formats.long_format;
  833. } else if (date_format.month == Unicode::CalendarPatternStyle::Short) {
  834. date_time_format_index = date_time_formats.medium_format;
  835. } else {
  836. date_time_format_index = date_time_formats.short_format;
  837. }
  838. for (auto const& time_format : time_formats) {
  839. auto format = locale_data.unique_patterns.get(date_time_format_index);
  840. if (time_format.pattern12_index != 0)
  841. format.pattern12_index = replace_pattern(format.pattern_index, time_format.pattern12_index, date_format.pattern_index);
  842. format.pattern_index = replace_pattern(format.pattern_index, time_format.pattern_index, date_format.pattern_index);
  843. format.for_each_calendar_field_zipped_with(date_format, [](auto& field, auto const& date_field, auto) {
  844. if (date_field.has_value())
  845. field = date_field;
  846. });
  847. format.for_each_calendar_field_zipped_with(time_format, [](auto& field, auto const& time_field, auto) {
  848. if (time_field.has_value())
  849. field = time_field;
  850. });
  851. append_if_unique(move(format));
  852. }
  853. }
  854. }
  855. static void parse_calendar_symbols(Calendar& calendar, JsonObject const& calendar_object, UnicodeLocaleData& locale_data)
  856. {
  857. auto create_symbol_lists = [](size_t size) {
  858. SymbolList narrow_symbol_list;
  859. SymbolList short_symbol_list;
  860. SymbolList long_symbol_list;
  861. narrow_symbol_list.resize(size);
  862. short_symbol_list.resize(size);
  863. long_symbol_list.resize(size);
  864. return Array<SymbolList, 3> { {
  865. move(narrow_symbol_list),
  866. move(short_symbol_list),
  867. move(long_symbol_list),
  868. } };
  869. };
  870. CalendarSymbolsList symbols_list {};
  871. auto store_symbol_lists = [&](auto symbol, auto symbol_lists) {
  872. auto symbol_index = to_underlying(symbol);
  873. if (symbol_index >= symbols_list.size())
  874. symbols_list.resize(symbol_index + 1);
  875. CalendarSymbols symbols {};
  876. symbols.narrow_symbols = locale_data.unique_symbol_lists.ensure(move(symbol_lists[0]));
  877. symbols.short_symbols = locale_data.unique_symbol_lists.ensure(move(symbol_lists[1]));
  878. symbols.long_symbols = locale_data.unique_symbol_lists.ensure(move(symbol_lists[2]));
  879. auto calendar_symbols_index = locale_data.unique_calendar_symbols.ensure(move(symbols));
  880. symbols_list[symbol_index] = calendar_symbols_index;
  881. };
  882. auto parse_era_symbols = [&](auto const& symbols_object) {
  883. auto const& narrow_symbols = symbols_object.get("eraNarrow"sv).as_object();
  884. auto const& short_symbols = symbols_object.get("eraAbbr"sv).as_object();
  885. auto const& long_symbols = symbols_object.get("eraNames"sv).as_object();
  886. auto symbol_lists = create_symbol_lists(2);
  887. auto append_symbol = [&](auto& symbols, auto const& key, auto symbol) {
  888. if (auto key_index = key.to_uint(); key_index.has_value())
  889. symbols[*key_index] = locale_data.unique_strings.ensure(move(symbol));
  890. };
  891. narrow_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  892. append_symbol(symbol_lists[0], key, value.as_string());
  893. });
  894. short_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  895. append_symbol(symbol_lists[1], key, value.as_string());
  896. });
  897. long_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  898. append_symbol(symbol_lists[2], key, value.as_string());
  899. });
  900. store_symbol_lists(Unicode::CalendarSymbol::Era, move(symbol_lists));
  901. };
  902. auto parse_month_symbols = [&](auto const& symbols_object) {
  903. auto const& narrow_symbols = symbols_object.get("narrow"sv).as_object();
  904. auto const& short_symbols = symbols_object.get("abbreviated"sv).as_object();
  905. auto const& long_symbols = symbols_object.get("wide"sv).as_object();
  906. auto symbol_lists = create_symbol_lists(12);
  907. auto append_symbol = [&](auto& symbols, auto const& key, auto symbol) {
  908. auto key_index = key.to_uint().value() - 1;
  909. symbols[key_index] = locale_data.unique_strings.ensure(move(symbol));
  910. };
  911. narrow_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  912. append_symbol(symbol_lists[0], key, value.as_string());
  913. });
  914. short_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  915. append_symbol(symbol_lists[1], key, value.as_string());
  916. });
  917. long_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  918. append_symbol(symbol_lists[2], key, value.as_string());
  919. });
  920. store_symbol_lists(Unicode::CalendarSymbol::Month, move(symbol_lists));
  921. };
  922. auto parse_weekday_symbols = [&](auto const& symbols_object) {
  923. auto const& narrow_symbols = symbols_object.get("narrow"sv).as_object();
  924. auto const& short_symbols = symbols_object.get("abbreviated"sv).as_object();
  925. auto const& long_symbols = symbols_object.get("wide"sv).as_object();
  926. auto symbol_lists = create_symbol_lists(7);
  927. auto append_symbol = [&](auto& symbols, auto const& key, auto symbol) {
  928. if (key == "sun"sv)
  929. symbols[to_underlying(Unicode::Weekday::Sunday)] = locale_data.unique_strings.ensure(move(symbol));
  930. else if (key == "mon"sv)
  931. symbols[to_underlying(Unicode::Weekday::Monday)] = locale_data.unique_strings.ensure(move(symbol));
  932. else if (key == "tue"sv)
  933. symbols[to_underlying(Unicode::Weekday::Tuesday)] = locale_data.unique_strings.ensure(move(symbol));
  934. else if (key == "wed"sv)
  935. symbols[to_underlying(Unicode::Weekday::Wednesday)] = locale_data.unique_strings.ensure(move(symbol));
  936. else if (key == "thu"sv)
  937. symbols[to_underlying(Unicode::Weekday::Thursday)] = locale_data.unique_strings.ensure(move(symbol));
  938. else if (key == "fri"sv)
  939. symbols[to_underlying(Unicode::Weekday::Friday)] = locale_data.unique_strings.ensure(move(symbol));
  940. else if (key == "sat"sv)
  941. symbols[to_underlying(Unicode::Weekday::Saturday)] = locale_data.unique_strings.ensure(move(symbol));
  942. };
  943. narrow_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  944. append_symbol(symbol_lists[0], key, value.as_string());
  945. });
  946. short_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  947. append_symbol(symbol_lists[1], key, value.as_string());
  948. });
  949. long_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  950. append_symbol(symbol_lists[2], key, value.as_string());
  951. });
  952. store_symbol_lists(Unicode::CalendarSymbol::Weekday, move(symbol_lists));
  953. };
  954. auto parse_day_period_symbols = [&](auto const& symbols_object) {
  955. auto const& narrow_symbols = symbols_object.get("narrow"sv).as_object();
  956. auto const& short_symbols = symbols_object.get("abbreviated"sv).as_object();
  957. auto const& long_symbols = symbols_object.get("wide"sv).as_object();
  958. auto symbol_lists = create_symbol_lists(10);
  959. auto append_symbol = [&](auto& symbols, auto const& key, auto symbol) {
  960. if (auto day_period = day_period_from_string(key); day_period.has_value())
  961. symbols[to_underlying(*day_period)] = locale_data.unique_strings.ensure(move(symbol));
  962. };
  963. narrow_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  964. append_symbol(symbol_lists[0], key, value.as_string());
  965. });
  966. short_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  967. append_symbol(symbol_lists[1], key, value.as_string());
  968. });
  969. long_symbols.for_each_member([&](auto const& key, JsonValue const& value) {
  970. append_symbol(symbol_lists[2], key, value.as_string());
  971. });
  972. store_symbol_lists(Unicode::CalendarSymbol::DayPeriod, move(symbol_lists));
  973. };
  974. parse_era_symbols(calendar_object.get("eras"sv).as_object());
  975. parse_month_symbols(calendar_object.get("months"sv).as_object().get("format"sv).as_object());
  976. parse_weekday_symbols(calendar_object.get("days"sv).as_object().get("format"sv).as_object());
  977. parse_day_period_symbols(calendar_object.get("dayPeriods"sv).as_object().get("format"sv).as_object());
  978. calendar.symbols = locale_data.unique_calendar_symbols_lists.ensure(move(symbols_list));
  979. }
  980. static ErrorOr<void> parse_calendars(String locale_calendars_path, UnicodeLocaleData& locale_data, Locale& locale)
  981. {
  982. LexicalPath calendars_path(move(locale_calendars_path));
  983. if (!calendars_path.basename().starts_with("ca-"sv))
  984. return {};
  985. auto calendars_file = TRY(Core::File::open(calendars_path.string(), Core::OpenMode::ReadOnly));
  986. auto calendars = TRY(JsonValue::from_string(calendars_file->read_all()));
  987. auto const& main_object = calendars.as_object().get("main"sv);
  988. auto const& locale_object = main_object.as_object().get(calendars_path.parent().basename());
  989. auto const& dates_object = locale_object.as_object().get("dates"sv);
  990. auto const& calendars_object = dates_object.as_object().get("calendars"sv);
  991. auto create_calendar = [&](auto const& calendar) {
  992. auto calendar_index = locale_data.unique_strings.ensure(calendar);
  993. return Calendar { .calendar = calendar_index };
  994. };
  995. auto parse_patterns = [&](auto const& patterns_object, auto const& skeletons_object, Vector<CalendarPattern>* patterns) {
  996. auto parse_pattern = [&](auto name) {
  997. auto format = patterns_object.get(name);
  998. auto skeleton = skeletons_object.get(name);
  999. auto format_index = parse_date_time_pattern(format.as_string(), skeleton.as_string_or(String::empty()), locale_data).value();
  1000. if (patterns)
  1001. patterns->append(locale_data.unique_patterns.get(format_index));
  1002. return format_index;
  1003. };
  1004. CalendarFormat formats {};
  1005. formats.full_format = parse_pattern("full"sv);
  1006. formats.long_format = parse_pattern("long"sv);
  1007. formats.medium_format = parse_pattern("medium"sv);
  1008. formats.short_format = parse_pattern("short"sv);
  1009. return locale_data.unique_formats.ensure(move(formats));
  1010. };
  1011. calendars_object.as_object().for_each_member([&](auto const& calendar_name, JsonValue const& value) {
  1012. // The generic calendar is not a supported Unicode calendar key, so skip it:
  1013. // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/Locale/calendar#unicode_calendar_keys
  1014. if (calendar_name == "generic"sv)
  1015. return;
  1016. auto calendar = create_calendar(calendar_name);
  1017. CalendarPatternList available_formats {};
  1018. if (!locale_data.calendars.contains_slow(calendar_name))
  1019. locale_data.calendars.append(calendar_name);
  1020. Vector<CalendarPattern> date_formats;
  1021. Vector<CalendarPattern> time_formats;
  1022. auto const& date_formats_object = value.as_object().get("dateFormats"sv);
  1023. auto const& date_skeletons_object = value.as_object().get("dateSkeletons"sv);
  1024. calendar.date_formats = parse_patterns(date_formats_object.as_object(), date_skeletons_object.as_object(), &date_formats);
  1025. auto const& time_formats_object = value.as_object().get("timeFormats"sv);
  1026. auto const& time_skeletons_object = value.as_object().get("timeSkeletons"sv);
  1027. calendar.time_formats = parse_patterns(time_formats_object.as_object(), time_skeletons_object.as_object(), &time_formats);
  1028. auto const& date_time_formats_object = value.as_object().get("dateTimeFormats"sv);
  1029. calendar.date_time_formats = parse_patterns(date_time_formats_object.as_object(), JsonObject {}, nullptr);
  1030. auto const& available_formats_object = date_time_formats_object.as_object().get("availableFormats"sv);
  1031. available_formats_object.as_object().for_each_member([&](auto const& skeleton, JsonValue const& pattern) {
  1032. auto pattern_index = parse_date_time_pattern(pattern.as_string(), skeleton, locale_data);
  1033. if (!pattern_index.has_value())
  1034. return;
  1035. auto const& format = locale_data.unique_patterns.get(*pattern_index);
  1036. if (format.contains_only_date_fields())
  1037. date_formats.append(format);
  1038. else if (format.contains_only_time_fields())
  1039. time_formats.append(format);
  1040. if (!available_formats.contains_slow(*pattern_index))
  1041. available_formats.append(*pattern_index);
  1042. });
  1043. auto const& interval_formats_object = date_time_formats_object.as_object().get("intervalFormats"sv);
  1044. parse_interval_patterns(calendar, interval_formats_object.as_object(), locale_data);
  1045. generate_missing_patterns(calendar, available_formats, move(date_formats), move(time_formats), locale_data);
  1046. parse_calendar_symbols(calendar, value.as_object(), locale_data);
  1047. calendar.available_formats = locale_data.unique_pattern_lists.ensure(move(available_formats));
  1048. locale.calendars.set(calendar_name, locale_data.unique_calendars.ensure(move(calendar)));
  1049. });
  1050. return {};
  1051. }
  1052. static ErrorOr<void> parse_time_zone_names(String locale_time_zone_names_path, UnicodeLocaleData& locale_data, Locale& locale)
  1053. {
  1054. LexicalPath time_zone_names_path(move(locale_time_zone_names_path));
  1055. time_zone_names_path = time_zone_names_path.append("timeZoneNames.json"sv);
  1056. auto time_zone_names_file = TRY(Core::File::open(time_zone_names_path.string(), Core::OpenMode::ReadOnly));
  1057. auto time_zone_names = TRY(JsonValue::from_string(time_zone_names_file->read_all()));
  1058. auto const& main_object = time_zone_names.as_object().get("main"sv);
  1059. auto const& locale_object = main_object.as_object().get(time_zone_names_path.parent().basename());
  1060. auto const& dates_object = locale_object.as_object().get("dates"sv);
  1061. auto const& time_zone_names_object = dates_object.as_object().get("timeZoneNames"sv);
  1062. auto const& meta_zone_object = time_zone_names_object.as_object().get("metazone"sv);
  1063. if (meta_zone_object.is_null())
  1064. return {};
  1065. auto parse_name = [&](StringView type, JsonObject const& meta_zone_object) -> Optional<StringIndexType> {
  1066. auto const& names = meta_zone_object.get(type);
  1067. if (!names.is_object())
  1068. return {};
  1069. auto const& daylight = names.as_object().get("daylight"sv);
  1070. if (daylight.is_string())
  1071. return locale_data.unique_strings.ensure(daylight.as_string());
  1072. auto const& standard = names.as_object().get("standard"sv);
  1073. if (standard.is_string())
  1074. return locale_data.unique_strings.ensure(standard.as_string());
  1075. return {};
  1076. };
  1077. auto parse_time_zone = [&](StringView meta_zone, JsonObject const& meta_zone_object) {
  1078. auto golden_zone = locale_data.meta_zones.get(meta_zone).value();
  1079. TimeZone time_zone { .time_zone = golden_zone };
  1080. if (auto long_name = parse_name("long"sv, meta_zone_object); long_name.has_value())
  1081. time_zone.long_name = long_name.value();
  1082. if (auto short_name = parse_name("short"sv, meta_zone_object); short_name.has_value())
  1083. time_zone.short_name = short_name.value();
  1084. auto const& time_zone_name = locale_data.unique_strings.get(golden_zone);
  1085. if (!locale_data.time_zones.contains_slow(time_zone_name))
  1086. locale_data.time_zones.append(time_zone_name);
  1087. auto time_zone_index = locale_data.unique_time_zones.ensure(move(time_zone));
  1088. locale.time_zones.set(time_zone_name, time_zone_index);
  1089. };
  1090. meta_zone_object.as_object().for_each_member([&](auto const& meta_zone, JsonValue const& value) {
  1091. parse_time_zone(meta_zone, value.as_object());
  1092. });
  1093. // The long and short names for UTC are not under the "timeZoneNames/metazone" object, but are under "timeZoneNames/zone/Etc".
  1094. auto const& zone_object = time_zone_names_object.as_object().get("zone"sv);
  1095. auto const& etc_object = zone_object.as_object().get("Etc"sv);
  1096. auto const& utc_object = etc_object.as_object().get("UTC"sv);
  1097. parse_time_zone("UTC"sv, utc_object.as_object());
  1098. return {};
  1099. }
  1100. static ErrorOr<void> parse_day_periods(String core_path, UnicodeLocaleData& locale_data)
  1101. {
  1102. // https://unicode.org/reports/tr35/tr35-dates.html#Day_Period_Rule_Sets
  1103. LexicalPath day_periods_path(move(core_path));
  1104. day_periods_path = day_periods_path.append("supplemental"sv);
  1105. day_periods_path = day_periods_path.append("dayPeriods.json"sv);
  1106. auto day_periods_file = TRY(Core::File::open(day_periods_path.string(), Core::OpenMode::ReadOnly));
  1107. auto day_periods = TRY(JsonValue::from_string(day_periods_file->read_all()));
  1108. auto const& supplemental_object = day_periods.as_object().get("supplemental"sv);
  1109. auto const& day_periods_object = supplemental_object.as_object().get("dayPeriodRuleSet"sv);
  1110. auto parse_hour = [](auto const& time) {
  1111. auto hour_end_index = time.find(':').value();
  1112. // The times are in the form "hours:minutes", but we only need the hour segment.
  1113. // TR-35 explicitly states that minutes other than :00 must not be used.
  1114. VERIFY(time.substring_view(hour_end_index) == ":00"sv);
  1115. auto hour = time.substring_view(0, hour_end_index);
  1116. return hour.template to_uint<u8>().value();
  1117. };
  1118. auto parse_day_period = [&](auto const& symbol, auto const& ranges) -> Optional<DayPeriod> {
  1119. auto day_period = day_period_from_string(symbol);
  1120. if (!day_period.has_value())
  1121. return {};
  1122. auto begin = parse_hour(ranges.get("_from"sv).as_string());
  1123. auto end = parse_hour(ranges.get("_before"sv).as_string());
  1124. return DayPeriod { *day_period, begin, end };
  1125. };
  1126. day_periods_object.as_object().for_each_member([&](auto const& language, JsonValue const& value) {
  1127. auto locale = locale_data.locales.find(language);
  1128. if (locale == locale_data.locales.end())
  1129. return;
  1130. value.as_object().for_each_member([&](auto const& symbol, JsonValue const& ranges) {
  1131. if (auto day_period = parse_day_period(symbol, ranges.as_object()); day_period.has_value())
  1132. locale->value.day_periods.append(day_period.release_value());
  1133. });
  1134. });
  1135. return {};
  1136. };
  1137. static ErrorOr<void> parse_all_locales(String core_path, String dates_path, UnicodeLocaleData& locale_data)
  1138. {
  1139. TRY(parse_hour_cycles(core_path, locale_data));
  1140. TRY(parse_meta_zones(core_path, locale_data));
  1141. auto dates_iterator = TRY(path_to_dir_iterator(move(dates_path)));
  1142. auto remove_variants_from_path = [&](String path) -> ErrorOr<String> {
  1143. auto parsed_locale = TRY(CanonicalLanguageID<StringIndexType>::parse(locale_data.unique_strings, LexicalPath::basename(path)));
  1144. StringBuilder builder;
  1145. builder.append(locale_data.unique_strings.get(parsed_locale.language));
  1146. if (auto script = locale_data.unique_strings.get(parsed_locale.script); !script.is_empty())
  1147. builder.appendff("-{}", script);
  1148. if (auto region = locale_data.unique_strings.get(parsed_locale.region); !region.is_empty())
  1149. builder.appendff("-{}", region);
  1150. return builder.build();
  1151. };
  1152. while (dates_iterator.has_next()) {
  1153. auto dates_path = TRY(next_path_from_dir_iterator(dates_iterator));
  1154. auto calendars_iterator = TRY(path_to_dir_iterator(dates_path, {}));
  1155. auto language = TRY(remove_variants_from_path(dates_path));
  1156. auto& locale = locale_data.locales.ensure(language);
  1157. while (calendars_iterator.has_next()) {
  1158. auto calendars_path = TRY(next_path_from_dir_iterator(calendars_iterator));
  1159. TRY(parse_calendars(move(calendars_path), locale_data, locale));
  1160. }
  1161. TRY(parse_time_zone_names(move(dates_path), locale_data, locale));
  1162. }
  1163. TRY(parse_day_periods(move(core_path), locale_data));
  1164. return {};
  1165. }
  1166. static String format_identifier(StringView owner, String identifier)
  1167. {
  1168. identifier = identifier.replace("-"sv, "_"sv, true);
  1169. identifier = identifier.replace("/"sv, "_"sv, true);
  1170. if (all_of(identifier, is_ascii_digit))
  1171. return String::formatted("{}_{}", owner[0], identifier);
  1172. if (is_ascii_lower_alpha(identifier[0]))
  1173. return String::formatted("{:c}{}", to_ascii_uppercase(identifier[0]), identifier.substring_view(1));
  1174. return identifier;
  1175. }
  1176. static void generate_unicode_locale_header(Core::File& file, UnicodeLocaleData& locale_data)
  1177. {
  1178. StringBuilder builder;
  1179. SourceGenerator generator { builder };
  1180. generator.append(R"~~~(
  1181. #pragma once
  1182. #include <AK/Optional.h>
  1183. #include <AK/StringView.h>
  1184. #include <LibUnicode/Forward.h>
  1185. namespace Unicode {
  1186. )~~~");
  1187. generate_enum(generator, format_identifier, "Calendar"sv, {}, locale_data.calendars, locale_data.calendar_aliases);
  1188. generate_enum(generator, format_identifier, "HourCycleRegion"sv, {}, locale_data.hour_cycle_regions);
  1189. generate_enum(generator, format_identifier, "TimeZone"sv, {}, locale_data.time_zones);
  1190. generator.append(R"~~~(
  1191. namespace Detail {
  1192. Optional<Calendar> calendar_from_string(StringView calendar);
  1193. Optional<HourCycleRegion> hour_cycle_region_from_string(StringView hour_cycle_region);
  1194. Vector<Unicode::HourCycle> get_regional_hour_cycles(StringView region);
  1195. Optional<Unicode::CalendarFormat> get_calendar_date_format(StringView locale, StringView calendar);
  1196. Optional<Unicode::CalendarFormat> get_calendar_time_format(StringView locale, StringView calendar);
  1197. Optional<Unicode::CalendarFormat> get_calendar_date_time_format(StringView locale, StringView calendar);
  1198. Vector<Unicode::CalendarPattern> get_calendar_available_formats(StringView locale, StringView calendar);
  1199. Optional<Unicode::CalendarRangePattern> get_calendar_default_range_format(StringView locale, StringView calendar);
  1200. Vector<Unicode::CalendarRangePattern> get_calendar_range_formats(StringView locale, StringView calendar, StringView skeleton);
  1201. Vector<Unicode::CalendarRangePattern> get_calendar_range12_formats(StringView locale, StringView calendar, StringView skeleton);
  1202. Optional<StringView> get_calendar_era_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Era value);
  1203. Optional<StringView> get_calendar_month_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Month value);
  1204. Optional<StringView> get_calendar_weekday_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Weekday value);
  1205. Optional<StringView> get_calendar_day_period_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::DayPeriod value);
  1206. Optional<StringView> get_calendar_day_period_symbol_for_hour(StringView locale, StringView calendar, CalendarPatternStyle style, u8 hour);
  1207. Optional<TimeZone> time_zone_from_string(StringView time_zone);
  1208. Optional<StringView> get_time_zone_name(StringView locale, StringView time_zone, CalendarPatternStyle style);
  1209. }
  1210. }
  1211. )~~~");
  1212. VERIFY(file.write(generator.as_string_view()));
  1213. }
  1214. static void generate_unicode_locale_implementation(Core::File& file, UnicodeLocaleData& locale_data)
  1215. {
  1216. StringBuilder builder;
  1217. SourceGenerator generator { builder };
  1218. generator.set("string_index_type"sv, s_string_index_type);
  1219. generator.set("calendar_pattern_index_type"sv, s_calendar_pattern_index_type);
  1220. generator.set("calendar_pattern_list_index_type"sv, s_calendar_pattern_list_index_type);
  1221. generator.set("calendar_range_pattern_index_type"sv, s_calendar_range_pattern_index_type);
  1222. generator.set("calendar_range_pattern_list_index_type"sv, s_calendar_range_pattern_list_index_type);
  1223. generator.set("calendar_format_index_type"sv, s_calendar_format_index_type);
  1224. generator.set("symbol_list_index_type"sv, s_symbol_list_index_type);
  1225. generator.set("calendar_symbols_index_type"sv, s_calendar_symbols_index_type);
  1226. generator.set("calendar_symbols_list_index_type"sv, s_calendar_symbols_list_index_type);
  1227. generator.set("calendar_index_type"sv, s_calendar_index_type);
  1228. generator.set("time_zone_index_type"sv, s_time_zone_index_type);
  1229. generator.append(R"~~~(
  1230. #include <AK/Array.h>
  1231. #include <AK/BinarySearch.h>
  1232. #include <LibUnicode/DateTimeFormat.h>
  1233. #include <LibUnicode/Locale.h>
  1234. #include <LibUnicode/UnicodeDateTimeFormat.h>
  1235. namespace Unicode::Detail {
  1236. )~~~");
  1237. locale_data.unique_strings.generate(generator);
  1238. generator.append(R"~~~(
  1239. template <typename SourceType, typename TargetType>
  1240. void convert_calendar_fields(SourceType const& source, TargetType& target)
  1241. {
  1242. if (source.era != -1)
  1243. target.era = static_cast<Unicode::CalendarPatternStyle>(source.era);
  1244. if (source.year != -1)
  1245. target.year = static_cast<Unicode::CalendarPatternStyle>(source.year);
  1246. if (source.month != -1)
  1247. target.month = static_cast<Unicode::CalendarPatternStyle>(source.month);
  1248. if (source.weekday != -1)
  1249. target.weekday = static_cast<Unicode::CalendarPatternStyle>(source.weekday);
  1250. if (source.day != -1)
  1251. target.day = static_cast<Unicode::CalendarPatternStyle>(source.day);
  1252. if (source.day_period != -1)
  1253. target.day_period = static_cast<Unicode::CalendarPatternStyle>(source.day_period);
  1254. if (source.hour != -1)
  1255. target.hour = static_cast<Unicode::CalendarPatternStyle>(source.hour);
  1256. if (source.minute != -1)
  1257. target.minute = static_cast<Unicode::CalendarPatternStyle>(source.minute);
  1258. if (source.second != -1)
  1259. target.second = static_cast<Unicode::CalendarPatternStyle>(source.second);
  1260. if (source.fractional_second_digits != -1)
  1261. target.fractional_second_digits = static_cast<u8>(source.fractional_second_digits);
  1262. if (source.time_zone_name != -1)
  1263. target.time_zone_name = static_cast<Unicode::CalendarPatternStyle>(source.time_zone_name);
  1264. }
  1265. struct CalendarPattern {
  1266. Unicode::CalendarPattern to_unicode_calendar_pattern() const {
  1267. Unicode::CalendarPattern calendar_pattern {};
  1268. calendar_pattern.skeleton = s_string_list[skeleton];
  1269. calendar_pattern.pattern = s_string_list[pattern];
  1270. if (pattern12 != 0)
  1271. calendar_pattern.pattern12 = s_string_list[pattern12];
  1272. convert_calendar_fields(*this, calendar_pattern);
  1273. return calendar_pattern;
  1274. }
  1275. @string_index_type@ skeleton { 0 };
  1276. @string_index_type@ pattern { 0 };
  1277. @string_index_type@ pattern12 { 0 };
  1278. i8 era { -1 };
  1279. i8 year { -1 };
  1280. i8 month { -1 };
  1281. i8 weekday { -1 };
  1282. i8 day { -1 };
  1283. i8 day_period { -1 };
  1284. i8 hour { -1 };
  1285. i8 minute { -1 };
  1286. i8 second { -1 };
  1287. i8 fractional_second_digits { -1 };
  1288. i8 time_zone_name { -1 };
  1289. };
  1290. struct CalendarRangePattern {
  1291. Unicode::CalendarRangePattern to_unicode_calendar_range_pattern() const {
  1292. Unicode::CalendarRangePattern calendar_range_pattern {};
  1293. if (field != -1)
  1294. calendar_range_pattern.field = static_cast<Unicode::CalendarRangePattern::Field>(field);
  1295. calendar_range_pattern.start_range = s_string_list[start_range];
  1296. calendar_range_pattern.separator = s_string_list[separator];
  1297. calendar_range_pattern.end_range = s_string_list[end_range];
  1298. convert_calendar_fields(*this, calendar_range_pattern);
  1299. return calendar_range_pattern;
  1300. }
  1301. @string_index_type@ skeleton { 0 };
  1302. i8 field { -1 };
  1303. @string_index_type@ start_range { 0 };
  1304. @string_index_type@ separator { 0 };
  1305. @string_index_type@ end_range { 0 };
  1306. i8 era { -1 };
  1307. i8 year { -1 };
  1308. i8 month { -1 };
  1309. i8 weekday { -1 };
  1310. i8 day { -1 };
  1311. i8 day_period { -1 };
  1312. i8 hour { -1 };
  1313. i8 minute { -1 };
  1314. i8 second { -1 };
  1315. i8 fractional_second_digits { -1 };
  1316. i8 time_zone_name { -1 };
  1317. };
  1318. )~~~");
  1319. locale_data.unique_patterns.generate(generator, "CalendarPattern"sv, "s_calendar_patterns"sv, 10);
  1320. locale_data.unique_pattern_lists.generate(generator, s_calendar_pattern_index_type, "s_calendar_pattern_lists"sv);
  1321. locale_data.unique_range_patterns.generate(generator, "CalendarRangePattern"sv, "s_calendar_range_patterns"sv, 10);
  1322. locale_data.unique_range_pattern_lists.generate(generator, s_calendar_range_pattern_index_type, "s_calendar_range_pattern_lists"sv);
  1323. generator.append(R"~~~(
  1324. struct CalendarFormat {
  1325. Unicode::CalendarFormat to_unicode_calendar_format() const {
  1326. Unicode::CalendarFormat calendar_format {};
  1327. calendar_format.full_format = s_calendar_patterns[full_format].to_unicode_calendar_pattern();
  1328. calendar_format.long_format = s_calendar_patterns[long_format].to_unicode_calendar_pattern();
  1329. calendar_format.medium_format = s_calendar_patterns[medium_format].to_unicode_calendar_pattern();
  1330. calendar_format.short_format = s_calendar_patterns[short_format].to_unicode_calendar_pattern();
  1331. return calendar_format;
  1332. }
  1333. @calendar_pattern_index_type@ full_format { 0 };
  1334. @calendar_pattern_index_type@ long_format { 0 };
  1335. @calendar_pattern_index_type@ medium_format { 0 };
  1336. @calendar_pattern_index_type@ short_format { 0 };
  1337. };
  1338. struct CalendarSymbols {
  1339. @symbol_list_index_type@ narrow_symbols { 0 };
  1340. @symbol_list_index_type@ short_symbols { 0 };
  1341. @symbol_list_index_type@ long_symbols { 0 };
  1342. };
  1343. struct CalendarData {
  1344. @string_index_type@ calendar { 0 };
  1345. @calendar_format_index_type@ date_formats { 0 };
  1346. @calendar_format_index_type@ time_formats { 0 };
  1347. @calendar_format_index_type@ date_time_formats { 0 };
  1348. @calendar_pattern_list_index_type@ available_formats { 0 };
  1349. @calendar_range_pattern_index_type@ default_range_format { 0 };
  1350. @calendar_range_pattern_list_index_type@ range_formats { 0 };
  1351. @calendar_range_pattern_list_index_type@ range12_formats { 0 };
  1352. @calendar_symbols_list_index_type@ symbols { 0 };
  1353. };
  1354. struct TimeZoneData {
  1355. @string_index_type@ time_zone { 0 };
  1356. @string_index_type@ long_name { 0 };
  1357. @string_index_type@ short_name { 0 };
  1358. };
  1359. struct DayPeriodData {
  1360. u8 day_period { 0 };
  1361. u8 begin { 0 };
  1362. u8 end { 0 };
  1363. };
  1364. )~~~");
  1365. locale_data.unique_formats.generate(generator, "CalendarFormat"sv, "s_calendar_formats"sv, 10);
  1366. locale_data.unique_symbol_lists.generate(generator, s_string_index_type, "s_symbol_lists"sv);
  1367. locale_data.unique_calendar_symbols.generate(generator, "CalendarSymbols"sv, "s_calendar_symbols"sv, 10);
  1368. locale_data.unique_calendar_symbols_lists.generate(generator, s_calendar_symbols_index_type, "s_calendar_symbol_lists"sv);
  1369. locale_data.unique_calendars.generate(generator, "CalendarData"sv, "s_calendars"sv, 10);
  1370. locale_data.unique_time_zones.generate(generator, "TimeZoneData"sv, "s_time_zones"sv, 30);
  1371. auto append_calendars = [&](String name, auto const& calendars) {
  1372. generator.set("name", name);
  1373. generator.set("size", String::number(calendars.size()));
  1374. generator.append(R"~~~(
  1375. static constexpr Array<@calendar_index_type@, @size@> @name@ { {)~~~");
  1376. bool first = true;
  1377. for (auto const& calendar_key : locale_data.calendars) {
  1378. auto calendar = calendars.find(calendar_key)->value;
  1379. generator.append(first ? " " : ", ");
  1380. generator.append(String::number(calendar));
  1381. first = false;
  1382. }
  1383. generator.append(" } };");
  1384. };
  1385. auto append_time_zones = [&](String name, auto const& time_zones) {
  1386. generator.set("name", name);
  1387. generator.set("size", String::number(locale_data.time_zones.size()));
  1388. generator.append(R"~~~(
  1389. static constexpr Array<@time_zone_index_type@, @size@> @name@ { {)~~~");
  1390. bool first = true;
  1391. for (auto const& time_zone_key : locale_data.time_zones) {
  1392. auto time_zone = time_zones.get(time_zone_key).value_or(0);
  1393. generator.append(first ? " " : ", ");
  1394. generator.append(String::number(time_zone));
  1395. first = false;
  1396. }
  1397. generator.append(" } };");
  1398. };
  1399. auto append_day_periods = [&](String name, auto const& day_periods) {
  1400. generator.set("name", move(name));
  1401. generator.set("size", String::number(day_periods.size()));
  1402. generator.append(R"~~~(
  1403. static constexpr Array<DayPeriodData, @size@> @name@ { {)~~~");
  1404. for (auto const& day_period : day_periods) {
  1405. generator.set("day_period", String::number(static_cast<u8>(day_period.day_period)));
  1406. generator.set("begin", String::number(day_period.begin));
  1407. generator.set("end", String::number(day_period.end));
  1408. generator.append(" { @day_period@, @begin@, @end@ },");
  1409. }
  1410. generator.append(" } };");
  1411. };
  1412. auto append_hour_cycles = [&](String name, auto const& hour_cycle_region) {
  1413. auto const& hour_cycles = locale_data.hour_cycles.find(hour_cycle_region)->value;
  1414. generator.set("name", name);
  1415. generator.set("size", String::number(hour_cycles.size()));
  1416. generator.append(R"~~~(
  1417. static constexpr Array<u8, @size@> @name@ { { )~~~");
  1418. for (auto hour_cycle : hour_cycles) {
  1419. generator.set("hour_cycle", String::number(static_cast<u8>(hour_cycle)));
  1420. generator.append("@hour_cycle@, ");
  1421. }
  1422. generator.append(" } };");
  1423. };
  1424. generate_mapping(generator, locale_data.locales, s_calendar_index_type, "s_locale_calendars"sv, "s_calendars_{}", [&](auto const& name, auto const& value) { append_calendars(name, value.calendars); });
  1425. generate_mapping(generator, locale_data.locales, s_time_zone_index_type, "s_locale_time_zones"sv, "s_time_zones_{}", [&](auto const& name, auto const& value) { append_time_zones(name, value.time_zones); });
  1426. generate_mapping(generator, locale_data.locales, "DayPeriodData"sv, "s_day_periods"sv, "s_day_periods_{}", [&](auto const& name, auto const& value) { append_day_periods(name, value.day_periods); });
  1427. generate_mapping(generator, locale_data.hour_cycle_regions, "u8"sv, "s_hour_cycles"sv, "s_hour_cycles_{}", [&](auto const& name, auto const& value) { append_hour_cycles(name, value); });
  1428. auto append_from_string = [&](StringView enum_title, StringView enum_snake, auto const& values, Vector<Alias> const& aliases = {}) {
  1429. HashValueMap<String> hashes;
  1430. hashes.ensure_capacity(values.size());
  1431. for (auto const& value : values)
  1432. hashes.set(value.hash(), format_identifier(enum_title, value));
  1433. for (auto const& alias : aliases)
  1434. hashes.set(alias.alias.hash(), format_identifier(enum_title, alias.alias));
  1435. generate_value_from_string(generator, "{}_from_string"sv, enum_title, enum_snake, move(hashes));
  1436. };
  1437. append_from_string("Calendar"sv, "calendar"sv, locale_data.calendars, locale_data.calendar_aliases);
  1438. append_from_string("HourCycleRegion"sv, "hour_cycle_region"sv, locale_data.hour_cycle_regions);
  1439. append_from_string("TimeZone"sv, "time_zone"sv, locale_data.time_zones);
  1440. generator.append(R"~~~(
  1441. Vector<Unicode::HourCycle> get_regional_hour_cycles(StringView region)
  1442. {
  1443. auto region_value = hour_cycle_region_from_string(region);
  1444. if (!region_value.has_value())
  1445. return {};
  1446. auto region_index = to_underlying(*region_value);
  1447. auto const& regional_hour_cycles = s_hour_cycles.at(region_index);
  1448. Vector<Unicode::HourCycle> hour_cycles;
  1449. hour_cycles.ensure_capacity(regional_hour_cycles.size());
  1450. for (auto hour_cycle : regional_hour_cycles)
  1451. hour_cycles.unchecked_append(static_cast<Unicode::HourCycle>(hour_cycle));
  1452. return hour_cycles;
  1453. }
  1454. static CalendarData const* find_calendar_data(StringView locale, StringView calendar)
  1455. {
  1456. auto locale_value = locale_from_string(locale);
  1457. if (!locale_value.has_value())
  1458. return nullptr;
  1459. auto calendar_value = calendar_from_string(calendar);
  1460. if (!calendar_value.has_value())
  1461. return nullptr;
  1462. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1463. size_t calendar_index = to_underlying(*calendar_value);
  1464. auto const& calendar_indices = s_locale_calendars.at(locale_index);
  1465. calendar_index = calendar_indices[calendar_index];
  1466. return &s_calendars[calendar_index];
  1467. }
  1468. Optional<Unicode::CalendarFormat> get_calendar_date_format(StringView locale, StringView calendar)
  1469. {
  1470. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1471. auto const& formats = s_calendar_formats.at(data->date_formats);
  1472. return formats.to_unicode_calendar_format();
  1473. }
  1474. return {};
  1475. }
  1476. Optional<Unicode::CalendarFormat> get_calendar_time_format(StringView locale, StringView calendar)
  1477. {
  1478. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1479. auto const& formats = s_calendar_formats.at(data->time_formats);
  1480. return formats.to_unicode_calendar_format();
  1481. }
  1482. return {};
  1483. }
  1484. Optional<Unicode::CalendarFormat> get_calendar_date_time_format(StringView locale, StringView calendar)
  1485. {
  1486. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1487. auto const& formats = s_calendar_formats.at(data->date_time_formats);
  1488. return formats.to_unicode_calendar_format();
  1489. }
  1490. return {};
  1491. }
  1492. Vector<Unicode::CalendarPattern> get_calendar_available_formats(StringView locale, StringView calendar)
  1493. {
  1494. Vector<Unicode::CalendarPattern> result {};
  1495. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1496. auto const& available_formats = s_calendar_pattern_lists.at(data->available_formats);
  1497. result.ensure_capacity(available_formats.size());
  1498. for (auto const& format : available_formats)
  1499. result.unchecked_append(s_calendar_patterns[format].to_unicode_calendar_pattern());
  1500. }
  1501. return result;
  1502. }
  1503. Optional<Unicode::CalendarRangePattern> get_calendar_default_range_format(StringView locale, StringView calendar)
  1504. {
  1505. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1506. auto const& pattern = s_calendar_range_patterns[data->default_range_format];
  1507. return pattern.to_unicode_calendar_range_pattern();
  1508. }
  1509. return {};
  1510. }
  1511. Vector<Unicode::CalendarRangePattern> get_calendar_range_formats(StringView locale, StringView calendar, StringView skeleton) {
  1512. Vector<Unicode::CalendarRangePattern> result {};
  1513. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1514. auto const& range_formats = s_calendar_range_pattern_lists.at(data->range_formats);
  1515. for (auto format : range_formats) {
  1516. auto const& pattern = s_calendar_range_patterns[format];
  1517. if (skeleton == s_string_list[pattern.skeleton])
  1518. result.append(pattern.to_unicode_calendar_range_pattern());
  1519. }
  1520. }
  1521. return result;
  1522. }
  1523. Vector<Unicode::CalendarRangePattern> get_calendar_range12_formats(StringView locale, StringView calendar, StringView skeleton) {
  1524. Vector<Unicode::CalendarRangePattern> result {};
  1525. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1526. auto const& range12_formats = s_calendar_range_pattern_lists.at(data->range12_formats);
  1527. for (auto format : range12_formats) {
  1528. auto const& pattern = s_calendar_range_patterns[format];
  1529. if (skeleton == s_string_list[pattern.skeleton])
  1530. result.append(pattern.to_unicode_calendar_range_pattern());
  1531. }
  1532. }
  1533. return result;
  1534. }
  1535. static Span<@string_index_type@ const> find_calendar_symbols(StringView locale, StringView calendar, CalendarSymbol symbol, CalendarPatternStyle style)
  1536. {
  1537. if (auto const* data = find_calendar_data(locale, calendar); data != nullptr) {
  1538. auto const& symbols_list = s_calendar_symbol_lists[data->symbols];
  1539. auto symbol_index = to_underlying(symbol);
  1540. auto calendar_symbols_index = symbols_list.at(symbol_index);
  1541. auto const& symbols = s_calendar_symbols.at(calendar_symbols_index);
  1542. @symbol_list_index_type@ symbol_list_index = 0;
  1543. switch (style) {
  1544. case CalendarPatternStyle::Narrow:
  1545. symbol_list_index = symbols.narrow_symbols;
  1546. break;
  1547. case CalendarPatternStyle::Short:
  1548. symbol_list_index = symbols.short_symbols;
  1549. break;
  1550. case CalendarPatternStyle::Long:
  1551. symbol_list_index = symbols.long_symbols;
  1552. break;
  1553. default:
  1554. VERIFY_NOT_REACHED();
  1555. }
  1556. return s_symbol_lists.at(symbol_list_index);
  1557. }
  1558. return {};
  1559. }
  1560. Optional<StringView> get_calendar_era_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Era value)
  1561. {
  1562. auto symbols = find_calendar_symbols(locale, calendar, CalendarSymbol::Era, style);
  1563. if (auto value_index = to_underlying(value); value_index < symbols.size())
  1564. return s_string_list[symbols.at(value_index)];
  1565. return {};
  1566. }
  1567. Optional<StringView> get_calendar_month_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Month value)
  1568. {
  1569. auto symbols = find_calendar_symbols(locale, calendar, CalendarSymbol::Month, style);
  1570. if (auto value_index = to_underlying(value); value_index < symbols.size())
  1571. return s_string_list[symbols.at(value_index)];
  1572. return {};
  1573. }
  1574. Optional<StringView> get_calendar_weekday_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::Weekday value)
  1575. {
  1576. auto symbols = find_calendar_symbols(locale, calendar, CalendarSymbol::Weekday, style);
  1577. if (auto value_index = to_underlying(value); value_index < symbols.size())
  1578. return s_string_list[symbols.at(value_index)];
  1579. return {};
  1580. }
  1581. Optional<StringView> get_calendar_day_period_symbol(StringView locale, StringView calendar, CalendarPatternStyle style, Unicode::DayPeriod value)
  1582. {
  1583. auto symbols = find_calendar_symbols(locale, calendar, CalendarSymbol::DayPeriod, style);
  1584. if (auto value_index = to_underlying(value); value_index < symbols.size())
  1585. return s_string_list[symbols.at(value_index)];
  1586. return {};
  1587. }
  1588. Optional<StringView> get_calendar_day_period_symbol_for_hour(StringView locale, StringView calendar, CalendarPatternStyle style, u8 hour)
  1589. {
  1590. auto locale_value = locale_from_string(locale);
  1591. if (!locale_value.has_value())
  1592. return {};
  1593. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1594. auto day_periods = s_day_periods[locale_index];
  1595. for (auto const& day_period : day_periods) {
  1596. if ((day_period.begin <= hour) && (hour < day_period.end)) {
  1597. auto period = static_cast<Unicode::DayPeriod>(day_period.day_period);
  1598. return Detail::get_calendar_day_period_symbol(locale, calendar, style, period);
  1599. }
  1600. }
  1601. // Fallback to fixed periods if the locale does not have flexible day periods.
  1602. // TR-35 states that the meaning of AM and PM does not change with locale.
  1603. if (hour < 12)
  1604. return Detail::get_calendar_day_period_symbol(locale, calendar, style, Unicode::DayPeriod::AM);
  1605. return Detail::get_calendar_day_period_symbol(locale, calendar, style, Unicode::DayPeriod::PM);
  1606. }
  1607. static TimeZoneData const* find_time_zone_data(StringView locale, StringView time_zone)
  1608. {
  1609. auto locale_value = locale_from_string(locale);
  1610. if (!locale_value.has_value())
  1611. return nullptr;
  1612. auto time_zone_value = time_zone_from_string(time_zone);
  1613. if (!time_zone_value.has_value())
  1614. return nullptr;
  1615. auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
  1616. size_t time_zone_index = to_underlying(*time_zone_value);
  1617. auto const& time_zone_indices = s_locale_time_zones.at(locale_index);
  1618. time_zone_index = time_zone_indices.at(time_zone_index);
  1619. return &s_time_zones[time_zone_index];
  1620. }
  1621. Optional<StringView> get_time_zone_name(StringView locale, StringView time_zone, CalendarPatternStyle style)
  1622. {
  1623. if (auto const* data = find_time_zone_data(locale, time_zone); data != nullptr) {
  1624. @string_index_type@ time_zone_index = 0;
  1625. switch (style) {
  1626. case CalendarPatternStyle::Long:
  1627. time_zone_index = data->long_name;
  1628. break;
  1629. case CalendarPatternStyle::Short:
  1630. time_zone_index = data->short_name;
  1631. break;
  1632. default:
  1633. VERIFY_NOT_REACHED();
  1634. }
  1635. if (time_zone_index != 0)
  1636. return s_string_list[time_zone_index];
  1637. }
  1638. return {};
  1639. }
  1640. }
  1641. )~~~");
  1642. VERIFY(file.write(generator.as_string_view()));
  1643. }
  1644. ErrorOr<int> serenity_main(Main::Arguments arguments)
  1645. {
  1646. StringView generated_header_path;
  1647. StringView generated_implementation_path;
  1648. StringView core_path;
  1649. StringView dates_path;
  1650. Core::ArgsParser args_parser;
  1651. args_parser.add_option(generated_header_path, "Path to the Unicode locale header file to generate", "generated-header-path", 'h', "generated-header-path");
  1652. args_parser.add_option(generated_implementation_path, "Path to the Unicode locale implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
  1653. args_parser.add_option(core_path, "Path to cldr-core directory", "core-path", 'r', "core-path");
  1654. args_parser.add_option(dates_path, "Path to cldr-dates directory", "dates-path", 'd', "dates-path");
  1655. args_parser.parse(arguments);
  1656. auto open_file = [&](StringView path) -> ErrorOr<NonnullRefPtr<Core::File>> {
  1657. if (path.is_empty()) {
  1658. args_parser.print_usage(stderr, arguments.argv[0]);
  1659. return Error::from_string_literal("Must provide all command line options"sv);
  1660. }
  1661. return Core::File::open(path, Core::OpenMode::ReadWrite);
  1662. };
  1663. auto generated_header_file = TRY(open_file(generated_header_path));
  1664. auto generated_implementation_file = TRY(open_file(generated_implementation_path));
  1665. UnicodeLocaleData locale_data;
  1666. TRY(parse_all_locales(core_path, dates_path, locale_data));
  1667. generate_unicode_locale_header(generated_header_file, locale_data);
  1668. generate_unicode_locale_implementation(generated_implementation_file, locale_data);
  1669. return 0;
  1670. }